Merged branch test into main

.pylintrc

@@ -69,7 +69,8 @@ disable=W0703,
         R0401,
         W0611,
         C0415,
-        W0602
+        W0602,
+        R0913
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.1.5 - 2021-10-12]
+
+### Added
+
+- Added some action loaders on Details page
+- Added links to the list of target directories
+
+### Fixed
+
+- Fixed video processing broken by previous update.
+- Fixed pip version parsing, install now works with last pip.
+- Fixed processing files from other NC instances and remote shares.
+
 ## [0.1.4 - 2021-10-09]
 
 ### Fixed

appinfo/info.xml

@@ -17,7 +17,7 @@ This app allows to find duplicate or similar 📸📹 photos and videos
 Quick start guide and further information in our [Wiki](https://github.com/andrey18106/mediadc/wiki).
 	]]>
 	</description>
-	<version>0.1.4</version>
+	<version>0.1.5</version>
 	<licence>agpl</licence>
 	<author mail="[email protected]" homepage="https://github.com/andrey18106">Andrey Borysenko</author>
 	<author mail="[email protected]" homepage="https://github.com/bigcat88">Alexander Piskun</author>

lib/Service/python/db/occ_cloud.py

@@ -87,10 +87,12 @@ def occ_call(occ_task, *params, decode: bool = True) -> [bool, Union[str, bytes]
     success, result = php_call(OCC, '--no-warnings', occ_task, *params, decode=decode)
     if not success:
         return False, result
-    clear_result = re.sub(r'.*app.*require.*upgrade.*\n?', '', result, flags=re.IGNORECASE)
-    clear_result = re.sub(r'.*occ.*upgrade.*command.*\n?', '', clear_result, flags=re.IGNORECASE)
-    clear_result = clear_result.strip('\n')
-    return True, clear_result
+    if decode:
+        clear_result = re.sub(r'.*app.*require.*upgrade.*\n?', '', result, flags=re.IGNORECASE)
+        clear_result = re.sub(r'.*occ.*upgrade.*command.*\n?', '', clear_result, flags=re.IGNORECASE)
+        clear_result = clear_result.strip('\n')
+        return True, clear_result
+    return True, result
 
 
 def get_cloud_config_value(value_name: str) -> [bool, str]:

lib/Service/python/dc_videos.py

@@ -120,10 +120,10 @@ def process_video_hash(algo: str, hash_size: int, file_info: dict, data_dir: str
             full_path = get_file_full_path(data_dir, file_info['storage'], file_info['path'])
             if not full_path:
                 break
-            video_info = ffprobe_get_video_info(full_path)
+            video_info = ffprobe_get_video_info(full_path, None)
             if not video_info:
                 break
-            if not do_hash_video(algo, hash_size, video_info, file_info, full_path):
+            if not do_hash_video(algo, hash_size, video_info, file_info, full_path, None):
                 raise InvalidVideo
             return
         if file_info['size'] > remote_filesize_limit:
@@ -133,10 +133,10 @@ def process_video_hash(algo: str, hash_size: int, file_info: dict, data_dir: str
         data = request_file_from_php(file_info)
         if len(data) == 0:
             return
-        video_info = ffprobe_get_video_info(data)
+        video_info = ffprobe_get_video_info(None, data)
         if not video_info.get('fast_start', False):
             raise InvalidVideo
-        if not do_hash_video(algo, hash_size, video_info, file_info, data):
+        if not do_hash_video(algo, hash_size, video_info, file_info, None, data):
             raise InvalidVideo
     except Exception as exception_info:
         store_err_video_hash(file_info['fileid'], video_info.get('duration', 0),
@@ -146,15 +146,16 @@ def process_video_hash(algo: str, hash_size: int, file_info: dict, data_dir: str
             print(f"Exception({exception_name}): `{file_info['path']}`:\n`{str(traceback.format_exc())}`")
 
 
-def do_hash_video(algo: str, hash_size: int, video_info: dict, file_info: dict, path_or_data) -> bool:
+def do_hash_video(algo: str, hash_size: int, video_info: dict, file_info: dict, path, data) -> bool:
+    """Accepts path(bytes/str) or data for processing in memory."""
     if video_info['duration'] < MinVideoDuration_ms:
         return False
-    first_timestamp = get_first_timestamp(video_info, path_or_data)
+    first_timestamp = get_first_timestamp(video_info, path, data)
     if first_timestamp == -1:
         return False
     frames_timestamps = build_times_for_hashes(video_info['duration'], first_timestamp)
     ex = ()
-    res = get_frames(frames_timestamps, path_or_data, *ex)
+    res = get_frames(frames_timestamps, path, data, *ex)
     if not res[0]:
         return False
     if any(len(x) == 0 for x in res[1:]):
@@ -192,19 +193,22 @@ def get_max_first_frame_time(duration_ms) -> int:
     return max_timestamp
 
 
-def get_first_timestamp(video_info: dict, data_or_filepath) -> int:
+def get_first_timestamp(video_info: dict, path, data) -> int:
+    """Accepts path(bytes/str) or data for processing in memory."""
     max_timestamp = get_max_first_frame_time(video_info['duration'])
     ffmpeg_input_params = ['-hide_banner', '-loglevel', 'fatal', '-an', '-sn', '-dn', '-to', f'{max_timestamp}ms']
-    if isinstance(data_or_filepath, str):
-        result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params, '-i', data_or_filepath,
+    if path is not None:
+        result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params, '-i', path,
                                    '-f', 'rawvideo', '-s', f'{FirstFrameResolution}x{FirstFrameResolution}',
                                    '-pix_fmt', 'rgb24', 'pipe:'
                                    )
-    else:
+    elif data is not None:
         result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params, '-i', 'pipe:0',
                                    '-f', 'rawvideo', '-s', f'{FirstFrameResolution}x{FirstFrameResolution}',
                                    '-pix_fmt', 'rgb24', 'pipe:1',
-                                   stdin_data=data_or_filepath)
+                                   stdin_data=data)
+    else:
+        raise ValueError("`path` or `data` argument must be specified.")
     if err:
         print('DEBUG:', err)
         return -1
@@ -223,22 +227,25 @@ def get_first_timestamp(video_info: dict, data_or_filepath) -> int:
     return 0
 
 
-def get_frames(timestamps: list, data_or_filepath, *ffmpeg_out_params) -> list:
+def get_frames(timestamps: list, path, data, *ffmpeg_out_params) -> list:
+    """Accepts path(bytes/str) or data for processing in memory."""
     ret = [False]
     for _ in range(len(timestamps)):
         ret.append(b'')
+    if path is None and data is None:
+        raise ValueError("`path` or `data` argument must be specified.")
     ffmpeg_input_params = ['-hide_banner', '-loglevel', 'fatal', '-an', '-sn', '-dn']
     for index, timestamp in enumerate(timestamps):
-        if isinstance(data_or_filepath, str):
+        if path is not None:
             result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params,
-                                       '-ss', f'{timestamp}ms', '-i', data_or_filepath,
+                                       '-ss', f'{timestamp}ms', '-i', path,
                                        '-f', 'image2', '-c:v', 'bmp', '-frames', '1', *ffmpeg_out_params, 'pipe:'
                                        )
         else:
             result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params,
                                        '-ss', f'{timestamp}ms', '-i', 'pipe:0',
                                        '-f', 'image2', '-c:v', 'bmp', '-frames', '1', *ffmpeg_out_params, 'pipe:1',
-                                       stdin_data=data_or_filepath)
+                                       stdin_data=data)
         if err:
             print('DEBUG:', err)
             return ret

lib/Service/python/ffmpeg_probe.py

@@ -87,21 +87,23 @@ def is_moov_at_start(std_log) -> bool:
     return False
 
 
-def ffprobe_get_video_info(path_or_data) -> dict:
-    """Accepts bytes/path. Returns {} or {duration:X ms}. If input is bytes also returns `fast_start` flag."""
-    if isinstance(path_or_data, str):
+def ffprobe_get_video_info(path, data) -> dict:
+    """Accepts path(bytes/str) or data. Returns {} or {duration:X ms}. For data input also returns `fast_start` flag."""
+    if path is not None:
         result, err = stub_call_ff('ffprobe', '-hide_banner', '-loglevel', 'fatal', '-print_format', 'json',
                                    '-show_entries', 'format=duration',
-                                   path_or_data)
-    else:
+                                   path)
+    elif data is not None:
         result, err = stub_call_ff('ffprobe', '-hide_banner', '-loglevel', 'trace', '-print_format', 'json',
                                    '-show_entries', 'format=duration', '-i', 'pipe:0',
-                                   stdin_data=path_or_data, ignore_errors=True)
+                                   stdin_data=data, ignore_errors=True)
+    else:
+        raise ValueError("`path` or `data` argument must be specified.")
     if err:
         print(err)
         return {}
     ret = ffprobe_parse_results(result)
-    if isinstance(path_or_data, bytes):
+    if path is None:
         if ret['duration'] != 0:
             ret['fast_start'] = is_moov_at_start(result.stderr)
     return ret

lib/Service/python/files.py

@@ -41,7 +41,8 @@ def get_file_data(file_info: dict, data_dir: str, remote_filesize_limit: int) ->
                 data = h_file.read()
                 return data
         except Exception as exception_info:
-            print(f"Exception during reading: {full_path}\n {str(exception_info)}")
+            print(f"Error during reading: {file_info['storage']}-{data_dir + file_info['path']}:\n"
+                  f" {str(exception_info)}")
             break
     if file_info['size'] > remote_filesize_limit:
         return b''
@@ -59,10 +60,10 @@ def request_file_from_php(file_info: dict) -> bytes:
     return err_or_data
 
 
-def get_file_full_path(data_dir: str, storage_id: int, relative_path: str) -> str:
+def get_file_full_path(data_dir: str, storage_id: int, relative_path: str) -> bytes:
     mount_point = get_storage_mount_point(storage_id)
     if not mount_point:
-        return ''
+        return b''
     return data_dir.encode('utf-8') + mount_point + relative_path.encode('utf-8')
 
 
@@ -94,15 +95,15 @@ def get_storage_info(storage_id: int):
     return {}
 
 
-def get_storage_mount_point(storage_id: int) -> str:
+def get_storage_mount_point(storage_id: int) -> bytes:
     for storage_info in StoragesInfo:
         if storage_info['numeric_id'] == storage_id:
             return storage_info['mount_point'].encode('utf-8')
-    return ''
+    return b''
 
 
-def get_storage_user_id(storage_id: int) -> str:
+def get_storage_user_id(storage_id: int) -> bytes:
     for storage_info in StoragesInfo:
         if storage_info['numeric_id'] == storage_id:
             return storage_info['user_id'].encode('utf-8')
-    return ''
+    return b''

lib/Service/python/install.py

@@ -236,7 +236,7 @@ def is_pip_present(only_global: bool = False) -> bool:
             full_reply = result.stdout.decode('utf-8')
             if PIP_DEBUG == 1:
                 print('Pip version:', full_reply)
-            m_groups = re.search(r'pip\s*(\d+\.\d+\.\d*)',
+            m_groups = re.search(r'pip\s*(\d+(\.\d+){0,2})',
                                  full_reply, flags=re.MULTILINE + re.IGNORECASE)
             if m_groups is None:
                 return False

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "mediadc",
 	"description": "Collect photo and video duplicates to save your cloud storage",
-	"version": "0.1.4",
+	"version": "0.1.5",
 	"author": "Andrey Borysenko <[email protected]>",
 	"contributors": [
 		"Andrey Borysenko <[email protected]>",

src/components/details/DetailsGroupList.vue

@@ -66,6 +66,10 @@ export default {
 			type: Boolean,
 			required: true,
 		},
+		updating: {
+			type: Boolean,
+			required: true,
+		},
 	},
 	computed: {
 		...mapGetters([
@@ -75,6 +79,7 @@ export default {
 	methods: {
 		deleteGroupFile(file) {
 			if (confirm(t('mediadc', 'Are you sure, you want delete this file?'))) {
+				this.$emit('update:updating', true)
 				axios.delete(generateUrl(`/apps/mediadc/api/v1/tasks/${file.taskId}/files/${file.detailId}/${file.fileid}`))
 					.then(res => {
 						if (res.data.success) {
@@ -83,16 +88,21 @@ export default {
 							files.splice(fileidIndex, 1)
 							this.$emit('update:files', files)
 							emit('updateTaskInfo')
-							this.$store.dispatch('setTask', res.data.task)
+							this.$store.dispatch('setTask', res.data.task).then(() => {
+								this.$emit('update:updating', false)
+							})
 						} else if ('locked' in res.data && res.data.locked) {
 							showWarning(t('mediadc', 'Wait until file loaded before deleting'))
+							this.$emit('update:updating', false)
 						} else {
 							showError(t('mediadc', 'Some error occured while deleting file'))
+							this.$emit('update:updating', false)
 						}
 					})
 					.catch(err => {
 						console.debug(err)
 						showError(t('mediadc', 'Some error occured while deleting file'))
+						this.$emit('update:updating', false)
 					})
 			}
 		},

src/components/details/DetailsListItem.vue

@@ -24,6 +24,9 @@
 
 <template>
 	<div class="details-list-item">
+		<div v-show="updating" class="action-blackout">
+			<span class="icon-loading" />
+		</div>
 		<div class="details-list-item-title">
 			<span class="icon-projects" style="margin: 0 10px 0 0;" />
 			<span class="group-info" @click="openDetailFiles(detail)">
@@ -41,7 +44,10 @@
 			<span class="icon-view-next pagination-button"
 				@click="openNextDetailFiles(detail)" />
 		</div>
-		<DetailsGroupList v-show="opened" :files="files" :loading-files="loadingFiles" />
+		<DetailsGroupList v-show="opened"
+			:files="files"
+			:loading-files="loadingFiles"
+			:updating.sync="updating" />
 	</div>
 </template>
 
@@ -73,6 +79,7 @@ export default {
 			loadingFiles: false,
 			files: [],
 			paginatedFiles: {},
+			updating: false,
 		}
 	},
 	computed: {
@@ -150,14 +157,22 @@ export default {
 		},
 		deleteTaskDetail(detail) {
 			if (confirm(t('mediadc', 'Are you sure, you want remove this group without deleting files?'))) {
+				this.updating = true
 				axios.delete(generateUrl(`/apps/mediadc/api/v1/tasks/${detail.task_id}/detail/${detail.id}`)).then(res => {
 					if (res.data.success) {
-						this.$store.dispatch('deleteDetail', detail)
-						emit('updateTaskInfo')
-						showSuccess(t('mediadc', 'Duplicate group succesffully removed'))
+						this.$store.dispatch('deleteDetail', detail).then(() => {
+							emit('updateTaskInfo')
+							showSuccess(t('mediadc', 'Duplicate group succesffully removed'))
+							this.updating = false
+						})
 					} else {
 						showError(t('mediadc', 'Some error occured while deleting duplicate group'))
+						this.updating = false
 					}
+				}).catch(err => {
+					console.debug(err)
+					showError(t('mediadc', 'Some error occured while deleting duplicate group'))
+					this.updating = false
 				})
 			}
 		},
@@ -173,6 +188,7 @@ export default {
 	align-items: center;
 	justify-content: space-between;
 	flex-direction: column;
+	position: relative;
 }
 
 .details-list-item-title {
@@ -242,4 +258,18 @@ body.theme--dark .pagination-button:active {
 .details-list-item:hover .delete-group-btn {
 	visibility: visible;
 }
+
+.action-blackout {
+	position: absolute;
+	top: 0;
+	left: 0;
+	right: 0;
+	bottom: 0;
+	background-color: rgba(0, 0, 0, 0.5);
+	z-index: 999;
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	border-radius: 5px;
+}
 </style>