From 71cd49b31bd4de4f80df158e8d3797eee52e587b Mon Sep 17 00:00:00 2001 From: Johnny Sequeira Date: Sat, 21 Sep 2024 15:06:41 -0600 Subject: [PATCH 1/9] Adding validator for files to upload on SDK --- qfieldcloud_sdk/sdk.py | 17 ++++++++++++++++- qfieldcloud_sdk/utils.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index 185d3a0..94041b3 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -13,7 +13,7 @@ from requests.adapters import HTTPAdapter, Retry from .interfaces import QfcException, QfcRequest, QfcRequestException -from .utils import calc_etag, log +from .utils import calc_etag, log, is_valid_windows_filename logger = logging.getLogger(__file__) @@ -408,6 +408,14 @@ def upload_files( local_files = self.list_local_files(project_path, filter_glob) + # Validate file names + for file in local_files: + is_valid, invalid_chars = is_valid_windows_filename(file["name"]) + if not is_valid: + raise ValueError( + f"Invalid file name: {file['name']}. Have invalid characters: {invalid_chars}" + ) + # we should always upload all package files if upload_type == FileTransferType.PACKAGE: force = True @@ -482,6 +490,13 @@ def upload_file( Returns: The response object from the upload request. """ + # Validate file name + is_valid, invalid_chars = is_valid_windows_filename(local_filename.name) + if not is_valid: + raise ValueError( + f"Invalid file name: {local_filename.name}. Have invalid characters: {invalid_chars}" + ) + with open(local_filename, "rb") as local_file: upload_file = local_file if show_progress: diff --git a/qfieldcloud_sdk/utils.py b/qfieldcloud_sdk/utils.py index a0e2ac7..5245a80 100644 --- a/qfieldcloud_sdk/utils.py +++ b/qfieldcloud_sdk/utils.py @@ -2,6 +2,8 @@ import json import os import sys +import re +from typing import Tuple def print_json(data): @@ -69,3 +71,14 @@ def calc_etag(filename: str, part_size: int = 8 * 1024 * 1024) -> str: final_md5sum = hashlib.md5(b"".join(md5sums)) return "{}-{}".format(final_md5sum.hexdigest(), len(md5sums)) + + +def is_valid_windows_filename(filename: str) -> Tuple[bool, str]: + """Check if the filename contains forbidden characters for Windows and return them.""" + # Forbidden characters in Windows + # https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names + invalid_chars = r'[<>:"/\\|?*]' + matches = re.findall(invalid_chars, filename) + if matches: + return False, "".join(set(matches)) + return True, "" From 25b3d79b1ec7881defeece16f4bb1f703bfbb4eb Mon Sep 17 00:00:00 2001 From: Johnny Sequeira Date: Mon, 23 Sep 2024 15:20:13 -0600 Subject: [PATCH 2/9] Replacing validator with the pathvalidate library --- qfieldcloud_sdk/sdk.py | 20 +++++++++++++------- qfieldcloud_sdk/utils.py | 13 ------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index 94041b3..65d1d6e 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -13,7 +13,9 @@ from requests.adapters import HTTPAdapter, Retry from .interfaces import QfcException, QfcRequest, QfcRequestException -from .utils import calc_etag, log, is_valid_windows_filename +from .utils import calc_etag, log +from pathvalidate import is_valid_filename, is_valid_filepath, ValidationError + logger = logging.getLogger(__file__) @@ -410,10 +412,12 @@ def upload_files( # Validate file names for file in local_files: - is_valid, invalid_chars = is_valid_windows_filename(file["name"]) - if not is_valid: + try: + is_valid_filename(file["name"], platform="auto") + is_valid_filepath(file["absolute_filename"], platform="auto") + except ValidationError as e: raise ValueError( - f"Invalid file name: {file['name']}. Have invalid characters: {invalid_chars}" + f"Invalid file name or path: {file['name']}. Error: {e}" ) # we should always upload all package files @@ -491,10 +495,12 @@ def upload_file( The response object from the upload request. """ # Validate file name - is_valid, invalid_chars = is_valid_windows_filename(local_filename.name) - if not is_valid: + try: + is_valid_filename(local_filename.name, platform="auto") + is_valid_filepath(str(local_filename), platform="auto") + except ValidationError as e: raise ValueError( - f"Invalid file name: {local_filename.name}. Have invalid characters: {invalid_chars}" + f"Invalid file name or path: {local_filename.name}. Error: {e}" ) with open(local_filename, "rb") as local_file: diff --git a/qfieldcloud_sdk/utils.py b/qfieldcloud_sdk/utils.py index 5245a80..a0e2ac7 100644 --- a/qfieldcloud_sdk/utils.py +++ b/qfieldcloud_sdk/utils.py @@ -2,8 +2,6 @@ import json import os import sys -import re -from typing import Tuple def print_json(data): @@ -71,14 +69,3 @@ def calc_etag(filename: str, part_size: int = 8 * 1024 * 1024) -> str: final_md5sum = hashlib.md5(b"".join(md5sums)) return "{}-{}".format(final_md5sum.hexdigest(), len(md5sums)) - - -def is_valid_windows_filename(filename: str) -> Tuple[bool, str]: - """Check if the filename contains forbidden characters for Windows and return them.""" - # Forbidden characters in Windows - # https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names - invalid_chars = r'[<>:"/\\|?*]' - matches = re.findall(invalid_chars, filename) - if matches: - return False, "".join(set(matches)) - return True, "" From 6a21299690b0c2941c6b76d8e6abab01aa8479ec Mon Sep 17 00:00:00 2001 From: Johnny Sequeira Date: Mon, 23 Sep 2024 15:51:51 -0600 Subject: [PATCH 3/9] Adding the pathvalidate library to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0905324..1bc98e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ idna>=3.4 requests>=2.31.0 tqdm>=4.65.0 urllib3>=2.0.7 +pathvalidate==3.2.1 From b5ad778bd03a239c98cba79dc3cb410dde8a3a3d Mon Sep 17 00:00:00 2001 From: Johnny <77129293+SeqLaz@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:37:18 -0600 Subject: [PATCH 4/9] Update qfieldcloud_sdk/sdk.py Co-authored-by: Ivan Ivanov --- qfieldcloud_sdk/sdk.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index 65d1d6e..81b2a30 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -410,16 +410,8 @@ def upload_files( local_files = self.list_local_files(project_path, filter_glob) - # Validate file names for file in local_files: - try: - is_valid_filename(file["name"], platform="auto") - is_valid_filepath(file["absolute_filename"], platform="auto") - except ValidationError as e: - raise ValueError( - f"Invalid file name or path: {file['name']}. Error: {e}" - ) - + is_valid_filepath(file["name"]) # we should always upload all package files if upload_type == FileTransferType.PACKAGE: force = True From 7d030435f4ae55e5ae7d6038af34aa22c0bc3fd6 Mon Sep 17 00:00:00 2001 From: Johnny <77129293+SeqLaz@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:38:26 -0600 Subject: [PATCH 5/9] Update qfieldcloud_sdk/sdk.py Co-authored-by: Ivan Ivanov --- qfieldcloud_sdk/sdk.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index 81b2a30..c5dbcc7 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -486,14 +486,7 @@ def upload_file( Returns: The response object from the upload request. """ - # Validate file name - try: - is_valid_filename(local_filename.name, platform="auto") - is_valid_filepath(str(local_filename), platform="auto") - except ValidationError as e: - raise ValueError( - f"Invalid file name or path: {local_filename.name}. Error: {e}" - ) + is_valid_filepath(str(local_filename)) with open(local_filename, "rb") as local_file: upload_file = local_file From 8957da608e40aa031457741a94e75f3673bbe13b Mon Sep 17 00:00:00 2001 From: Johnny Sequeira Date: Mon, 23 Sep 2024 16:44:02 -0600 Subject: [PATCH 6/9] Removing platform specific check and removing unnecessary check validations --- qfieldcloud_sdk/sdk.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index c5dbcc7..cbb21e2 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -14,7 +14,7 @@ from .interfaces import QfcException, QfcRequest, QfcRequestException from .utils import calc_etag, log -from pathvalidate import is_valid_filename, is_valid_filepath, ValidationError +from pathvalidate import is_valid_filepath logger = logging.getLogger(__file__) @@ -410,8 +410,6 @@ def upload_files( local_files = self.list_local_files(project_path, filter_glob) - for file in local_files: - is_valid_filepath(file["name"]) # we should always upload all package files if upload_type == FileTransferType.PACKAGE: force = True From ad655d91f2cddcb3ed20f11483baabc56c971008 Mon Sep 17 00:00:00 2001 From: Johnny <77129293+SeqLaz@users.noreply.github.com> Date: Tue, 24 Sep 2024 00:14:21 -0600 Subject: [PATCH 7/9] Update requirements.txt Co-authored-by: Ivan Ivanov --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1bc98e0..7f57a4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ idna>=3.4 requests>=2.31.0 tqdm>=4.65.0 urllib3>=2.0.7 -pathvalidate==3.2.1 +pathvalidate>=3.2.1 From 77f8c41ce157fe9c74dd90c3e5063d62103902af Mon Sep 17 00:00:00 2001 From: Johnny <77129293+SeqLaz@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:08:17 -0600 Subject: [PATCH 8/9] Update qfieldcloud_sdk/sdk.py Co-authored-by: Ivan Ivanov --- qfieldcloud_sdk/sdk.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index cbb21e2..a111803 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -481,6 +481,9 @@ def upload_file( show_progress: Whether to display a progress bar during upload. job_id: The job ID, required if `upload_type` is PACKAGE. Defaults to an empty string. + Raises: + pathvalidate.ValidationError: Raised when the uploaded file does not have a valid filename. + Returns: The response object from the upload request. """ From ff0538a5348267eddffa6f90612706493be9474d Mon Sep 17 00:00:00 2001 From: Johnny <77129293+SeqLaz@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:08:31 -0600 Subject: [PATCH 9/9] Update qfieldcloud_sdk/sdk.py Co-authored-by: Ivan Ivanov --- qfieldcloud_sdk/sdk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index a111803..93592b8 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -487,6 +487,7 @@ def upload_file( Returns: The response object from the upload request. """ + # if the filepath is invalid, it will throw a new error `pathvalidate.ValidationError` is_valid_filepath(str(local_filename)) with open(local_filename, "rb") as local_file: