Fix Upload|Download Loop for AIP Protected File in Monitor Mode

.github/actions/spelling/allow.txt

@@ -1,5 +1,5 @@
 #
-# Other allow entries
+# Allowed words not found in configured dictionaries
 #
 AADSTS
 aarch
@@ -9,6 +9,7 @@ accrightslen
 adamdruppe
 addrepo
 adr
+aip
 alex
 alpinelinux
 annobin

docs/application-config-options.md

@@ -259,6 +259,13 @@ _**CLI Option Use:**_ `--disable-download-validation`
 > [!CAUTION]
 > If you're downloading data from SharePoint or OneDrive Business Shared Folders, you might find it necessary to activate this option. It's important to note that any issues encountered aren't due to a problem with this client; instead, they should be regarded as issues with the Microsoft OneDrive technology stack. Enabling this option disables all download integrity checks.
 
+> [!CAUTION]
+> If you are using OneDrive Business Accounts and your organisation implements Azure Information Protection, these AIP files will report as one size & hash online, but when downloaded, will report a totally different size and hash. 
+>
+> By default these files will fail integrity checking and be deleted, meaning that AIP files will not reside on your platform. 
+> 
+> When you enable this option, the AIP files will download to your platform, however, if there are any other genuine download failures where the size and hash are different, these too will be retained locally meaning you may experience data integrity loss. Use this option with extreme caution.
+
 ### disable_notifications
 _**Description:**_ This setting controls whether GUI notifications are sent from the client to your display manager session. 
 

docs/usage.md

@@ -8,6 +8,7 @@ Before reading this document, please ensure you are running application version
   - [Memory Usage](#memory-usage)
   - [Upgrading from the 'skilion' Client](#upgrading-from-the-skilion-client)
   - [Guidelines for Local File and Folder Naming in the Synchronisation Directory](#guidelines-for-local-file-and-folder-naming-in-the-synchronisation-directory)
+  - [Support for Microsoft Azure Information Protected Files](#support-for-microsoft-azure-information-protected-files)
   - [Compatibility with curl](#compatibility-with-curl)
 - [First Steps](#first-steps)
   - [Authorise the Application with Your Microsoft OneDrive Account](#authorise-the-application-with-your-microsoft-onedrive-account)
@@ -117,6 +118,16 @@ The above guidelines are essential for maintaining synchronisation integrity wit
 
 **Adherence to these guidelines is not optional but mandatory to avoid sync disruptions.**
 
+### Support for Microsoft Azure Information Protected Files
+> [!CAUTION]
+> If you are using OneDrive Business Accounts and your organisation implements Azure Information Protection, these AIP files will report as one size & hash online, but when downloaded, will report a totally different size and hash. 
+>
+> By default these files will fail integrity checking and be deleted locally, meaning that AIP files will not reside on your platform. These AIP files will be flagged as a failed download.
+> 
+> If you chose to enable `--disable-download-validation` , the AIP files will download to your platform, however, if there are any other genuine download failures where the size and hash are different, these too will be retained locally meaning you may experience data integrity loss. This is due to the Microsoft Graph API lacking any capability to identify up-front that a file utilises AIP, thus zero capability to differentiate between AIP and non-AIP files for failure.
+> 
+> Please use the `--disable-download-validation` option with extreme caution and understand the risk if you enable it.
+
 ### Compatibility with curl
 If your system uses curl < 7.47.0, curl will default to HTTP/1.1 for HTTPS operations, and the client will follow suit, using HTTP/1.1.
 

src/sync.d

@@ -648,28 +648,10 @@ class SyncEngine {
 
 		// If the JSON response is a correct JSON object, and has an 'id' we can set these details
 		if ((defaultOneDriveRootDetails.type() == JSONType.object) && (hasId(defaultOneDriveRootDetails))) {
-
-			// Read the returned JSON data for the root drive details
 			if (debugLogging) {addLogEntry("OneDrive Account Default Root Details:       " ~ to!string(defaultOneDriveRootDetails), ["debug"]);}
 			appConfig.defaultRootId = defaultOneDriveRootDetails["id"].str;
 			if (debugLogging) {addLogEntry("appConfig.defaultRootId      = " ~ appConfig.defaultRootId, ["debug"]);}
 
-			// Issue #2957 Handling for the Personal Account Root ID issues. Shared Folders coming from another account where this issue exists will need a different approach.
-			// If the returned data for appConfig.defaultRootId contains the string 'sea8cc6beffdb43d7976fbc7da445c639' .. this account has account issues with Microsoft
-			// This is only applicable for Microsoft Personal Accounts
-			if (appConfig.accountType == "personal") {
-				// Does the string 'sea8cc6beffdb43d7976fbc7da445c639' exist in the root id for the account?
-				if (appConfig.defaultRootId.indexOf("sea8cc6beffdb43d7976fbc7da445c639") != -1) {
-					// Yes ... flag account issue, we cannot proceed
-					addLogEntry();
-					addLogEntry("ERROR: You have a Microsoft OneDrive Account Problem. Please raise a support request with Microsoft. You cannot use Microsoft OneDrive at this point in time.", ["info", "notify"]);
-					addLogEntry("ERROR: Account Root ID contains the string 'sea8cc6beffdb43d7976fbc7da445c639'.");
-					addLogEntry();
-					// Force Exit
-					forceExit();
-				}
-			}
-
 			// Save the item to the database, so the account root drive is is always going to be present in the DB
 			saveItem(defaultOneDriveRootDetails);
 		} else {
@@ -3068,6 +3050,38 @@ class SyncEngine {
 							// Whilst the download integrity checks were disabled, we still have to set the correct timestamp on the file
 							// Set the timestamp, logging and error handling done within function
 							setPathTimestamp(dryRun, newItemPath, itemModifiedTime);
+
+							// Azure Information Protection (AIP) protected files potentially have missing data and/or inconsistent data
+							if (appConfig.accountType != "personal") {
+								// AIP Protected Files cause issues here, as the online size & hash are not what has been downloaded
+								// There is ZERO way to determine if this is an AIP protected file either from the JSON data
+
+								// Calculate the local file hash and get the local file size
+								string localFileHash = computeQuickXorHash(newItemPath);
+								long downloadFileSize = getSize(newItemPath);
+
+								if ((OneDriveFileXORHash != localFileHash) && (jsonFileSize != downloadFileSize)) {
+
+									// High potential to be an AIP protected file given the following scenario
+									// Business | SharePoint Account Type (not a personal account)
+									// --disable-download-validation is being used .. meaning the user has specifically configured this due the Microsoft SharePoint Enrichment Feature (bug)
+									// The file downloaded but the XOR hash and file size locally is not as per the provided JSON - both are different
+									//
+									// Update the 'onedriveJSONItem' JSON data with the local values ..... 
+									if (debugLogging) {
+										string aipLogMessage = format("POTENTIAL AIP FILE (Issue 3070) - Changing the source JSON data provided by Graph API to use actual on-disk values (quickXorHash,size): %s", newItemPath);
+										addLogEntry(aipLogMessage, ["debug"]);
+										addLogEntry(" - Online XOR   : " ~ to!string(OneDriveFileXORHash), ["debug"]);
+										addLogEntry(" - Online Size  : " ~ to!string(jsonFileSize), ["debug"]);
+										addLogEntry(" - Local XOR    : " ~ to!string(computeQuickXorHash(newItemPath)), ["debug"]);
+										addLogEntry(" - Local Size   : " ~ to!string(getSize(newItemPath)), ["debug"]);
+									}
+
+									// Make the change in the JSON using local values
+									onedriveJSONItem["file"]["hashes"]["quickXorHash"] = localFileHash;
+									onedriveJSONItem["size"] = downloadFileSize;
+								}
+							}
 						}	// end of (!disableDownloadValidation)
 					} else {
 						addLogEntry("ERROR: File failed to download. Increase logging verbosity to determine why.");