Implemented undupe, undupe-dryrun, undupe-prioritylist, undupe-whitel…

…ist and undupe-old-versions to remove duplicate games

nsz.pyproj

@@ -25,6 +25,7 @@
     <Content Include="dev\test.cmd" />
     <Content Include="dev\testPip.cmd" />
     <Content Include="dev\testPyInstaller.cmd" />
+    <Content Include="install_linux.sh" />
     <Content Include="LICENSE" />
     <Content Include="MANIFEST.in" />
     <Content Include="nsz\gui\json\settings_advanced.json" />
@@ -44,7 +45,6 @@
     <Content Include="README.md" />
     <Content Include="requirements-gui.txt" />
     <Content Include="requirements.txt" />
-    <Content Include="requirements_linux.sh" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="nsz.py" />
@@ -91,6 +91,7 @@
     <Compile Include="nsz\SectionFs.py" />
     <Compile Include="nsz\SolidCompressor.py" />
     <Compile Include="nsz\ThreadSafeCounter.py" />
+    <Compile Include="nsz\undupe.py" />
     <Compile Include="nsz\__init__.py" />
     <Compile Include="setup.py" />
   </ItemGroup>

nsz/FileExistingChecks.py

@@ -59,17 +59,15 @@ def ExtractTitleIDAndVersion(gamePath, parseCnmt):
 		return(titleId, version)
 	return None
 
-def CreateTargetDict(targetFolder, parseCnmt, extension):
-	filesAtTarget = {}
-	alreadyExists = {}
-	for file in scandir(str(targetFolder)):
+def CreateTargetDict(targetFolder, parseCnmt, extension, filesAtTarget = {}, alreadyExists = {}):
+	for filePath in expandFiles(targetFolder):
 		try:
-			filePath = Path(targetFolder).joinpath(file.name)
 			filePath_str = str(filePath)
-			if filePath.suffix == extension:
-				Print.infoNoNewline('Extract TitleID/Version: {0} '.format(file.name))
-				filesAtTarget[file.name.lower()] = filePath_str
-				extractedIdVersion = ExtractTitleIDAndVersion(file, parseCnmt)
+			if (isGame(filePath) or filePath.suffix == ".nspz") and (extension == None or filePath.suffix == extension):
+				print(filePath)
+				Print.infoNoNewline('Extract TitleID/Version: {0} '.format(filePath.name))
+				filesAtTarget[filePath.name.lower()] = filePath_str
+				extractedIdVersion = ExtractTitleIDAndVersion(filePath, parseCnmt)
 				if extractedIdVersion == None:
 					if parseCnmt:
 						Print.error('Failed to extract TitleID/Version from booth filename "{0}" and Cnmt - Outdated keys.txt?'.format(Path(filePath).name))

nsz/ParseArguments.py

@@ -24,6 +24,12 @@ def parse():
 		parser.add_argument('-x', '--extract', action="store_true", help='Extract a NSP/XCI/NSZ/XCZ/NSPZ')
 		parser.add_argument('--extractregex', type=str, default="", help='Regex specifying which files inside the container should be extracted. Example: "^.*\.(cert|tik)$"')
 		parser.add_argument('--titlekeys', action='store_true', default=False, help="Extracts titlekeys from your NSP/NSZ files and adds missing keys to ./titlekeys.txt and JSON files inside ./titledb/ (obtainable from https://github.com/blawar/titledb). Titlekeys can be used to unlock updates using NUT OG (OG fork obtainable from https://github.com/plato79/nut). There is currently no publicly known way of optioning NSX files. To MitM: Apply disable_ca_verification & disable_browser_ca_verification patches, use your device's nx_tls_client_cert.pfx (Password: switch, Install to OS and import for Fiddler or import into Charles/OWASP ZAP). Use it for aauth-lp1.ndas.srv.nintendo.net:443, dauth-lp1.ndas.srv.nintendo.net:443 and app-b01-lp1.npns.srv.nintendo.net:443. Try with your WiiU first as there you won't get banned if you mess up.")
+		parser.add_argument('--undupe', action='store_true', help="Deleted all duplicates (games with same ID and Version). The Files folder will get parsed in order so the later in the argument list the more likely the file is to be deleted")
+		parser.add_argument('--undupe-dryrun' , action='store_true', help="Shows what files would get deleted using --undupe")
+		parser.add_argument('--undupe-prioritylist', type=str, default="", help='Regex specifying which dublicates delegtion should be prioritized before following the normal deletion order. Example: "^.*\.(nsp|xci)$"')
+		parser.add_argument('--undupe-whitelist', type=str, default="", help='Regex specifying which dublicates should under no circumstances be deleted. Example: "^.*\.(nsz|xcz)$"')
+		parser.add_argument('--undupe-blacklist', type=str, default="", help='Regex specifying which files should always be deleted - even if they are not even a dublicate! Be careful! Example: "^.*\.(nsp|xci)$"')
+		parser.add_argument('--undupe-old-versions',action="store_true", default=False, help='Removes every old version as long there is a newer one of the same titleID.')
 		parser.add_argument('-c', '--create', help='create / pack a NSP')
 
 		args = parser.parse_args()

nsz/__init__.py

@@ -19,6 +19,7 @@
 from nsz.ParseArguments import *
 from nsz.PathTools import *
 from nsz.ExtractTitlekeys import *
+from nsz.undupe import undupe
 import enlighten
 import time
 
@@ -132,6 +133,9 @@ def main():
 						container.unpack(outFolder, args.extractregex)
 					container.close()
 
+		if args.undupe or args.undupe_dryrun:
+			undupe(args);
+
 		if args.create:
 			Print.info('Creating "{0}"'.format(args.create))
 			nsp = Nsp.Nsp(None, None)

nsz/undupe.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+from nsz.FileExistingChecks import CreateTargetDict
+from nsz.nut import Print	
+import re
+
+def isOnWhitelist(args, file):
+	if not args.undupe_whitelist == "" and re.match(args.undupe_whitelist, file):
+		if args.undupe_dryrun:
+			Print.info("[DRYRUN] [WHITELISTED]: " + file)
+		else:
+			Print.info("[WHITELISTED]: " + file)
+		return True
+	return False
+
+def undupe(args):
+	filesAtTarget = {}
+	alreadyExists = {}
+	for f_str in args.file:
+		(filesAtTarget, alreadyExists) = CreateTargetDict(Path(f_str).absolute(), True, None, filesAtTarget, alreadyExists)
+	Print.info("")
+
+	for (titleID_key, titleID_value) in alreadyExists.items():
+		maxVersion = max(titleID_value.keys())
+		for (version_key, version_value) in titleID_value.items():
+
+			if args.undupe_old_versions and version_key < maxVersion:
+				for file in list(version_value):
+					if not isOnWhitelist(args, file):
+						if args.undupe_dryrun:
+							Print.info("[DRYRUN] [DELETE] [OLD_VERSION]: " + file)
+						else:
+							del file
+							Print.info("[DELETED] [OLD_VERSION]: " + file)
+				continue
+
+
+			if not args.undupe_blacklist == "":
+				for file in list(version_value):
+					if not isOnWhitelist(args, file) and re.match(args.undupe_blacklist, file):
+						version_value.remove(file)
+						if args.undupe_dryrun:
+							Print.info("[DRYRUN] [DELETE] [BLACKLIST]: " + file)
+						else:
+							del file
+							Print.info("[DRYRUN] [DELETED] [BLACKLIST]: " + file)
+
+			if not args.undupe_prioritylist == "":
+				for file in list(version_value.reverse()):
+					if len(version_value) > 1 and not isOnWhitelisth(args, file) and re.match(args.undupe_prioritylist, file):
+						version_value.remove(file)
+						if args.undupe_dryrun:
+							Print.info("[DRYRUN] [DELETE] [PRIORITYLIST]: " + file)
+						else:
+							del file
+							Print.info("[DRYRUN] [DELETED] [PRIORITYLIST]: " + file)
+
+			for file in list(version_value[1:]):
+				if not isOnWhitelist(args, file):
+					if args.undupe_dryrun:
+						Print.info("[DRYRUN] [DELETE] [DUPE]: " + file)
+					else:
+						del file
+						Print.info("[DELETED] [DUPE]: " + file)