diff --git a/README.md b/README.md index 81125ef..661695c 100644 --- a/README.md +++ b/README.md @@ -85,14 +85,13 @@ options: 14 and 32. Default: 20 => 1 MB -V, --verify Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and - NSZ files when given as parameter. Requires --keep- - delta when used during compression. + NSZ files when given as parameter. Requires --keep + when used during compression. -Q, --quick-verify Same as --verify but skips the NSP SHA256 hash verification and only verifies NCA hashes. Does not - require --keep-delta when used during compression. - -K, --keep-delta Keep all useless delta fragments (NDV0) during - compression so the NSP (PFS0) can be recreated bit- - identical during decompression + require --keep when used during compression. + -K, --keep Keep all useless files and partitions during + compression to allow bit-identical recreation -F, --fix-padding Fixes PFS0 padding to match the nxdumptool/no-intro standard. Incompatible with --verify so --quick-verify will be used instead. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ae058a..989ac14 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -30,24 +30,24 @@ steps: - task: CmdLine@2 inputs: - script: python nsz.py -D -V --keep-delta -w -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsp_from_solid_out C:\Users\Azure\.switch\tests\nsz_solid + script: python nsz.py -D -V --keep -w -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsp_from_solid_out C:\Users\Azure\.switch\tests\nsz_solid failOnStderr: true displayName: 'Test NSZ solid decompression' - task: CmdLine@2 inputs: - script: python nsz.py -D -V --keep-delta -w -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsp_from_block_out C:\Users\Azure\.switch\tests\nsz_block + script: python nsz.py -D -V --keep -w -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsp_from_block_out C:\Users\Azure\.switch\tests\nsz_block failOnStderr: true displayName: 'Test NSZ block decompression' - task: CmdLine@2 inputs: - script: python nsz.py -C -V --keep-delta -w -p -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsz_solid_out C:\Users\Azure\.switch\tests\nsp + script: python nsz.py -C -V --keep -w -p -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsz_solid_out C:\Users\Azure\.switch\tests\nsp failOnStderr: true displayName: 'Test NSP solid compression' - task: CmdLine@2 inputs: - script: python nsz.py -C -B -V --keep-delta -w -p -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsz_block_out C:\Users\Azure\.switch\tests\nsp + script: python nsz.py -C -B -V --keep -w -p -m 4 -t 4 -o C:\Users\Azure\.switch\tests\nsz_block_out C:\Users\Azure\.switch\tests\nsp failOnStderr: true displayName: 'Test NSP block compression' diff --git a/nsz/BlockCompressor.py b/nsz/BlockCompressor.py index 8da023e..d160eef 100644 --- a/nsz/BlockCompressor.py +++ b/nsz/BlockCompressor.py @@ -29,13 +29,13 @@ def compressBlockTask(in_queue, out_list, readyForWork, pleaseKillYourself, bloc compressed = ZstdCompressor(compression_params=params).compress(buffer) out_list[chunkRelativeBlockID] = compressed if len(compressed) < len(buffer) else buffer -def blockCompress(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads): +def blockCompress(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads): if filePath.suffix == '.nsp': - return blockCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads) + return blockCompressNsp(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads) elif filePath.suffix == '.xci': - return blockCompressXci(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads) + return blockCompressXci(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads) -def blockCompressContainer(readContainer, writeContainer, compressionLevel, keepDelta, useLongDistanceMode, blockSizeExponent, threads): +def blockCompressContainer(readContainer, writeContainer, compressionLevel, keep, useLongDistanceMode, blockSizeExponent, threads): CHUNK_SZ = 0x100000 UNCOMPRESSABLE_HEADER_SIZE = 0x4000 if blockSizeExponent < 14 or blockSizeExponent > 32: @@ -57,7 +57,7 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, keep pool.append(p) for nspf in readContainer: - if not keepDelta: + if not keep: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA: Print.info('[SKIPPED] Delta fragment {0}'.format(nspf._path)) continue @@ -201,7 +201,7 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, keep sleep(0.02) -def blockCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads): +def blockCompressNsp(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') @@ -211,7 +211,7 @@ def blockCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongD try: with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if fixPadding else container.getFirstFileOffset(), None if fixPadding else container.getStringTableSize(), str(nszPath)) as nsp: - blockCompressContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, blockSizeExponent, threads) + blockCompressContainer(container, nsp, compressionLevel, keep, useLongDistanceMode, blockSizeExponent, threads) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) @@ -221,7 +221,7 @@ def blockCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongD container.close() return nszPath -def blockCompressXci(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads): +def blockCompressXci(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, blockSizeExponent, outputDir, threads): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') @@ -233,7 +233,7 @@ def blockCompressXci(filePath, compressionLevel, keepDelta, fixPadding, useLongD try: with Xci.XciStream(str(xczPath), originalXciPath = filePath) as xci: # need filepath to copy XCI container settings with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0), xci.f.tell()) as secureOut: - blockCompressContainer(secureIn, secureOut, compressionLevel, keepDelta, useLongDistanceMode, blockSizeExponent, threads) + blockCompressContainer(secureIn, secureOut, compressionLevel, keep, useLongDistanceMode, blockSizeExponent, threads) xci.hfs0.resize('secure', secureOut.actualSize) except BaseException as ex: diff --git a/nsz/ParseArguments.py b/nsz/ParseArguments.py index 08ccf81..00b07a4 100644 --- a/nsz/ParseArguments.py +++ b/nsz/ParseArguments.py @@ -12,9 +12,9 @@ def parse(): parser.add_argument('-B', '--block', action="store_true", default=False, help="Use block compression option. This mode allows highly multi-threaded compression/decompression with random read access allowing compressed games to be played without decompression in the future however this comes with a slightly lower compression ratio cost. This is the default option for XCZ.") parser.add_argument('-S', '--solid', action="store_true", default=False, help="Use solid compression option. Slightly higher compression ratio but won't allow for random read access. File compressed this way will never be mountable (have to be installed or decompressed first to run). This is the default option for NSZ.") parser.add_argument('-s', '--bs', type=int, default=20, help='Block Size for random read access 2^x while x between 14 and 32. Default: 20 => 1 MB') - parser.add_argument('-V', '--verify', action="store_true", default=False, help='Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and NSZ files when given as parameter. Requires --keep-delta when used during compression.') - parser.add_argument('-Q', '--quick-verify', action="store_true", default=False, help='Same as --verify but skips the NSP SHA256 hash verification and only verifies NCA hashes. Does not require --keep-delta when used during compression.') - parser.add_argument('-K', '--keep-delta', action="store_true", default=False, help='Keep all useless delta fragments (NDV0) during compression so the NSP (PFS0) can be recreated bit-identical during decompression') + parser.add_argument('-V', '--verify', action="store_true", default=False, help='Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and NSZ files when given as parameter. Requires --keep when used during compression.') + parser.add_argument('-Q', '--quick-verify', action="store_true", default=False, help='Same as --verify but skips the NSP SHA256 hash verification and only verifies NCA hashes. Does not require --keep when used during compression.') + parser.add_argument('-K', '--keep', action="store_true", default=False, help='Keep all useless files and partitions during compression to allow bit-identical recreation') parser.add_argument('-F', '--fix-padding', action="store_true", default=False, help='Fixes PFS0 padding to match the nxdumptool/no-intro standard. Incompatible with --verify so --quick-verify will be used instead.') parser.add_argument('-p', '--parseCnmt', action="store_true", default=False, help='Extract TitleId/Version from Cnmt if this information cannot be obtained from the filename. Required for skipping/overwriting existing files and --rm-old-version to work properly if some not every file is named properly. Supported filenames: *TitleID*[vVersion]*') parser.add_argument('-P', '--alwaysParseCnmt', action="store_true", default=False, help='Always extract TitleId/Version from Cnmt and never trust filenames') diff --git a/nsz/SolidCompressor.py b/nsz/SolidCompressor.py index ae106d4..3b47586 100644 --- a/nsz/SolidCompressor.py +++ b/nsz/SolidCompressor.py @@ -11,15 +11,15 @@ CHUNK_SZ = 0x1000000 -def solidCompress(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint): +def solidCompress(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint): if filePath.suffix == '.nsp': - return solidCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint) + return solidCompressNsp(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint) elif filePath.suffix == '.xci': - return solidCompressXci(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint) + return solidCompressXci(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint) -def processContainer(readContainer, writeContainer, compressionLevel, keepDelta, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint): +def processContainer(readContainer, writeContainer, compressionLevel, keep, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint): for nspf in readContainer: - if not keepDelta: + if not keep: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA: Print.info('[SKIPPED] Delta fragment {0}'.format(nspf._path), pleaseNoPrint) continue @@ -120,7 +120,7 @@ def processContainer(readContainer, writeContainer, compressionLevel, keepDelta, f.write(buffer) -def solidCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint): +def solidCompressNsp(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') @@ -130,7 +130,7 @@ def solidCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongD try: with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if fixPadding else container.getFirstFileOffset(), None if fixPadding else container.getStringTableSize(), str(nszPath)) as nsp: - processContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint) + processContainer(container, nsp, compressionLevel, keep, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) @@ -140,7 +140,7 @@ def solidCompressNsp(filePath, compressionLevel, keepDelta, fixPadding, useLongD container.close() return nszPath -def solidCompressXci(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint): +def solidCompressXci(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') @@ -152,7 +152,7 @@ def solidCompressXci(filePath, compressionLevel, keepDelta, fixPadding, useLongD try: with Xci.XciStream(str(xczPath), originalXciPath = filePath) as xci: # need filepath to copy XCI container settings with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint), xci.f.tell()) as secureOut: - processContainer(secureIn, secureOut, compressionLevel, keepDelta, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint) + processContainer(secureIn, secureOut, compressionLevel, keep, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint) xci.hfs0.resize('secure', secureOut.actualSize) except BaseException as ex: diff --git a/nsz/__init__.py b/nsz/__init__.py index c2ad664..cef0fd7 100644 --- a/nsz/__init__.py +++ b/nsz/__init__.py @@ -32,12 +32,12 @@ def solidCompressTask(in_queue, statusReport, readyForWork, pleaseNoPrint, pleas if pleaseKillYourself.value() > 0: break try: - filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threadsToUse, verifyArg, quickVerify = item - outFile = solidCompress(filePath, compressionLevel, keepDelta, fixPadding, useLongDistanceMode, outputDir, threadsToUse, statusReport, id, pleaseNoPrint) + filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threadsToUse, verifyArg, quickVerify = item + outFile = solidCompress(filePath, compressionLevel, keep, fixPadding, useLongDistanceMode, outputDir, threadsToUse, statusReport, id, pleaseNoPrint) if verifyArg: Print.info("[VERIFY NSZ] {0}".format(outFile)) try: - verify(outFile, fixPadding, True, keepDelta, None if quickVerify else filePath, [statusReport, id], pleaseNoPrint) + verify(outFile, fixPadding, True, keep, None if quickVerify else filePath, [statusReport, id], pleaseNoPrint) except VerificationException: Print.error("[BAD VERIFY] {0}".format(outFile)) Print.error("[DELETE NSZ] {0}".format(outFile)) @@ -54,11 +54,11 @@ def compress(filePath, outputDir, args, work, amountOfTastkQueued): if filePath.suffix == ".xci" and not args.solid or args.block: threadsToUseForBlockCompression = args.threads if args.threads > 0 else cpu_count() - outFile = blockCompress(filePath, compressionLevel, args.keep_delta, args.fix_padding, args.long, args.bs, outputDir, threadsToUseForBlockCompression) + outFile = blockCompress(filePath, compressionLevel, args.keep, args.fix_padding, args.long, args.bs, outputDir, threadsToUseForBlockCompression) if args.verify: Print.info("[VERIFY NSZ] {0}".format(outFile)) try: - verify(outFile, args.fix_padding, True, args.keep_delta, None if args.quick_verify else filePath) + verify(outFile, args.fix_padding, True, args.keep, None if args.quick_verify else filePath) except VerificationException: Print.error("[BAD VERIFY] {0}".format(outFile)) Print.error("[DELETE NSZ] {0}".format(outFile)) @@ -66,7 +66,7 @@ def compress(filePath, outputDir, args, work, amountOfTastkQueued): raise else: threadsToUseForSolidCompression = args.threads if args.threads > 0 else 3 - work.put([filePath, compressionLevel, args.keep_delta, args.fix_padding, args.long, outputDir, threadsToUseForSolidCompression, args.verify, args.quick_verify]) + work.put([filePath, compressionLevel, args.keep, args.fix_padding, args.long, outputDir, threadsToUseForSolidCompression, args.verify, args.quick_verify]) amountOfTastkQueued.increment() @@ -158,8 +158,8 @@ def main(): nsp.pack(args.file) if args.C: - if args.verify and not args.quick_verify and not args.keep_delta: - Print.info("Warning: --verify requires --keep-delta when used during compression or it will detect removed NDV0 fragments as errors. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.3.0.") + if args.verify and not args.quick_verify and not args.keep: + Print.info("Warning: --verify requires --keep when used during compression or it will detect removed NDV0 fragments as errors. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.3.0.") args.quick_verify = True if args.verify and not args.quick_verify and args.fix_padding: Print.info("Warning: --verify and --fix-padding are incompatible with each others. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.6.0.") diff --git a/nsz/gui/NSZ_GUI.py b/nsz/gui/NSZ_GUI.py index 7d3db3b..eea176a 100644 --- a/nsz/gui/NSZ_GUI.py +++ b/nsz/gui/NSZ_GUI.py @@ -65,7 +65,7 @@ def build_config(self, config): 'solid': 0, 'bs': "1 MB (default)", 'verify_options': "Quick (NCA hashes)", - 'keepDelta': 0, + 'keep': 0, }) config.setdefaults('Advanced', { 'threads': -1, @@ -151,7 +151,7 @@ def __init__(self, config, rootWidget): self.quick_verify = True else: self.quick_verify = None - self.keep_delta = True if int(config.get('Settings', 'keepDelta')) == 1 else False + self.keep = True if int(config.get('Settings', 'keep')) == 1 else False self.threads = int(config.get('Advanced', 'threads')) self.multi = int(config.get('Advanced', 'multi')) self.fix_padding = True if int(config.get('Advanced', 'fixPadding')) == 1 else False diff --git a/nsz/gui/json/settings_basic.json b/nsz/gui/json/settings_basic.json index 01f3d85..ce98483 100644 --- a/nsz/gui/json/settings_basic.json +++ b/nsz/gui/json/settings_basic.json @@ -32,16 +32,16 @@ { "type": "scrolloptions", "title": "Verify", - "desc": "Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and NSZ files when given as parameter. Quick skips the expensive and mostly useless PFS0 hash verification and only verifies NCA hashes. Full requires --keep-delta when used for compression.", + "desc": "Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and NSZ files when given as parameter. Quick skips the expensive and mostly useless PFS0 hash verification and only verifies NCA hashes. Full requires --keep when used for compression.", "section": "Settings", "key": "verify_options", "options": [ "None", "Quick (NCA hashes)", "Full (NCA & PFS0 hashes)" ] }, { "type": "bool", - "title": "Keep Delta Fragments", - "desc": "Keep all useless delta fragments (NDV0) during compression so the NSP (PFS0) can be recreated bit-identical during decompression", + "title": "Keep Everything", + "desc": "Keep all useless files and partitions during compression to allow bit-identical recreation", "section": "Settings", - "key": "keepDelta" + "key": "keep" } ]