diff --git a/.travis.yml b/.travis.yml index 977b0df7..27745c0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,11 @@ matrix: include: - os: osx language: generic - osx_image: xcode6.4 + osx_image: xcode9.2 before_install: - brew update -- brew install python3 +- brew upgrade python3 - brew uninstall node - travis_wait 30 brew install node@6 - brew link node@6 --force --overwrite @@ -15,7 +15,7 @@ before_install: install: - pip3 install -r requirements.txt -- pip3 install certifi https://github.com/bjones1/pyinstaller/archive/pyqt5_fix.zip +- pip3 install certifi https://github.com/pyinstaller/pyinstaller/archive/develop.zip - npm install -g appdmg script: python3 setup.py build_binary diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ce0bfe9..1db386ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ # CHANGELOG +#### 5.5.0: +* Added support for WebP format +* Added profiles for Kindle Paperwhite 4 and Kobo Forma +* All archives are now handled by 7z +* Removed MCD support +* Fixed multiple smaller issues + #### 5.4.5: * Fixed EPUB output for non-Kindle devices diff --git a/LICENSE.txt b/LICENSE.txt index a060b54f..a5097472 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ ISC LICENSE Copyright (c) 2012-2014 Ciro Mattia Gonano -Copyright (c) 2013-2018 Paweł Jastrzębski +Copyright (c) 2013-2019 Paweł Jastrzębski Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the diff --git a/README.md b/README.md index b406bb82..32836a4a 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,8 @@ If you find **KCC** valuable you can consider donating to the authors: ## BINARY RELEASES You can find the latest released binary at the following links: - **[Windows](http://kcc.iosphe.re/Windows/) (64-bit only)** -- **[macOS](http://kcc.iosphe.re/OSX/) (10.10+)** -- **Linux:** - - [Ubuntu 17.10](http://kcc.iosphe.re/LinuxArtful/) - - [Ubuntu 16.04 / Debian 9](http://kcc.iosphe.re/LinuxXenial/) - - [Ubuntu 14.04 / Debian 8](http://kcc.iosphe.re/LinuxTrusty/) +- **[macOS](http://kcc.iosphe.re/OSX/) (10.12+)** +- **Linux:** Currently unavailable. ## PYPI **KCC** is also available on PyPI. @@ -51,21 +48,20 @@ Following software is required to run Linux version of **KCC** and/or bare sourc On Debian based distributions these two commands should install all needed dependencies: ``` -sudo apt-get install python3 python3-dev python3-pip libpng-dev libjpeg-dev p7zip-full unrar +sudo apt-get install python3 python3-dev python3-pip libpng-dev libjpeg-dev p7zip-full sudo pip3 install --upgrade pillow python-slugify psutil pyqt5 raven ``` ### Optional dependencies - [KindleGen](http://www.amazon.com/gp/feature.html?ie=UTF8&docId=1000765211) v2.9+ in a directory reachable by your _PATH_ or in _KCC_ directory *(For MOBI generation)* -- [UnRAR](http://www.rarlab.com/download.htm) *(For CBR/RAR support)* -- [7za](http://www.7-zip.org/download.html) *(For 7z/CB7 support)* +- [7z](http://www.7-zip.org/download.html) *(For CBZ/ZIP, CBR/RAR, 7z/CB7 support)* ## INPUT FORMATS **KCC** can understand and convert, at the moment, the following input types: -- Folders containing: PNG, JPG or GIF files -- CBZ, ZIP -- CBR, RAR *(With `unrar` executable)* -- CB7, 7Z *(With `7za` executable)* +- Folders containing: PNG, JPG, GIF or WebP files +- CBZ, ZIP *(With `7z` executable)* +- CBR, RAR *(With `7z` executable)* +- CB7, 7Z *(With `7z` executable)* - PDF *(Only extracting JPG images)* ## USAGE @@ -87,7 +83,7 @@ Options: -p PROFILE, --profile=PROFILE Device profile (Available options: K1, K2, K34, K578, KDX, KPW, KV, KO, KoMT, KoG, KoGHD, KoA, KoAHD, KoAH2O, - KoAO) [Default=KV] + KoAO, KoF) [Default=KV] -m, --manga-style Manga style (right-to-left reading and splitting) -q, --hq Try to increase the quality of magnification -2, --two-panel Display two not four panels in Panel View mode @@ -160,29 +156,28 @@ This script born as a cross-platform alternative to `KindleComicParser` by **Dc5 The app relies and includes the following scripts: - `DualMetaFix` script by **K. Hendricks**. Released with GPL-3 License. - - `rarfile.py` script © 2005-2014 **Marko Kreen** . Released with ISC License. - `image.py` class from **Alex Yatskov**'s [Mangle](https://github.com/FooSoft/mangle/) with subsequent [proDOOMman](https://github.com/proDOOMman/Mangle)'s and [Birua](https://github.com/Birua/Mangle)'s patches. - Icon is by **Nikolay Verin** ([http://ncrow.deviantart.com/](http://ncrow.deviantart.com/)) and released under [CC BY-NC-SA 3.0](http://creativecommons.org/licenses/by-nc-sa/3.0/) License. ## SAMPLE FILES CREATED BY KCC * [Kindle Oasis 2](http://kcc.iosphe.re/Samples/Ubunchu!-KO.mobi) -* [Kindle Paperwhite 3 / Voyage / Oasis](http://kcc.iosphe.re/Samples/Ubunchu!-KV.mobi) +* [Kindle Paperwhite 3 / 4 / Voyage / Oasis](http://kcc.iosphe.re/Samples/Ubunchu!-KV.mobi) * [Kindle Paperwhite 1 / 2](http://kcc.iosphe.re/Samples/Ubunchu!-KPW.mobi) * [Kindle](http://kcc.iosphe.re/Samples/Ubunchu!-K578.mobi) * [Kobo Aura](http://kcc.iosphe.re/Samples/Ubunchu-KoA.kepub.epub) * [Kobo Aura HD](http://kcc.iosphe.re/Samples/Ubunchu-KoAHD.kepub.epub) * [Kobo Aura H2O](http://kcc.iosphe.re/Samples/Ubunchu-KoAH2O.kepub.epub) * [Kobo Aura ONE](http://kcc.iosphe.re/Samples/Ubunchu-KoAO.kepub.epub) +* [Kobo Forma](http://kcc.iosphe.re/Samples/Ubunchu-KoF.kepub.epub) ## PRIVACY -**KCC** is initiating internet connections in three cases: -* During startup - Version check -* When MCD metadata are used - Cover download -* When error occurs - Automatic reporting +**KCC** is initiating internet connections in two cases: +* During startup - Version check. +* When error occurs - Automatic reporting on Windows and MacOS. ## KNOWN ISSUES Please check [wiki page](https://github.com/ciromattia/kcc/wiki/Known-issues). ## COPYRIGHT -Copyright (c) 2012-2018 Ciro Mattia Gonano and Paweł Jastrzębski. +Copyright (c) 2012-2019 Ciro Mattia Gonano and Paweł Jastrzębski. **KCC** is released under ISC LICENSE; see LICENSE.txt for further details. diff --git a/appveyor.yml b/appveyor.yml index 2eb446ce..b603c3cc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,11 +1,11 @@ environment: - PYTHON: "C:\\Python36-x64" + PYTHON: "C:\\Python37-x64" install: - set PATH="%PYTHON%\\Scripts";"C:\\Program Files (x86)\\Inno Setup 5";%PATH% - "%PYTHON%\\python.exe -m pip install --upgrade pip setuptools wheel" - "%PYTHON%\\python.exe -m pip install -r requirements.txt" - - "%PYTHON%\\python.exe -m pip install certifi https://github.com/bjones1/pyinstaller/archive/pyqt5_fix.zip" + - "%PYTHON%\\python.exe -m pip install certifi https://github.com/pyinstaller/pyinstaller/archive/develop.zip" - nuget install secure-file -ExcludeVersion - secure-file\tools\secure-file -decrypt other\windows\sentry.py.enc -out kindlecomicconverter\sentry.py -secret %ENCRYPTION% diff --git a/gui/MetaEditor.ui b/gui/MetaEditor.ui index 8f1a9919..a8691ff1 100644 --- a/gui/MetaEditor.ui +++ b/gui/MetaEditor.ui @@ -112,19 +112,6 @@ - - - - <html><head/><body><p><a href="https://github.com/ciromattia/kcc/wiki/Manga-Cover-Database-support"><span style=" text-decoration: underline; color:#0000ff;">MUid:</span></a></p></body></html> - - - true - - - - - - diff --git a/kcc-c2e.py b/kcc-c2e.py index 24005609..d2188a70 100755 --- a/kcc-c2e.py +++ b/kcc-c2e.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/kcc-c2p.py b/kcc-c2p.py index 4d5b1e1e..b7c98922 100755 --- a/kcc-c2p.py +++ b/kcc-c2p.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/kcc.iss b/kcc.iss index 28a8d410..ac4e92fa 100644 --- a/kcc.iss +++ b/kcc.iss @@ -1,5 +1,5 @@ #define MyAppName "Kindle Comic Converter" -#define MyAppVersion "5.4.5" +#define MyAppVersion "5.5.0" #define MyAppPublisher "Ciro Mattia Gonano, Paweł Jastrzębski" #define MyAppURL "http://kcc.iosphe.re/" #define MyAppExeName "KCC.exe" @@ -12,7 +12,7 @@ AppPublisher={#MyAppPublisher} AppPublisherURL={#MyAppURL} AppSupportURL={#MyAppURL} AppUpdatesURL={#MyAppURL} -AppCopyright=Copyright (C) 2012-2018 Ciro Mattia Gonano and Paweł Jastrzębski +AppCopyright=Copyright (C) 2012-2019 Ciro Mattia Gonano and Paweł Jastrzębski ArchitecturesAllowed=x64 DefaultDirName={pf}\{#MyAppName} DefaultGroupName={#MyAppName} @@ -47,8 +47,8 @@ Name: "CB7association"; Description: "CB7"; GroupDescription: "File associations Source: "dist\{#MyAppExeName}"; DestDir: "{app}"; Flags: ignoreversion Source: "LICENSE.txt"; DestDir: "{app}"; Flags: ignoreversion solidbreak Source: "other\windows\Additional-LICENSE.txt"; DestDir: "{app}"; Flags: ignoreversion -Source: "other\windows\UnRAR.exe"; DestDir: "{app}"; Flags: ignoreversion -Source: "other\windows\7za.exe"; DestDir: "{app}"; Flags: ignoreversion +Source: "other\windows\7z.exe"; DestDir: "{app}"; Flags: ignoreversion +Source: "other\windows\7z.dll"; DestDir: "{app}"; Flags: ignoreversion [Icons] Name: "{group}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}" diff --git a/kcc.py b/kcc.py index 347a4ac5..b05e8c10 100755 --- a/kcc.py +++ b/kcc.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/kindlecomicconverter/KCC_gui.py b/kindlecomicconverter/KCC_gui.py index 7959b40e..8449e372 100644 --- a/kindlecomicconverter/KCC_gui.py +++ b/kindlecomicconverter/KCC_gui.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the @@ -22,8 +22,9 @@ from urllib.parse import unquote from urllib.request import urlopen, urlretrieve, Request from time import sleep -from shutil import move +from shutil import move, rmtree from subprocess import STDOUT, PIPE +# noinspection PyUnresolvedReferences from PyQt5 import QtGui, QtCore, QtWidgets, QtNetwork from xml.dom.minidom import parse from xml.sax.saxutils import escape @@ -31,7 +32,8 @@ from copy import copy from distutils.version import StrictVersion from raven import Client -from .shared import md5Checksum, HTMLStripper, sanitizeTrace +from tempfile import gettempdir +from .shared import md5Checksum, HTMLStripper, sanitizeTrace, walkLevel from . import __version__ from . import comic2ebook from . import metadata @@ -173,7 +175,7 @@ def getNewVersion(self): move(path[0], path[0] + '.exe') MW.hideProgressBar.emit() MW.modeConvert.emit(1) - Popen(path[0] + '.exe /SP- /silent /noicons', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) + Popen(path[0] + '.exe /SP- /silent /noicons', stdout=PIPE, stderr=STDOUT, shell=True) MW.forceShutdown.emit() except Exception: MW.addMessage.emit('Failed to download the update!', 'warning', False) @@ -238,6 +240,7 @@ def clean(self): MW.addTrayMessage.emit('Conversion interrupted.', 'Critical') MW.modeConvert.emit(1) + # noinspection PyUnboundLocalVariable def run(self): MW.modeConvert.emit(0) @@ -477,20 +480,11 @@ def selectFile(self): if self.needClean: self.needClean = False GUI.jobList.clear() - if self.UnRAR: - if self.sevenza: - fnames = QtWidgets.QFileDialog.getOpenFileNames(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.cbr *.cb7 *.zip *.rar *.7z *.pdf)') - else: - fnames = QtWidgets.QFileDialog.getOpenFileNames(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.cbr *.zip *.rar *.pdf)') + if self.sevenzip: + fnames = QtWidgets.QFileDialog.getOpenFileNames(MW, 'Select file', self.lastPath, + 'Comic (*.cbz *.cbr *.cb7 *.zip *.rar *.7z *.pdf)') else: - if self.sevenza: - fnames = QtWidgets.QFileDialog.getOpenFileNames(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.cb7 *.zip *.7z *.pdf)') - else: - fnames = QtWidgets.QFileDialog.getOpenFileNames(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.zip *.pdf)') + fnames = QtWidgets.QFileDialog.getOpenFileNames(MW, 'Select file', self.lastPath, 'Comic (*.pdf)') for fname in fnames[0]: if fname != '': if sys.platform.startswith('win'): @@ -509,20 +503,12 @@ def selectFileMetaEditor(self): sname = sname.replace('/', '\\') self.lastPath = os.path.abspath(sname) else: - if self.UnRAR: - if self.sevenza: - fname = QtWidgets.QFileDialog.getOpenFileName(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.cbr *.cb7)') - else: - fname = QtWidgets.QFileDialog.getOpenFileName(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.cbr)') + if self.sevenzip: + fname = QtWidgets.QFileDialog.getOpenFileName(MW, 'Select file', self.lastPath, + 'Comic (*.cbz *.cbr *.cb7)') else: - if self.sevenza: - fname = QtWidgets.QFileDialog.getOpenFileName(MW, 'Select file', self.lastPath, - 'Comic (*.cbz *.cb7)') - else: - fname = QtWidgets.QFileDialog.getOpenFileName(MW, 'Select file', self.lastPath, - 'Comic (*.cbz)') + fname = [''] + self.showDialog("Editor is disabled due to a lack of 7z.", 'error') if fname[0] != '': if sys.platform.startswith('win'): sname = fname[0].replace('/', '\\') @@ -812,16 +798,9 @@ def handleMessage(self, message): if self.needClean: self.needClean = False GUI.jobList.clear() - if self.UnRAR: - if self.sevenza: - formats = ['.cbz', '.cbr', '.cb7', '.zip', '.rar', '.7z', '.pdf'] - else: - formats = ['.cbz', '.cbr', '.zip', '.rar', '.pdf'] - else: - if self.sevenza: - formats = ['.cbz', '.cb7', '.zip', '.7z', '.pdf'] - else: - formats = ['.cbz', '.zip', '.pdf'] + formats = ['.pdf'] + if self.sevenzip: + formats.extend(['.cb7', '.7z', '.cbz', '.zip', '.cbr', '.rar']) if os.path.isdir(message): GUI.jobList.addItem(message) GUI.jobList.scrollToBottom() @@ -831,7 +810,7 @@ def handleMessage(self, message): GUI.jobList.addItem(message) GUI.jobList.scrollToBottom() else: - self.addMessage('This file type is unsupported!', 'error') + self.addMessage('Unsupported file type for ' + message, 'error') def dragAndDrop(self, e): e.accept() @@ -857,10 +836,11 @@ def detectKindleGen(self, startup=False): os.chmod('/usr/local/bin/kindlegen', 0o755) except Exception: pass - kindleGenExitCode = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - if kindleGenExitCode.wait() == 0: + kindleGenExitCode = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, shell=True) + kindleGenExitCode.communicate() + if kindleGenExitCode.returncode == 0: self.kindleGen = True - versionCheck = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) + versionCheck = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, shell=True) for line in versionCheck.stdout: line = line.decode("utf-8") if 'Amazon kindlegen' in line: @@ -909,6 +889,7 @@ def __init__(self, kccapp, kccwindow): self.targetDirectory = '' self.sentry = Client(release=__version__) if sys.platform.startswith('win'): + # noinspection PyUnresolvedReferences from psutil import BELOW_NORMAL_PRIORITY_CLASS self.p = Process(os.getpid()) self.p.nice(BELOW_NORMAL_PRIORITY_CLASS) @@ -934,8 +915,8 @@ def __init__(self, kccapp, kccwindow): 'DefaultUpscale': True, 'Label': 'KV'}, "Kindle Voyage": {'PVOptions': True, 'ForceExpert': False, 'DefaultFormat': 0, 'DefaultUpscale': True, 'Label': 'KV'}, - "Kindle PW 3": {'PVOptions': True, 'ForceExpert': False, 'DefaultFormat': 0, - 'DefaultUpscale': True, 'Label': 'KV'}, + "Kindle PW 3/4": {'PVOptions': True, 'ForceExpert': False, 'DefaultFormat': 0, + 'DefaultUpscale': True, 'Label': 'KV'}, "Kindle PW 1/2": {'PVOptions': True, 'ForceExpert': False, 'DefaultFormat': 0, 'DefaultUpscale': False, 'Label': 'KPW'}, "Kindle": {'PVOptions': True, 'ForceExpert': False, 'DefaultFormat': 0, @@ -956,6 +937,8 @@ def __init__(self, kccapp, kccwindow): 'DefaultUpscale': True, 'Label': 'KoAH2O'}, "Kobo Aura ONE": {'PVOptions': False, 'ForceExpert': False, 'DefaultFormat': 1, 'DefaultUpscale': True, 'Label': 'KoAO'}, + "Kobo Forma": {'PVOptions': False, 'ForceExpert': False, 'DefaultFormat': 1, + 'DefaultUpscale': True, 'Label': 'KoF'}, "Other": {'PVOptions': False, 'ForceExpert': True, 'DefaultFormat': 1, 'DefaultUpscale': False, 'Label': 'OTHER'}, "Kindle 1": {'PVOptions': False, 'ForceExpert': False, 'DefaultFormat': 0, @@ -971,10 +954,11 @@ def __init__(self, kccapp, kccwindow): "Kindle Oasis 2", "Kindle Oasis", "Kindle Voyage", - "Kindle PW 3", + "Kindle PW 3/4", "Kindle PW 1/2", "Kindle", "Separator", + "Kobo Forma", "Kobo Aura ONE", "Kobo Aura H2O", "Kobo Aura HD", @@ -1007,22 +991,14 @@ def __init__(self, kccapp, kccwindow): self.addMessage('Since you are a new user of KCC please see few ' 'important tips.', 'info') - rarExitCode = Popen('unrar', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - rarExitCode = rarExitCode.wait() - if rarExitCode == 0 or rarExitCode == 1 or rarExitCode == 7: - self.UnRAR = True - else: - self.UnRAR = False - self.addMessage('Cannot find UnRAR!' - ' Processing of CBR/RAR files will be disabled.', 'warning') - sevenzaExitCode = Popen('7za', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - sevenzaExitCode = sevenzaExitCode.wait() - if sevenzaExitCode == 0 or sevenzaExitCode == 7: - self.sevenza = True + process = Popen('7z', stdout=PIPE, stderr=STDOUT, shell=True) + process.communicate() + if process.returncode == 0 or process.returncode == 7: + self.sevenzip = True else: - self.sevenza = False - self.addMessage('Cannot find 7za!' - ' Processing of CB7/7Z files will be disabled.', 'warning') + self.sevenzip = False + self.addMessage('Cannot find 7z!' + ' Processing of archives will be disabled.', 'warning') self.detectKindleGen(True) APP.messageFromOtherInstance.connect(self.handleMessage) @@ -1092,6 +1068,12 @@ def __init__(self, kccapp, kccwindow): self.versionCheck.start() self.tray.show() + # Cleanup unfisnished conversion + for root, dirs, _ in walkLevel(gettempdir(), 0): + for tempdir in dirs: + if tempdir.startswith('KCC-'): + rmtree(os.path.join(root, tempdir), True) + if self.windowSize != '0x0': x, y = self.windowSize.split('x') MW.resize(int(x), int(y)) @@ -1103,7 +1085,7 @@ def __init__(self, kccapp, kccwindow): class KCCGUI_MetaEditor(KCC_ui_editor.Ui_editorDialog): def loadData(self, file): self.parser = metadata.MetadataParser(file) - if self.parser.compressor == 'rar': + if self.parser.format in ['RAR', 'RAR5']: self.editorWidget.setEnabled(False) self.okButton.setEnabled(False) self.statusLabel.setText('CBR metadata are read-only.') @@ -1111,11 +1093,8 @@ def loadData(self, file): self.editorWidget.setEnabled(True) self.okButton.setEnabled(True) self.statusLabel.setText('Separate authors with a comma.') - for field in (self.seriesLine, self.volumeLine, self.numberLine, self.muidLine): - if field.objectName() == 'muidLine': - field.setText(self.parser.data['MUid']) - else: - field.setText(self.parser.data[field.objectName().capitalize()[:-4]]) + for field in (self.seriesLine, self.volumeLine, self.numberLine): + field.setText(self.parser.data[field.objectName().capitalize()[:-4]]) for field in (self.writerLine, self.pencillerLine, self.inkerLine, self.coloristLine): field.setText(', '.join(self.parser.data[field.objectName().capitalize()[:-4] + 's'])) if self.seriesLine.text() == '': @@ -1125,12 +1104,9 @@ def loadData(self, file): self.seriesLine.setText(file.split('\\')[-1].split('/')[-1].split('.')[0]) def saveData(self): - for field in (self.volumeLine, self.numberLine, self.muidLine): + for field in (self.volumeLine, self.numberLine): if field.text().isnumeric() or self.cleanData(field.text()) == '': - if field.objectName() == 'muidLine': - self.parser.data['MUid'] = self.cleanData(field.text()) - else: - self.parser.data[field.objectName().capitalize()[:-4]] = self.cleanData(field.text()) + self.parser.data[field.objectName().capitalize()[:-4]] = self.cleanData(field.text()) else: self.statusLabel.setText(field.objectName().capitalize()[:-4] + ' field must be a number.') break diff --git a/kindlecomicconverter/KCC_ui_editor.py b/kindlecomicconverter/KCC_ui_editor.py index 56dc07db..5deba910 100644 --- a/kindlecomicconverter/KCC_ui_editor.py +++ b/kindlecomicconverter/KCC_ui_editor.py @@ -66,13 +66,6 @@ def setupUi(self, editorDialog): self.coloristLine = QtWidgets.QLineEdit(self.editorWidget) self.coloristLine.setObjectName("coloristLine") self.gridLayout.addWidget(self.coloristLine, 6, 1, 1, 1) - self.label_8 = QtWidgets.QLabel(self.editorWidget) - self.label_8.setOpenExternalLinks(True) - self.label_8.setObjectName("label_8") - self.gridLayout.addWidget(self.label_8, 7, 0, 1, 1) - self.muidLine = QtWidgets.QLineEdit(self.editorWidget) - self.muidLine.setObjectName("muidLine") - self.gridLayout.addWidget(self.muidLine, 7, 1, 1, 1) self.verticalLayout.addWidget(self.editorWidget) self.optionWidget = QtWidgets.QWidget(editorDialog) self.optionWidget.setObjectName("optionWidget") @@ -117,7 +110,6 @@ def retranslateUi(self, editorDialog): self.label_5.setText(_translate("editorDialog", "Penciller:")) self.label_6.setText(_translate("editorDialog", "Inker:")) self.label_7.setText(_translate("editorDialog", "Colorist:")) - self.label_8.setText(_translate("editorDialog", "

MUid:

")) self.okButton.setText(_translate("editorDialog", "Save")) self.cancelButton.setText(_translate("editorDialog", "Cancel")) diff --git a/kindlecomicconverter/__init__.py b/kindlecomicconverter/__init__.py index 2ccec97f..fe7588f4 100644 --- a/kindlecomicconverter/__init__.py +++ b/kindlecomicconverter/__init__.py @@ -1,4 +1,4 @@ -__version__ = '5.4.5' +__version__ = '5.5.0' __license__ = 'ISC' -__copyright__ = '2012-2018, Ciro Mattia Gonano , Pawel Jastrzebski ' +__copyright__ = '2012-2019, Ciro Mattia Gonano , Pawel Jastrzebski ' __docformat__ = 'restructuredtext en' diff --git a/kindlecomicconverter/cbxarchive.py b/kindlecomicconverter/cbxarchive.py deleted file mode 100644 index 94545aed..00000000 --- a/kindlecomicconverter/cbxarchive.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all -# copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE -# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL -# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA -# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. -# - -import os -from zipfile import is_zipfile, ZipFile -from subprocess import STDOUT, PIPE -from psutil import Popen -from shutil import move -from . import rarfile -from .shared import check7ZFile as is_7zfile - - -class CBxArchive: - def __init__(self, fname): - self.fname = fname - if is_zipfile(fname): - self.compressor = 'zip' - elif rarfile.is_rarfile(fname): - self.compressor = 'rar' - elif is_7zfile(fname): - self.compressor = '7z' - else: - self.compressor = None - - def isCbxFile(self): - return self.compressor is not None - - def extractCBZ(self, targetdir): - cbzFile = ZipFile(self.fname) - filelist = [] - for f in cbzFile.namelist(): - if f.startswith('__MACOSX') or f.endswith('.DS_Store') or f.endswith('humbs.db'): - pass - elif f.endswith('/'): - os.makedirs(os.path.join(targetdir, f), exist_ok=True) - else: - filelist.append(f) - cbzFile.extractall(targetdir, filelist) - - def extractCBR(self, targetdir): - cbrFile = rarfile.RarFile(self.fname) - cbrFile.extractall(targetdir) - for root, _, filenames in os.walk(targetdir): - for filename in filenames: - if filename.startswith('__MACOSX') or filename.endswith('.DS_Store') or filename.endswith('humbs.db'): - os.remove(os.path.join(root, filename)) - - def extractCB7(self, targetdir): - output = Popen('7za x "' + self.fname + '" -xr!__MACOSX -xr!.DS_Store -xr!thumbs.db -xr!Thumbs.db -o"' + - targetdir + '"', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - extracted = False - for line in output.stdout: - if b"Everything is Ok" in line: - extracted = True - if not extracted: - raise OSError - - def extract(self, targetdir): - if self.compressor == 'rar': - self.extractCBR(targetdir) - elif self.compressor == 'zip': - self.extractCBZ(targetdir) - elif self.compressor == '7z': - self.extractCB7(targetdir) - adir = os.listdir(targetdir) - if 'ComicInfo.xml' in adir: - adir.remove('ComicInfo.xml') - if len(adir) == 1 and os.path.isdir(os.path.join(targetdir, adir[0])): - for f in os.listdir(os.path.join(targetdir, adir[0])): - move(os.path.join(targetdir, adir[0], f), targetdir) - os.rmdir(os.path.join(targetdir, adir[0])) - return targetdir diff --git a/kindlecomicconverter/comic2ebook.py b/kindlecomicconverter/comic2ebook.py index b5608251..78004143 100755 --- a/kindlecomicconverter/comic2ebook.py +++ b/kindlecomicconverter/comic2ebook.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the @@ -22,9 +22,7 @@ import sys from time import strftime, gmtime from copy import copy -from glob import glob -from json import loads -from urllib.request import Request, urlopen +from glob import glob, escape from re import sub from stat import S_IWRITE, S_IREAD, S_IEXEC from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED @@ -37,7 +35,7 @@ from PIL import Image from subprocess import STDOUT, PIPE from psutil import Popen, virtual_memory, disk_usage -from html import escape +from html import escape as hescape try: from PyQt5 import QtCore except ImportError: @@ -45,7 +43,7 @@ from .shared import md5Checksum, getImageFileName, walkSort, walkLevel, sanitizeTrace from . import comic2panel from . import image -from . import cbxarchive +from . import comicarchive from . import pdfjpgextract from . import dualmetafix from . import metadata @@ -61,7 +59,7 @@ def main(argv=None): parser.print_help() return 0 if sys.platform.startswith('win'): - sources = set([source for arg in args for source in glob(arg)]) + sources = set([source for arg in args for source in glob(escape(arg))]) else: sources = set(args) if len(sources) == 0: @@ -112,7 +110,7 @@ def buildHTML(path, imgfile, imgfilepath): "\n", "\n", "\n", - "", escape(filename[0]), "\n", + "", hescape(filename[0]), "\n", "\n", "\n" @@ -210,7 +208,7 @@ def buildNCX(dstdir, title, chapters, chapternames): "\n", "\n", "\n", - "", escape(title), "\n", + "", hescape(title), "\n", "\n"]) for chapter in chapters: folder = chapter[0].replace(os.path.join(dstdir, 'OEBPS'), '').lstrip('/').lstrip('\\\\') @@ -222,7 +220,7 @@ def buildNCX(dstdir, title, chapters, chapternames): elif os.path.basename(folder) != "Text": title = chapternames[os.path.basename(folder)] f.write("" + - escape(title) + "\n") f.write("\n") f.close() @@ -235,7 +233,7 @@ def buildNAV(dstdir, title, chapters, chapternames): "\n", "\n", "\n", - "" + escape(title) + "\n", + "" + hescape(title) + "\n", "\n", "\n", "\n", @@ -248,7 +246,7 @@ def buildNAV(dstdir, title, chapters, chapternames): title = chapternames[chapter[1]] elif os.path.basename(folder) != "Text": title = chapternames[os.path.basename(folder)] - f.write("
  • " + escape(title) + "
    • \n") + f.write("
  • " + hescape(title) + "
    • \n") f.writelines(["\n", "\n", "\n\n") f.close() @@ -278,7 +276,7 @@ def buildOPF(dstdir, title, filelist, cover=None): "xmlns=\"http://www.idpf.org/2007/opf\">\n", "\n", - "", title, "\n", + "", hescape(title), "\n", "en-US\n", "urn:uuid:", options.uuid, "\n", "KindleComicConverter-" + __version__ + "\n"]) @@ -597,16 +595,12 @@ def getWorkFolder(afile): raise UserWarning("Failed to extract images from PDF file.") else: workdir = mkdtemp('', 'KCC-') - cbx = cbxarchive.CBxArchive(afile) - if cbx.isCbxFile(): - try: - path = cbx.extract(workdir) - except Exception: - rmtree(workdir, True) - raise UserWarning("Failed to extract archive.") - else: + try: + cbx = comicarchive.ComicArchive(afile) + path = cbx.extract(workdir) + except OSError as e: rmtree(workdir, True) - raise UserWarning("Failed to detect archive format.") + raise UserWarning(e.strerror) else: raise UserWarning("Failed to open source file/directory.") sanitizePermissions(path) @@ -652,7 +646,6 @@ def getOutputFilename(srcpath, wantedname, ext, tomenumber): def getComicInfo(path, originalpath): xmlPath = os.path.join(path, 'ComicInfo.xml') options.authors = ['KCC'] - options.remoteCovers = {} options.chapters = [] options.summary = '' titleSuffix = '' @@ -673,7 +666,7 @@ def getComicInfo(path, originalpath): options.authors = [] if defaultTitle: if xml.data['Series']: - options.title = escape(xml.data['Series']) + options.title = hescape(xml.data['Series']) if xml.data['Volume']: titleSuffix += ' V' + xml.data['Volume'].zfill(2) if xml.data['Number']: @@ -681,35 +674,19 @@ def getComicInfo(path, originalpath): options.title += titleSuffix for field in ['Writers', 'Pencillers', 'Inkers', 'Colorists']: for person in xml.data[field]: - options.authors.append(escape(person)) + options.authors.append(hescape(person)) if len(options.authors) > 0: options.authors = list(set(options.authors)) options.authors.sort() else: options.authors = ['KCC'] - if xml.data['MUid']: - options.remoteCovers = getCoversFromMCB(xml.data['MUid']) if xml.data['Bookmarks']: options.chapters = xml.data['Bookmarks'] if xml.data['Summary']: - options.summary = escape(xml.data['Summary']) + options.summary = hescape(xml.data['Summary']) os.remove(xmlPath) -def getCoversFromMCB(mangaid): - covers = {} - try: - jsonRaw = urlopen(Request('http://mcd.iosphe.re/api/v1/series/' + mangaid + '/', - headers={'User-Agent': 'KindleComicConverter/' + __version__})) - jsonData = loads(jsonRaw.read().decode('utf-8')) - for volume in jsonData['Covers']['a']: - if volume['Side'] == 'front': - covers[int(volume['Volume'])] = volume['Raw'] - except Exception: - return {} - return covers - - def getDirectorySize(start_path='.'): total_size = 0 for dirpath, _, filenames in os.walk(start_path): @@ -790,7 +767,8 @@ def splitDirectory(path): level = -1 for root, _, files in os.walk(os.path.join(path, 'OEBPS', 'Images')): for f in files: - if f.endswith('.jpg') or f.endswith('.jpeg') or f.endswith('.png') or f.endswith('.gif'): + if f.endswith('.jpg') or f.endswith('.jpeg') or f.endswith('.png') or f.endswith('.gif') or \ + f.endswith('.webp'): newLevel = os.path.join(root, f).replace(os.path.join(path, 'OEBPS', 'Images'), '').count(os.sep) if level != -1 and level != newLevel: level = 0 @@ -872,7 +850,7 @@ def detectCorruption(tmppath, orgpath): if 'decoder' in str(err) and 'not available' in str(err): raise RuntimeError('Pillow was compiled without JPG and/or PNG decoder.') else: - raise RuntimeError('Image file %s is corrupted.' % pathOrg) + raise RuntimeError('Image file %s is corrupted. Error: %s' % (pathOrg, str(err))) else: os.remove(os.path.join(root, name)) if alreadyProcessed: @@ -932,7 +910,7 @@ def makeParser(): mainOptions.add_option("-p", "--profile", action="store", dest="profile", default="KV", help="Device profile (Available options: K1, K2, K34, K578, KDX, KPW, KV, KO, KoMT, KoG," - " KoGHD, KoA, KoAHD, KoAH2O, KoAO) [Default=KV]") + " KoGHD, KoA, KoAHD, KoAH2O, KoAO, KoF) [Default=KV]") mainOptions.add_option("-m", "--manga-style", action="store_true", dest="righttoleft", default=False, help="Manga style (right-to-left reading and splitting)") mainOptions.add_option("-q", "--hq", action="store_true", dest="hq", default=False, @@ -1053,21 +1031,17 @@ def checkOptions(): def checkTools(source): source = source.upper() - if source.endswith('.CBR') or source.endswith('.RAR'): - rarExitCode = Popen('unrar', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - rarExitCode = rarExitCode.wait() - if rarExitCode != 0 and rarExitCode != 1 and rarExitCode != 7: - print('ERROR: UnRAR is missing!') - exit(1) - elif source.endswith('.CB7') or source.endswith('.7Z'): - sevenzaExitCode = Popen('7za', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - sevenzaExitCode = sevenzaExitCode.wait() - if sevenzaExitCode != 0 and sevenzaExitCode != 7: - print('ERROR: 7za is missing!') + if source.endswith('.CB7') or source.endswith('.7Z') or source.endswith('.RAR') or source.endswith('.CBR') or \ + source.endswith('.ZIP') or source.endswith('.CBZ'): + process = Popen('7z', stdout=PIPE, stderr=STDOUT, shell=True) + process.communicate() + if process.returncode != 0 and process.returncode != 7: + print('ERROR: 7z is missing!') exit(1) if options.format == 'MOBI': - kindleGenExitCode = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - if kindleGenExitCode.wait() != 0: + kindleGenExitCode = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, shell=True) + kindleGenExitCode.communicate() + if kindleGenExitCode.returncode != 0: print('ERROR: KindleGen is missing!') exit(1) @@ -1214,7 +1188,7 @@ def makeMOBIWorker(item): try: if os.path.getsize(item) < 629145600: output = Popen('kindlegen -dont_append_source -locale en "' + item + '"', - stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) + stdout=PIPE, stderr=STDOUT, shell=True) for line in output.stdout: line = line.decode('utf-8') # ERROR: Generic error diff --git a/kindlecomicconverter/comic2panel.py b/kindlecomicconverter/comic2panel.py index c3d9e1ce..cd5d87ac 100644 --- a/kindlecomicconverter/comic2panel.py +++ b/kindlecomicconverter/comic2panel.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/kindlecomicconverter/comicarchive.py b/kindlecomicconverter/comicarchive.py new file mode 100644 index 00000000..435a99cf --- /dev/null +++ b/kindlecomicconverter/comicarchive.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2012-2014 Ciro Mattia Gonano +# Copyright (c) 2013-2019 Pawel Jastrzebski +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all +# copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA +# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. +# + +import os +from psutil import Popen +from shutil import move +from subprocess import STDOUT, PIPE +from xml.dom.minidom import parseString +from xml.parsers.expat import ExpatError + + +class ComicArchive: + def __init__(self, filepath): + self.filepath = filepath + self.type = None + if not os.path.isfile(self.filepath): + raise OSError('File not found.') + process = Popen('7z l -y -p1 "' + self.filepath + '"', stderr=STDOUT, stdout=PIPE, shell=True) + for line in process.stdout: + if b'Type =' in line: + self.type = line.rstrip().decode().split(' = ')[1].upper() + break + process.communicate() + if process.returncode != 0: + raise OSError('Archive is corrupted or encrypted.') + elif self.type not in ['7Z', 'RAR', 'RAR5', 'ZIP']: + raise OSError('Unsupported archive format.') + + def extract(self, targetdir): + if not os.path.isdir(targetdir): + raise OSError('Target directory don\'t exist.') + process = Popen('7z x -y -xr!__MACOSX -xr!.DS_Store -xr!thumbs.db -xr!Thumbs.db -o"' + targetdir + '" "' + + self.filepath + '"', stdout=PIPE, stderr=STDOUT, shell=True) + process.communicate() + if process.returncode != 0: + raise OSError('Failed to extract archive.') + tdir = os.listdir(targetdir) + if 'ComicInfo.xml' in tdir: + tdir.remove('ComicInfo.xml') + if len(tdir) == 1 and os.path.isdir(os.path.join(targetdir, tdir[0])): + for f in os.listdir(os.path.join(targetdir, tdir[0])): + move(os.path.join(targetdir, tdir[0], f), targetdir) + os.rmdir(os.path.join(targetdir, tdir[0])) + return targetdir + + def addFile(self, sourcefile): + if self.type in ['RAR', 'RAR5']: + raise NotImplementedError + process = Popen('7z a -y "' + self.filepath + '" "' + sourcefile + '"', + stdout=PIPE, stderr=STDOUT, shell=True) + process.communicate() + if process.returncode != 0: + raise OSError('Failed to add the file.') + + def extractMetadata(self): + process = Popen('7z x -y -so "' + self.filepath + '" ComicInfo.xml', + stdout=PIPE, stderr=STDOUT, shell=True) + xml = process.communicate() + if process.returncode != 0: + raise OSError('Failed to extract archive.') + try: + return parseString(xml[0]) + except ExpatError: + return None diff --git a/kindlecomicconverter/dualmetafix.py b/kindlecomicconverter/dualmetafix.py index e6be2557..c05dc857 100644 --- a/kindlecomicconverter/dualmetafix.py +++ b/kindlecomicconverter/dualmetafix.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Based on initial version of DualMetaFix. Copyright (C) 2013 Kevin Hendricks -# Changes for KCC Copyright (C) 2014-2018 Pawel Jastrzebski +# Changes for KCC Copyright (C) 2014-2019 Pawel Jastrzebski # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ class DualMetaFixException(Exception): pass + # palm database offset constants number_of_pdb_records = 76 first_pdb_record = 78 diff --git a/kindlecomicconverter/image.py b/kindlecomicconverter/image.py index 5f195a59..118ef3c8 100755 --- a/kindlecomicconverter/image.py +++ b/kindlecomicconverter/image.py @@ -4,7 +4,7 @@ # Copyright (C) 2011 Stanislav (proDOOMman) Kosolapov # Copyright (c) 2016 Alberto Planas # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,12 +20,8 @@ # along with this program. If not, see . import os -from io import BytesIO -from urllib.request import Request, urlopen -from urllib.parse import quote from PIL import Image, ImageOps, ImageStat, ImageChops, ImageFilter from .shared import md5Checksum -from . import __version__ class ProfileData: @@ -86,7 +82,7 @@ def __init__(self): 'K578': ("Kindle", (600, 800), Palette16, 1.8), 'KDX': ("Kindle DX/DXG", (824, 1000), Palette16, 1.8), 'KPW': ("Kindle Paperwhite 1/2", (758, 1024), Palette16, 1.8), - 'KV': ("Kindle Paperwhite 3/Voyage/Oasis", (1072, 1448), Palette16, 1.8), + 'KV': ("Kindle Paperwhite 3/4/Voyage/Oasis", (1072, 1448), Palette16, 1.8), 'KO': ("Kindle Oasis 2", (1264, 1680), Palette16, 1.8), 'KoMT': ("Kobo Mini/Touch", (600, 800), Palette16, 1.8), 'KoG': ("Kobo Glo", (768, 1024), Palette16, 1.8), @@ -95,12 +91,14 @@ def __init__(self): 'KoAHD': ("Kobo Aura HD", (1080, 1440), Palette16, 1.8), 'KoAH2O': ("Kobo Aura H2O", (1080, 1430), Palette16, 1.8), 'KoAO': ("Kobo Aura ONE", (1404, 1872), Palette16, 1.8), + 'KoF': ("Kobo Forma", (1440, 1920), Palette16, 1.8), 'OTHER': ("Other", (0, 0), Palette16, 1.8), } class ComicPageParser: def __init__(self, source, options): + Image.MAX_IMAGE_PIXELS = int(2048 * 2048 * 2048 // 4 // 3) self.opt = options self.source = source self.size = self.opt.profileData[1] @@ -345,15 +343,7 @@ def __init__(self, source, target, opt, tomeid): self.tomeid = 1 else: self.tomeid = tomeid - if self.tomeid in self.options.remoteCovers: - try: - source = urlopen(Request(quote(self.options.remoteCovers[self.tomeid]).replace('%3A', ':', 1), - headers={'User-Agent': 'KindleComicConverter/' + __version__})).read() - self.image = Image.open(BytesIO(source)) - except Exception: - self.image = Image.open(source) - else: - self.image = Image.open(source) + self.image = Image.open(source) self.process() def process(self): diff --git a/kindlecomicconverter/kindle.py b/kindlecomicconverter/kindle.py index f9595f6f..99435904 100644 --- a/kindlecomicconverter/kindle.py +++ b/kindlecomicconverter/kindle.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/kindlecomicconverter/metadata.py b/kindlecomicconverter/metadata.py index da16718a..b431be17 100644 --- a/kindlecomicconverter/metadata.py +++ b/kindlecomicconverter/metadata.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the @@ -18,14 +18,9 @@ import os from xml.dom.minidom import parse, Document -from re import compile -from zipfile import is_zipfile, ZipFile, ZIP_DEFLATED -from subprocess import STDOUT, PIPE -from psutil import Popen from tempfile import mkdtemp from shutil import rmtree -from .shared import removeFromZIP, check7ZFile as is_7zfile -from . import rarfile +from . import comicarchive class MetadataParser: @@ -39,50 +34,20 @@ def __init__(self, source): 'Inkers': [], 'Colorists': [], 'Summary': '', - 'MUid': '', 'Bookmarks': []} self.rawdata = None - self.compressor = None + self.format = None if self.source.endswith('.xml') and os.path.exists(self.source): self.rawdata = parse(self.source) - self.parseXML() elif not self.source.endswith('.xml'): - if is_zipfile(self.source): - self.compressor = 'zip' - with ZipFile(self.source) as zip_file: - for member in zip_file.namelist(): - if member != 'ComicInfo.xml': - continue - with zip_file.open(member) as xml_file: - self.rawdata = parse(xml_file) - elif rarfile.is_rarfile(self.source): - self.compressor = 'rar' - with rarfile.RarFile(self.source) as rar_file: - for member in rar_file.namelist(): - if member != 'ComicInfo.xml': - continue - with rar_file.open(member) as xml_file: - self.rawdata = parse(xml_file) - elif is_7zfile(self.source): - self.compressor = '7z' - workdir = mkdtemp('', 'KCC-') - tmpXML = os.path.join(workdir, 'ComicInfo.xml') - output = Popen('7za e "' + self.source + '" ComicInfo.xml -o"' + workdir + '"', - stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - extracted = False - for line in output.stdout: - if b"Everything is Ok" in line or b"No files to process" in line: - extracted = True - if not extracted: - rmtree(workdir) - raise OSError('Failed to extract 7ZIP file.') - if os.path.isfile(tmpXML): - self.rawdata = parse(tmpXML) - rmtree(workdir) - else: - raise OSError('Failed to detect archive format.') - if self.rawdata: - self.parseXML() + try: + cbx = comicarchive.ComicArchive(self.source) + self.rawdata = cbx.extractMetadata() + self.format = cbx.type + except OSError as e: + raise UserWarning(e.strerror) + if self.rawdata: + self.parseXML() def parseXML(self): if len(self.rawdata.getElementsByTagName('Series')) != 0: @@ -99,11 +64,6 @@ def parseXML(self): self.data[field + 's'].append(person) self.data[field + 's'] = list(set(self.data[field + 's'])) self.data[field + 's'].sort() - if len(self.rawdata.getElementsByTagName('ScanInformation')) != 0: - coverId = compile('(MCD\\()(\\d+)(\\))')\ - .search(self.rawdata.getElementsByTagName('ScanInformation')[0].firstChild.nodeValue) - if coverId: - self.data['MUid'] = coverId.group(2) if len(self.rawdata.getElementsByTagName('Page')) != 0: for page in self.rawdata.getElementsByTagName('Page'): if 'Bookmark' in page.attributes and 'Image' in page.attributes: @@ -116,8 +76,7 @@ def saveXML(self): for row in (['Series', self.data['Series']], ['Volume', self.data['Volume']], ['Number', self.data['Number']], ['Writer', ', '.join(self.data['Writers'])], ['Penciller', ', '.join(self.data['Pencillers'])], ['Inker', ', '.join(self.data['Inkers'])], - ['Colorist', ', '.join(self.data['Colorists'])], ['Summary', self.data['Summary']], - ['ScanInformation', 'MCD(' + self.data['MUid'] + ')' if self.data['MUid'] else '']): + ['Colorist', ', '.join(self.data['Colorists'])], ['Summary', self.data['Summary']]): if self.rawdata.getElementsByTagName(row[0]): node = self.rawdata.getElementsByTagName(row[0])[0] if row[1]: @@ -138,8 +97,7 @@ def saveXML(self): for row in (['Series', self.data['Series']], ['Volume', self.data['Volume']], ['Number', self.data['Number']], ['Writer', ', '.join(self.data['Writers'])], ['Penciller', ', '.join(self.data['Pencillers'])], ['Inker', ', '.join(self.data['Inkers'])], - ['Colorist', ', '.join(self.data['Colorists'])], ['Summary', self.data['Summary']], - ['ScanInformation', 'MCD(' + self.data['MUid'] + ')' if self.data['MUid'] else '']): + ['Colorist', ', '.join(self.data['Colorists'])], ['Summary', self.data['Summary']]): if row[1]: main = doc.createElement(row[0]) root.appendChild(main) @@ -154,20 +112,9 @@ def saveXML(self): tmpXML = os.path.join(workdir, 'ComicInfo.xml') with open(tmpXML, 'w', encoding='utf-8') as f: self.rawdata.writexml(f, encoding='utf-8') - if is_zipfile(self.source): - removeFromZIP(self.source, 'ComicInfo.xml') - with ZipFile(self.source, mode='a', compression=ZIP_DEFLATED) as zip_file: - zip_file.write(tmpXML, arcname=tmpXML.split(os.sep)[-1]) - elif rarfile.is_rarfile(self.source): - raise NotImplementedError - elif is_7zfile(self.source): - output = Popen('7za a "' + self.source + '" "' + tmpXML + '"', - stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) - extracted = False - for line in output.stdout: - if b"Everything is Ok" in line: - extracted = True - if not extracted: - rmtree(workdir) - raise OSError('Failed to modify 7ZIP file.') + try: + cbx = comicarchive.ComicArchive(self.source) + cbx.addFile(tmpXML) + except OSError as e: + raise UserWarning(e.strerror) rmtree(workdir) diff --git a/kindlecomicconverter/pdfjpgextract.py b/kindlecomicconverter/pdfjpgextract.py index 258b1b45..9a24771f 100644 --- a/kindlecomicconverter/pdfjpgextract.py +++ b/kindlecomicconverter/pdfjpgextract.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Based upon the code snippet by Ned Batchelder # (http://nedbatchelder.com/blog/200712/extracting_jpgs_from_pdfs.html) diff --git a/kindlecomicconverter/rarfile.py b/kindlecomicconverter/rarfile.py deleted file mode 100644 index afb19a7b..00000000 --- a/kindlecomicconverter/rarfile.py +++ /dev/null @@ -1,1990 +0,0 @@ -# rarfile.py -# -# Copyright (c) 2005-2014 Marko Kreen -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -r"""RAR archive reader. - -This is Python module for Rar archive reading. The interface -is made as :mod:`zipfile`-like as possible. - -Basic logic: - - Parse archive structure with Python. - - Extract non-compressed files with Python - - Extract compressed files with unrar. - - Optionally write compressed data to temp file to speed up unrar, - otherwise it needs to scan whole archive on each execution. - -Example:: - - import rarfile - - rf = rarfile.RarFile('myarchive.rar') - for f in rf.infolist(): - print f.filename, f.file_size - if f.filename == 'README': - print(rf.read(f)) - -Archive files can also be accessed via file-like object returned -by :meth:`RarFile.open`:: - - import rarfile - - with rarfile.RarFile('archive.rar') as rf: - with rf.open('README') as f: - for ln in f: - print(ln.strip()) - -There are few module-level parameters to tune behaviour, -here they are with defaults, and reason to change it:: - - import rarfile - - # Set to full path of unrar.exe if it is not in PATH - rarfile.UNRAR_TOOL = "unrar" - - # Set to 0 if you don't look at comments and want to - # avoid wasting time for parsing them - rarfile.NEED_COMMENTS = 1 - - # Set up to 1 if you don't want to deal with decoding comments - # from unknown encoding. rarfile will try couple of common - # encodings in sequence. - rarfile.UNICODE_COMMENTS = 0 - - # Set to 1 if you prefer timestamps to be datetime objects - # instead tuples - rarfile.USE_DATETIME = 0 - - # Set to '/' to be more compatible with zipfile - rarfile.PATH_SEP = '\\' - -For more details, refer to source. - -""" - -__version__ = '2.7-kcc' - -# export only interesting items -__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] - -## -## Imports and compat - support both Python 2.x and 3.x -## - -import sys, os, struct, errno -from struct import pack, unpack -from binascii import crc32 -from tempfile import mkstemp -from subprocess import Popen, PIPE, STDOUT -from datetime import datetime - -# only needed for encryped headers -try: - from Crypto.Cipher import AES - try: - from hashlib import sha1 - except ImportError: - from sha import new as sha1 - _have_crypto = 1 -except ImportError: - _have_crypto = 0 - -# compat with 2.x -if sys.hexversion < 0x3000000: - # prefer 3.x behaviour - range = xrange - # py2.6 has broken bytes() - def bytes(s, enc): - return str(s) -else: - unicode = str - -# see if compat bytearray() is needed -try: - bytearray -except NameError: - import array - class bytearray: - def __init__(self, val = ''): - self.arr = array.array('B', val) - self.append = self.arr.append - self.__getitem__ = self.arr.__getitem__ - self.__len__ = self.arr.__len__ - def decode(self, *args): - return self.arr.tostring().decode(*args) - -# Optimized .readinto() requires memoryview -try: - memoryview - have_memoryview = 1 -except NameError: - have_memoryview = 0 - -# Struct() for older python -try: - from struct import Struct -except ImportError: - class Struct: - def __init__(self, fmt): - self.format = fmt - self.size = struct.calcsize(fmt) - def unpack(self, buf): - return unpack(self.format, buf) - def unpack_from(self, buf, ofs = 0): - return unpack(self.format, buf[ofs : ofs + self.size]) - def pack(self, *args): - return pack(self.format, *args) - -# file object superclass -try: - from io import RawIOBase -except ImportError: - class RawIOBase(object): - def close(self): - pass - - -## -## Module configuration. Can be tuned after importing. -## - -#: default fallback charset -DEFAULT_CHARSET = "windows-1252" - -#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed -TRY_ENCODINGS = ('utf8', 'utf-16le') - -#: 'unrar', 'rar' or full path to either one -UNRAR_TOOL = "unrar" - -#: Command line args to use for opening file for reading. -OPEN_ARGS = ('p', '-inul') - -#: Command line args to use for extracting file to disk. -EXTRACT_ARGS = ('x', '-y', '-idq') - -#: args for testrar() -TEST_ARGS = ('t', '-idq') - -# -# Allow use of tool that is not compatible with unrar. -# -# By default use 'bsdtar' which is 'tar' program that -# sits on top of libarchive. -# -# Problems with libarchive RAR backend: -# - Does not support solid archives. -# - Does not support password-protected archives. -# - -ALT_TOOL = 'bsdtar' -ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') -ALT_EXTRACT_ARGS = ('-x', '-f') -ALT_TEST_ARGS = ('-t', '-f') -ALT_CHECK_ARGS = ('--help',) - -#: whether to speed up decompression by using tmp archive -USE_EXTRACT_HACK = 0 - -#: limit the filesize for tmp archive usage -HACK_SIZE_LIMIT = 20*1024*1024 - -#: whether to parse file/archive comments. -NEED_COMMENTS = 1 - -#: whether to convert comments to unicode strings -UNICODE_COMMENTS = 0 - -#: Convert RAR time tuple into datetime() object -USE_DATETIME = 0 - -#: Separator for path name components. RAR internally uses '\\'. -#: Use '/' to be similar with zipfile. -PATH_SEP = '\\' - -## -## rar constants -## - -# block types -RAR_BLOCK_MARK = 0x72 # r -RAR_BLOCK_MAIN = 0x73 # s -RAR_BLOCK_FILE = 0x74 # t -RAR_BLOCK_OLD_COMMENT = 0x75 # u -RAR_BLOCK_OLD_EXTRA = 0x76 # v -RAR_BLOCK_OLD_SUB = 0x77 # w -RAR_BLOCK_OLD_RECOVERY = 0x78 # x -RAR_BLOCK_OLD_AUTH = 0x79 # y -RAR_BLOCK_SUB = 0x7a # z -RAR_BLOCK_ENDARC = 0x7b # { - -# flags for RAR_BLOCK_MAIN -RAR_MAIN_VOLUME = 0x0001 -RAR_MAIN_COMMENT = 0x0002 -RAR_MAIN_LOCK = 0x0004 -RAR_MAIN_SOLID = 0x0008 -RAR_MAIN_NEWNUMBERING = 0x0010 -RAR_MAIN_AUTH = 0x0020 -RAR_MAIN_RECOVERY = 0x0040 -RAR_MAIN_PASSWORD = 0x0080 -RAR_MAIN_FIRSTVOLUME = 0x0100 -RAR_MAIN_ENCRYPTVER = 0x0200 - -# flags for RAR_BLOCK_FILE -RAR_FILE_SPLIT_BEFORE = 0x0001 -RAR_FILE_SPLIT_AFTER = 0x0002 -RAR_FILE_PASSWORD = 0x0004 -RAR_FILE_COMMENT = 0x0008 -RAR_FILE_SOLID = 0x0010 -RAR_FILE_DICTMASK = 0x00e0 -RAR_FILE_DICT64 = 0x0000 -RAR_FILE_DICT128 = 0x0020 -RAR_FILE_DICT256 = 0x0040 -RAR_FILE_DICT512 = 0x0060 -RAR_FILE_DICT1024 = 0x0080 -RAR_FILE_DICT2048 = 0x00a0 -RAR_FILE_DICT4096 = 0x00c0 -RAR_FILE_DIRECTORY = 0x00e0 -RAR_FILE_LARGE = 0x0100 -RAR_FILE_UNICODE = 0x0200 -RAR_FILE_SALT = 0x0400 -RAR_FILE_VERSION = 0x0800 -RAR_FILE_EXTTIME = 0x1000 -RAR_FILE_EXTFLAGS = 0x2000 - -# flags for RAR_BLOCK_ENDARC -RAR_ENDARC_NEXT_VOLUME = 0x0001 -RAR_ENDARC_DATACRC = 0x0002 -RAR_ENDARC_REVSPACE = 0x0004 -RAR_ENDARC_VOLNR = 0x0008 - -# flags common to all blocks -RAR_SKIP_IF_UNKNOWN = 0x4000 -RAR_LONG_BLOCK = 0x8000 - -# Host OS types -RAR_OS_MSDOS = 0 -RAR_OS_OS2 = 1 -RAR_OS_WIN32 = 2 -RAR_OS_UNIX = 3 -RAR_OS_MACOS = 4 -RAR_OS_BEOS = 5 - -# Compression methods - '0'..'5' -RAR_M0 = 0x30 -RAR_M1 = 0x31 -RAR_M2 = 0x32 -RAR_M3 = 0x33 -RAR_M4 = 0x34 -RAR_M5 = 0x35 - -## -## internal constants -## - -RAR_ID = bytes("Rar!\x1a\x07\x00", 'ascii') -RAR5_ID = bytes("Rar!\x1a\x07\x01", 'ascii') -ZERO = bytes("\0", 'ascii') -EMPTY = bytes("", 'ascii') - -S_BLK_HDR = Struct(' HACK_SIZE_LIMIT: - use_hack = 0 - elif not USE_EXTRACT_HACK: - use_hack = 0 - - # now extract - if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0: - return self._open_clear(inf) - elif use_hack: - return self._open_hack(inf, psw) - else: - return self._open_unrar(self.rarfile, inf, psw) - - def read(self, fname, psw = None): - """Return uncompressed data for archive entry. - - For longer files using :meth:`RarFile.open` may be better idea. - - Parameters: - - fname - filename or RarInfo instance - psw - password to use for extracting. - """ - - f = self.open(fname, 'r', psw) - try: - return f.read() - finally: - f.close() - - def close(self): - """Release open resources.""" - pass - - def printdir(self): - """Print archive file list to stdout.""" - for f in self._info_list: - print(f.filename) - - def extract(self, member, path=None, pwd=None): - """Extract single file into current directory. - - Parameters: - - member - filename or :class:`RarInfo` instance - path - optional destination path - pwd - optional password to use - """ - if isinstance(member, RarInfo): - fname = member.filename - else: - fname = member - self._extract([fname], path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all files into current directory. - - Parameters: - - path - optional destination path - members - optional filename or :class:`RarInfo` instance list to extract - pwd - optional password to use - """ - fnlist = [] - if members is not None: - for m in members: - if isinstance(m, RarInfo): - fnlist.append(m.filename) - else: - fnlist.append(m) - self._extract(fnlist, path, pwd) - - def testrar(self): - """Let 'unrar' test the archive. - """ - cmd = [UNRAR_TOOL] + list(TEST_ARGS) - add_password_arg(cmd, self._password) - cmd.append(self.rarfile) - p = custom_popen(cmd) - output = p.communicate()[0] - check_returncode(p, output) - - def strerror(self): - """Return error string if parsing failed, - or None if no problems. - """ - return self._parse_error - - ## - ## private methods - ## - - def _set_error(self, msg, *args): - if args: - msg = msg % args - self._parse_error = msg - if self._strict: - raise BadRarFile(msg) - - # store entry - def _process_entry(self, item): - if item.type == RAR_BLOCK_FILE: - # use only first part - if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: - self._info_map[item.filename] = item - self._info_list.append(item) - # remember if any items require password - if item.needs_password(): - self._needs_password = True - elif len(self._info_list) > 0: - # final crc is in last block - old = self._info_list[-1] - old.CRC = item.CRC - old.compress_size += item.compress_size - - # parse new-style comment - if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': - if not NEED_COMMENTS: - pass - elif item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): - pass - elif item.flags & RAR_FILE_SOLID: - # file comment - cmt = self._read_comment_v3(item, self._password) - if len(self._info_list) > 0: - old = self._info_list[-1] - old.comment = cmt - else: - # archive comment - cmt = self._read_comment_v3(item, self._password) - self.comment = cmt - - if self._info_callback: - self._info_callback(item) - - # read rar - def _parse(self): - self._fd = None - try: - self._parse_real() - finally: - if self._fd: - self._fd.close() - self._fd = None - - def _parse_real(self): - fd = XFile(self.rarfile) - self._fd = fd - id = fd.read(len(RAR_ID)) - if id != RAR_ID and id != RAR5_ID: - raise NotRarFile("Not a Rar archive: "+self.rarfile) - - volume = 0 # first vol (.rar) is 0 - more_vols = 0 - endarc = 0 - volfile = self.rarfile - self._vol_list = [self.rarfile] - while 1: - if endarc: - h = None # don't read past ENDARC - else: - h = self._parse_header(fd) - if not h: - if more_vols: - volume += 1 - fd.close() - try: - volfile = self._next_volname(volfile) - fd = XFile(volfile) - except IOError: - self._set_error("Cannot open next volume: %s", volfile) - break - self._fd = fd - more_vols = 0 - endarc = 0 - self._vol_list.append(volfile) - continue - break - h.volume = volume - h.volume_file = volfile - - if h.type == RAR_BLOCK_MAIN and not self._main: - self._main = h - if h.flags & RAR_MAIN_NEWNUMBERING: - # RAR 2.x does not set FIRSTVOLUME, - # so check it only if NEWNUMBERING is used - if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: - raise NeedFirstVolume("Need to start from first volume") - if h.flags & RAR_MAIN_PASSWORD: - self._needs_password = True - if not self._password: - self._main = None - break - elif h.type == RAR_BLOCK_ENDARC: - more_vols = h.flags & RAR_ENDARC_NEXT_VOLUME - endarc = 1 - elif h.type == RAR_BLOCK_FILE: - # RAR 2.x does not write RAR_BLOCK_ENDARC - if h.flags & RAR_FILE_SPLIT_AFTER: - more_vols = 1 - # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME - if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: - raise NeedFirstVolume("Need to start from first volume") - - # store it - self._process_entry(h) - - # go to next header - if h.add_size > 0: - fd.seek(h.file_offset + h.add_size, 0) - - # AES encrypted headers - _last_aes_key = (None, None, None) # (salt, key, iv) - def _decrypt_header(self, fd): - if not _have_crypto: - raise NoCrypto('Cannot parse encrypted headers - no crypto') - salt = fd.read(8) - if self._last_aes_key[0] == salt: - key, iv = self._last_aes_key[1:] - else: - key, iv = rar3_s2k(self._password, salt) - self._last_aes_key = (salt, key, iv) - return HeaderDecrypt(fd, key, iv) - - # read single header - def _parse_header(self, fd): - try: - # handle encrypted headers - if self._main and self._main.flags & RAR_MAIN_PASSWORD: - if not self._password: - return - fd = self._decrypt_header(fd) - - # now read actual header - return self._parse_block_header(fd) - except struct.error: - self._set_error('Broken header in RAR file') - return None - - # common header - def _parse_block_header(self, fd): - h = RarInfo() - h.header_offset = fd.tell() - h.comment = None - - # read and parse base header - buf = fd.read(S_BLK_HDR.size) - if not buf: - return None - t = S_BLK_HDR.unpack_from(buf) - h.header_crc, h.type, h.flags, h.header_size = t - h.header_base = S_BLK_HDR.size - pos = S_BLK_HDR.size - - # read full header - if h.header_size > S_BLK_HDR.size: - h.header_data = buf + fd.read(h.header_size - S_BLK_HDR.size) - else: - h.header_data = buf - h.file_offset = fd.tell() - - # unexpected EOF? - if len(h.header_data) != h.header_size: - self._set_error('Unexpected EOF when reading header') - return None - - # block has data assiciated with it? - if h.flags & RAR_LONG_BLOCK: - h.add_size = S_LONG.unpack_from(h.header_data, pos)[0] - else: - h.add_size = 0 - - # parse interesting ones, decide header boundaries for crc - if h.type == RAR_BLOCK_MARK: - return h - elif h.type == RAR_BLOCK_MAIN: - h.header_base += 6 - if h.flags & RAR_MAIN_ENCRYPTVER: - h.header_base += 1 - if h.flags & RAR_MAIN_COMMENT: - self._parse_subblocks(h, h.header_base) - self.comment = h.comment - elif h.type == RAR_BLOCK_FILE: - self._parse_file_header(h, pos) - elif h.type == RAR_BLOCK_SUB: - self._parse_file_header(h, pos) - h.header_base = h.header_size - elif h.type == RAR_BLOCK_OLD_AUTH: - h.header_base += 8 - elif h.type == RAR_BLOCK_OLD_EXTRA: - h.header_base += 7 - else: - h.header_base = h.header_size - - # check crc - if h.type == RAR_BLOCK_OLD_SUB: - crcdat = h.header_data[2:] + fd.read(h.add_size) - else: - crcdat = h.header_data[2:h.header_base] - - calc_crc = crc32(crcdat) & 0xFFFF - - # return good header - if h.header_crc == calc_crc: - return h - - # header parsing failed. - self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', - h.type, h.header_crc, calc_crc, len(crcdat)) - - # instead panicing, send eof - return None - - # read file-specific header - def _parse_file_header(self, h, pos): - fld = S_FILE_HDR.unpack_from(h.header_data, pos) - h.compress_size = fld[0] - h.file_size = fld[1] - h.host_os = fld[2] - h.CRC = fld[3] - h.date_time = parse_dos_time(fld[4]) - h.extract_version = fld[5] - h.compress_type = fld[6] - h.name_size = fld[7] - h.mode = fld[8] - pos += S_FILE_HDR.size - - if h.flags & RAR_FILE_LARGE: - h1 = S_LONG.unpack_from(h.header_data, pos)[0] - h2 = S_LONG.unpack_from(h.header_data, pos + 4)[0] - h.compress_size |= h1 << 32 - h.file_size |= h2 << 32 - pos += 8 - h.add_size = h.compress_size - - name = h.header_data[pos : pos + h.name_size ] - pos += h.name_size - if h.flags & RAR_FILE_UNICODE: - nul = name.find(ZERO) - h.orig_filename = name[:nul] - u = UnicodeFilename(h.orig_filename, name[nul + 1 : ]) - h.filename = u.decode() - - # if parsing failed fall back to simple name - if u.failed: - h.filename = self._decode(h.orig_filename) - else: - h.orig_filename = name - h.filename = self._decode(name) - - # change separator, if requested - if PATH_SEP != '\\': - h.filename = h.filename.replace('\\', PATH_SEP) - - if h.flags & RAR_FILE_SALT: - h.salt = h.header_data[pos : pos + 8] - pos += 8 - else: - h.salt = None - - # optional extended time stamps - if h.flags & RAR_FILE_EXTTIME: - pos = self._parse_ext_time(h, pos) - else: - h.mtime = h.atime = h.ctime = h.arctime = None - - # base header end - h.header_base = pos - - if h.flags & RAR_FILE_COMMENT: - self._parse_subblocks(h, pos) - - # convert timestamps - if USE_DATETIME: - h.date_time = to_datetime(h.date_time) - h.mtime = to_datetime(h.mtime) - h.atime = to_datetime(h.atime) - h.ctime = to_datetime(h.ctime) - h.arctime = to_datetime(h.arctime) - - # .mtime is .date_time with more precision - if h.mtime: - if USE_DATETIME: - h.date_time = h.mtime - else: - # keep seconds int - h.date_time = h.mtime[:5] + (int(h.mtime[5]),) - - return pos - - # find old-style comment subblock - def _parse_subblocks(self, h, pos): - hdata = h.header_data - while pos < len(hdata): - # ordinary block header - t = S_BLK_HDR.unpack_from(hdata, pos) - scrc, stype, sflags, slen = t - pos_next = pos + slen - pos += S_BLK_HDR.size - - # corrupt header - if pos_next < pos: - break - - # followed by block-specific header - if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: - declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) - pos += S_COMMENT_HDR.size - data = hdata[pos : pos_next] - cmt = rar_decompress(ver, meth, data, declen, sflags, - crc, self._password) - if not self._crc_check: - h.comment = self._decode_comment(cmt) - elif crc32(cmt) & 0xFFFF == crc: - h.comment = self._decode_comment(cmt) - - pos = pos_next - - def _parse_ext_time(self, h, pos): - data = h.header_data - - # flags and rest of data can be missing - flags = 0 - if pos + 2 <= len(data): - flags = S_SHORT.unpack_from(data, pos)[0] - pos += 2 - - h.mtime, pos = self._parse_xtime(flags >> 3*4, data, pos, h.date_time) - h.ctime, pos = self._parse_xtime(flags >> 2*4, data, pos) - h.atime, pos = self._parse_xtime(flags >> 1*4, data, pos) - h.arctime, pos = self._parse_xtime(flags >> 0*4, data, pos) - return pos - - def _parse_xtime(self, flag, data, pos, dostime = None): - unit = 10000000.0 # 100 ns units - if flag & 8: - if not dostime: - t = S_LONG.unpack_from(data, pos)[0] - dostime = parse_dos_time(t) - pos += 4 - rem = 0 - cnt = flag & 3 - for i in range(cnt): - b = S_BYTE.unpack_from(data, pos)[0] - rem = (b << 16) | (rem >> 8) - pos += 1 - sec = dostime[5] + rem / unit - if flag & 4: - sec += 1 - dostime = dostime[:5] + (sec,) - return dostime, pos - - # given current vol name, construct next one - def _next_volname(self, volfile): - if is_filelike(volfile): - raise IOError("Working on single FD") - if self._main.flags & RAR_MAIN_NEWNUMBERING: - return self._next_newvol(volfile) - return self._next_oldvol(volfile) - - # new-style next volume - def _next_newvol(self, volfile): - i = len(volfile) - 1 - while i >= 0: - if volfile[i] >= '0' and volfile[i] <= '9': - return self._inc_volname(volfile, i) - i -= 1 - raise BadRarName("Cannot construct volume name: "+volfile) - - # old-style next volume - def _next_oldvol(self, volfile): - # rar -> r00 - if volfile[-4:].lower() == '.rar': - return volfile[:-2] + '00' - return self._inc_volname(volfile, len(volfile) - 1) - - # increase digits with carry, otherwise just increment char - def _inc_volname(self, volfile, i): - fn = list(volfile) - while i >= 0: - if fn[i] != '9': - fn[i] = chr(ord(fn[i]) + 1) - break - fn[i] = '0' - i -= 1 - return ''.join(fn) - - def _open_clear(self, inf): - return DirectReader(self, inf) - - # put file compressed data into temporary .rar archive, and run - # unrar on that, thus avoiding unrar going over whole archive - def _open_hack(self, inf, psw = None): - BSIZE = 32*1024 - - size = inf.compress_size + inf.header_size - rf = XFile(inf.volume_file, 0) - rf.seek(inf.header_offset) - - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - - try: - # create main header: crc, type, flags, size, res1, res2 - mh = S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2+4) - tmpf.write(RAR_ID + mh) - while size > 0: - if size > BSIZE: - buf = rf.read(BSIZE) - else: - buf = rf.read(size) - if not buf: - raise BadRarFile('read failed: ' + inf.filename) - tmpf.write(buf) - size -= len(buf) - tmpf.close() - rf.close() - except: - rf.close() - tmpf.close() - os.unlink(tmpname) - raise - - return self._open_unrar(tmpname, inf, psw, tmpname) - - def _read_comment_v3(self, inf, psw=None): - - # read data - rf = XFile(inf.volume_file) - rf.seek(inf.file_offset) - data = rf.read(inf.compress_size) - rf.close() - - # decompress - cmt = rar_decompress(inf.extract_version, inf.compress_type, data, - inf.file_size, inf.flags, inf.CRC, psw, inf.salt) - - # check crc - if self._crc_check: - crc = crc32(cmt) - if crc < 0: - crc += (long(1) << 32) - if crc != inf.CRC: - return None - - return self._decode_comment(cmt) - - # extract using unrar - def _open_unrar(self, rarfile, inf, psw = None, tmpfile = None): - if is_filelike(rarfile): - raise ValueError("Cannot use unrar directly on memory buffer") - cmd = [UNRAR_TOOL] + list(OPEN_ARGS) - add_password_arg(cmd, psw) - cmd.append(rarfile) - - # not giving filename avoids encoding related problems - if not tmpfile: - fn = inf.filename - if PATH_SEP != os.sep: - fn = fn.replace(PATH_SEP, os.sep) - cmd.append(fn) - - # read from unrar pipe - return PipeReader(self, inf, cmd, tmpfile) - - def _decode(self, val): - for c in TRY_ENCODINGS: - try: - return val.decode(c) - except UnicodeError: - pass - return val.decode(self._charset, 'replace') - - def _decode_comment(self, val): - if UNICODE_COMMENTS: - return self._decode(val) - return val - - # call unrar to extract a file - def _extract(self, fnlist, path=None, psw=None): - cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) - - # pasoword - psw = psw or self._password - add_password_arg(cmd, psw) - - # rar file - cmd.append(self.rarfile) - - # file list - for fn in fnlist: - if os.sep != PATH_SEP: - fn = fn.replace(PATH_SEP, os.sep) - cmd.append(fn) - - # destination path - if path is not None: - cmd.append(path + os.sep) - - # call - p = custom_popen(cmd) - output = p.communicate()[0] - check_returncode(p, output) - -## -## Utility classes -## - -class UnicodeFilename: - """Handle unicode filename decompression""" - - def __init__(self, name, encdata): - self.std_name = bytearray(name) - self.encdata = bytearray(encdata) - self.pos = self.encpos = 0 - self.buf = bytearray() - self.failed = 0 - - def enc_byte(self): - try: - c = self.encdata[self.encpos] - self.encpos += 1 - return c - except IndexError: - self.failed = 1 - return 0 - - def std_byte(self): - try: - return self.std_name[self.pos] - except IndexError: - self.failed = 1 - return ord('?') - - def put(self, lo, hi): - self.buf.append(lo) - self.buf.append(hi) - self.pos += 1 - - def decode(self): - hi = self.enc_byte() - flagbits = 0 - while self.encpos < len(self.encdata): - if flagbits == 0: - flags = self.enc_byte() - flagbits = 8 - flagbits -= 2 - t = (flags >> flagbits) & 3 - if t == 0: - self.put(self.enc_byte(), 0) - elif t == 1: - self.put(self.enc_byte(), hi) - elif t == 2: - self.put(self.enc_byte(), self.enc_byte()) - else: - n = self.enc_byte() - if n & 0x80: - c = self.enc_byte() - for i in range((n & 0x7f) + 2): - lo = (self.std_byte() + c) & 0xFF - self.put(lo, hi) - else: - for i in range(n + 2): - self.put(self.std_byte(), 0) - return self.buf.decode("utf-16le", "replace") - - -class RarExtFile(RawIOBase): - """Base class for file-like object that :meth:`RarFile.open` returns. - - Provides public methods and common crc checking. - - Behaviour: - - no short reads - .read() and .readinfo() read as much as requested. - - no internal buffer, use io.BufferedReader for that. - - If :mod:`io` module is available (Python 2.6+, 3.x), then this calls - will inherit from :class:`io.RawIOBase` class. This makes line-based - access available: :meth:`RarExtFile.readline` and ``for ln in f``. - """ - - #: Filename of the archive entry - name = None - - def __init__(self, rf, inf): - RawIOBase.__init__(self) - - # standard io.* properties - self.name = inf.filename - self.mode = 'rb' - - self.rf = rf - self.inf = inf - self.crc_check = rf._crc_check - self.fd = None - self.CRC = 0 - self.remain = 0 - self.returncode = 0 - - self._open() - - def _open(self): - if self.fd: - self.fd.close() - self.fd = None - self.CRC = 0 - self.remain = self.inf.file_size - - def read(self, cnt = None): - """Read all or specified amount of data from archive entry.""" - - # sanitize cnt - if cnt is None or cnt < 0: - cnt = self.remain - elif cnt > self.remain: - cnt = self.remain - if cnt == 0: - return EMPTY - - # actual read - data = self._read(cnt) - if data: - self.CRC = crc32(data, self.CRC) - self.remain -= len(data) - if len(data) != cnt: - raise BadRarFile("Failed the read enough data") - - # done? - if not data or self.remain == 0: - #self.close() - self._check() - return data - - def _check(self): - """Check final CRC.""" - if not self.crc_check: - return - if self.returncode: - check_returncode(self, '') - if self.remain != 0: - raise BadRarFile("Failed the read enough data") - crc = self.CRC - if crc < 0: - crc += (long(1) << 32) - if crc != self.inf.CRC: - raise BadRarFile("Corrupt file - CRC check failed: " + self.inf.filename) - - def _read(self, cnt): - """Actual read that gets sanitized cnt.""" - - def close(self): - """Close open resources.""" - - RawIOBase.close(self) - - if self.fd: - self.fd.close() - self.fd = None - - def __del__(self): - """Hook delete to make sure tempfile is removed.""" - self.close() - - def readinto(self, buf): - """Zero-copy read directly into buffer. - - Returns bytes read. - """ - - data = self.read(len(buf)) - n = len(data) - try: - buf[:n] = data - except TypeError: - import array - if not isinstance(buf, array.array): - raise - buf[:n] = array.array(buf.typecode, data) - return n - - def tell(self): - """Return current reading position in uncompressed data.""" - return self.inf.file_size - self.remain - - def seek(self, ofs, whence = 0): - """Seek in data. - - On uncompressed files, the seeking works by actual - seeks so it's fast. On compresses files its slow - - forward seeking happends by reading ahead, - backwards by re-opening and decompressing from the start. - """ - - # disable crc check when seeking - self.crc_check = 0 - - fsize = self.inf.file_size - cur_ofs = self.tell() - - if whence == 0: # seek from beginning of file - new_ofs = ofs - elif whence == 1: # seek from current position - new_ofs = cur_ofs + ofs - elif whence == 2: # seek from end of file - new_ofs = fsize + ofs - else: - raise ValueError('Invalid value for whence') - - # sanity check - if new_ofs < 0: - new_ofs = 0 - elif new_ofs > fsize: - new_ofs = fsize - - # do the actual seek - if new_ofs >= cur_ofs: - self._skip(new_ofs - cur_ofs) - else: - # process old data ? - #self._skip(fsize - cur_ofs) - # reopen and seek - self._open() - self._skip(new_ofs) - return self.tell() - - def _skip(self, cnt): - """Read and discard data""" - while cnt > 0: - if cnt > 8192: - buf = self.read(8192) - else: - buf = self.read(cnt) - if not buf: - break - cnt -= len(buf) - - def readable(self): - """Returns True""" - return True - - def writable(self): - """Returns False. - - Writing is not supported.""" - return False - - def seekable(self): - """Returns True. - - Seeking is supported, although it's slow on compressed files. - """ - return True - - def readall(self): - """Read all remaining data""" - # avoid RawIOBase default impl - return self.read() - - -class PipeReader(RarExtFile): - """Read data from pipe, handle tempfile cleanup.""" - - def __init__(self, rf, inf, cmd, tempfile=None): - self.cmd = cmd - self.proc = None - self.tempfile = tempfile - RarExtFile.__init__(self, rf, inf) - - def _close_proc(self): - if not self.proc: - return - if self.proc.stdout: - self.proc.stdout.close() - if self.proc.stdin: - self.proc.stdin.close() - if self.proc.stderr: - self.proc.stderr.close() - self.proc.wait() - self.returncode = self.proc.returncode - self.proc = None - - def _open(self): - RarExtFile._open(self) - - # stop old process - self._close_proc() - - # launch new process - self.returncode = 0 - self.proc = custom_popen(self.cmd) - self.fd = self.proc.stdout - - # avoid situation where unrar waits on stdin - if self.proc.stdin: - self.proc.stdin.close() - - def _read(self, cnt): - """Read from pipe.""" - - # normal read is usually enough - data = self.fd.read(cnt) - if len(data) == cnt or not data: - return data - - # short read, try looping - buf = [data] - cnt -= len(data) - while cnt > 0: - data = self.fd.read(cnt) - if not data: - break - cnt -= len(data) - buf.append(data) - return EMPTY.join(buf) - - def close(self): - """Close open resources.""" - - self._close_proc() - RarExtFile.close(self) - - if self.tempfile: - try: - os.unlink(self.tempfile) - except OSError: - pass - self.tempfile = None - - if have_memoryview: - def readinto(self, buf): - """Zero-copy read directly into buffer.""" - cnt = len(buf) - if cnt > self.remain: - cnt = self.remain - vbuf = memoryview(buf) - res = got = 0 - while got < cnt: - res = self.fd.readinto(vbuf[got : cnt]) - if not res: - break - if self.crc_check: - self.CRC = crc32(vbuf[got : got + res], self.CRC) - self.remain -= res - got += res - return got - - -class DirectReader(RarExtFile): - """Read uncompressed data directly from archive.""" - - def _open(self): - RarExtFile._open(self) - - self.volfile = self.inf.volume_file - self.fd = XFile(self.volfile, 0) - self.fd.seek(self.inf.header_offset, 0) - self.cur = self.rf._parse_header(self.fd) - self.cur_avail = self.cur.add_size - - def _skip(self, cnt): - """RAR Seek, skipping through rar files to get to correct position - """ - - while cnt > 0: - # next vol needed? - if self.cur_avail == 0: - if not self._open_next(): - break - - # fd is in read pos, do the read - if cnt > self.cur_avail: - cnt -= self.cur_avail - self.remain -= self.cur_avail - self.cur_avail = 0 - else: - self.fd.seek(cnt, 1) - self.cur_avail -= cnt - self.remain -= cnt - cnt = 0 - - def _read(self, cnt): - """Read from potentially multi-volume archive.""" - - buf = [] - while cnt > 0: - # next vol needed? - if self.cur_avail == 0: - if not self._open_next(): - break - - # fd is in read pos, do the read - if cnt > self.cur_avail: - data = self.fd.read(self.cur_avail) - else: - data = self.fd.read(cnt) - if not data: - break - - # got some data - cnt -= len(data) - self.cur_avail -= len(data) - buf.append(data) - - if len(buf) == 1: - return buf[0] - return EMPTY.join(buf) - - def _open_next(self): - """Proceed to next volume.""" - - # is the file split over archives? - if (self.cur.flags & RAR_FILE_SPLIT_AFTER) == 0: - return False - - if self.fd: - self.fd.close() - self.fd = None - - # open next part - self.volfile = self.rf._next_volname(self.volfile) - fd = open(self.volfile, "rb", 0) - self.fd = fd - - # loop until first file header - while 1: - cur = self.rf._parse_header(fd) - if not cur: - raise BadRarFile("Unexpected EOF") - if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): - if cur.add_size: - fd.seek(cur.add_size, 1) - continue - if cur.orig_filename != self.inf.orig_filename: - raise BadRarFile("Did not found file entry") - self.cur = cur - self.cur_avail = cur.add_size - return True - - if have_memoryview: - def readinto(self, buf): - """Zero-copy read directly into buffer.""" - got = 0 - vbuf = memoryview(buf) - while got < len(buf): - # next vol needed? - if self.cur_avail == 0: - if not self._open_next(): - break - - # lenght for next read - cnt = len(buf) - got - if cnt > self.cur_avail: - cnt = self.cur_avail - - # read into temp view - res = self.fd.readinto(vbuf[got : got + cnt]) - if not res: - break - if self.crc_check: - self.CRC = crc32(vbuf[got : got + res], self.CRC) - self.cur_avail -= res - self.remain -= res - got += res - return got - - -class HeaderDecrypt: - """File-like object that decrypts from another file""" - def __init__(self, f, key, iv): - self.f = f - self.ciph = AES.new(key, AES.MODE_CBC, iv) - self.buf = EMPTY - - def tell(self): - return self.f.tell() - - def read(self, cnt=None): - if cnt > 8*1024: - raise BadRarFile('Bad count to header decrypt - wrong password?') - - # consume old data - if cnt <= len(self.buf): - res = self.buf[:cnt] - self.buf = self.buf[cnt:] - return res - res = self.buf - self.buf = EMPTY - cnt -= len(res) - - # decrypt new data - BLK = self.ciph.block_size - while cnt > 0: - enc = self.f.read(BLK) - if len(enc) < BLK: - break - dec = self.ciph.decrypt(enc) - if cnt >= len(dec): - res += dec - cnt -= len(dec) - else: - res += dec[:cnt] - self.buf = dec[cnt:] - cnt = 0 - - return res - -# handle (filename|filelike) object -class XFile(object): - __slots__ = ('_fd', '_need_close') - def __init__(self, xfile, bufsize = 1024): - if is_filelike(xfile): - self._need_close = False - self._fd = xfile - self._fd.seek(0) - else: - self._need_close = True - self._fd = open(xfile, 'rb', bufsize) - def read(self, n=None): - return self._fd.read(n) - def tell(self): - return self._fd.tell() - def seek(self, ofs, whence=0): - return self._fd.seek(ofs, whence) - def readinto(self, dst): - return self._fd.readinto(dst) - def close(self): - if self._need_close: - self._fd.close() - def __enter__(self): - return self - def __exit__(self, typ, val, tb): - self.close() - -## -## Utility functions -## - -def is_filelike(obj): - if isinstance(obj, str) or isinstance(obj, unicode): - return False - res = True - for a in ('read', 'tell', 'seek'): - res = res and hasattr(obj, a) - if not res: - raise ValueError("Invalid object passed as file") - return True - -def rar3_s2k(psw, salt): - """String-to-key hash for RAR3.""" - - seed = psw.encode('utf-16le') + salt - iv = EMPTY - h = sha1() - for i in range(16): - for j in range(0x4000): - cnt = S_LONG.pack(i*0x4000 + j) - h.update(seed + cnt[:3]) - if j == 0: - iv += h.digest()[19:20] - key_be = h.digest()[:16] - key_le = pack("LLLL", key_be)) - return key_le, iv - -def rar_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): - """Decompress blob of compressed data. - - Used for data with non-standard header - eg. comments. - """ - - # already uncompressed? - if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: - return data - - # take only necessary flags - flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) - flags |= RAR_LONG_BLOCK - - # file header - fname = bytes('data', 'ascii') - date = 0 - mode = 0x20 - fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, - date, vers, meth, len(fname), mode) - fhdr += fname - if flags & RAR_FILE_SALT: - if not salt: - return EMPTY - fhdr += salt - - # full header - hlen = S_BLK_HDR.size + len(fhdr) - hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr - hcrc = crc32(hdr[2:]) & 0xFFFF - hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr - - # archive main header - mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2+4) - - # decompress via temp rar - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - try: - tmpf.write(RAR_ID + mh + hdr + data) - tmpf.close() - - cmd = [UNRAR_TOOL] + list(OPEN_ARGS) - add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) - cmd.append(tmpname) - - p = custom_popen(cmd) - return p.communicate()[0] - finally: - tmpf.close() - os.unlink(tmpname) - -def to_datetime(t): - """Convert 6-part time tuple into datetime object.""" - - if t is None: - return None - - # extract values - year, mon, day, h, m, xs = t - s = int(xs) - us = int(1000000 * (xs - s)) - - # assume the values are valid - try: - return datetime(year, mon, day, h, m, s, us) - except ValueError: - pass - - # sanitize invalid values - MDAY = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) - if mon < 1: mon = 1 - if mon > 12: mon = 12 - if day < 1: day = 1 - if day > MDAY[mon]: day = MDAY[mon] - if h > 23: h = 23 - if m > 59: m = 59 - if s > 59: s = 59 - if mon == 2 and day == 29: - try: - return datetime(year, mon, day, h, m, s, us) - except ValueError: - day = 28 - return datetime(year, mon, day, h, m, s, us) - -def parse_dos_time(stamp): - """Parse standard 32-bit DOS timestamp.""" - - sec = stamp & 0x1F; stamp = stamp >> 5 - min = stamp & 0x3F; stamp = stamp >> 6 - hr = stamp & 0x1F; stamp = stamp >> 5 - day = stamp & 0x1F; stamp = stamp >> 5 - mon = stamp & 0x0F; stamp = stamp >> 4 - yr = (stamp & 0x7F) + 1980 - return (yr, mon, day, hr, min, sec * 2) - -def custom_popen(cmd): - """Disconnect cmd from parent fds, read only from stdout.""" - - # needed for py2exe - creationflags = 0 - if sys.platform == 'win32': - creationflags = 0x08000000 # CREATE_NO_WINDOW - - # run command - try: - p = Popen(cmd, bufsize = 0, - stdout = PIPE, stdin = PIPE, stderr = STDOUT, - creationflags = creationflags) - except OSError: - ex = sys.exc_info()[1] - if ex.errno == errno.ENOENT: - raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) - raise - return p - -def custom_check(cmd, ignore_retcode=False): - """Run command, collect output, raise error if needed.""" - p = custom_popen(cmd) - out, err = p.communicate() - if p.returncode and not ignore_retcode: - raise RarExecError("Check-run failed") - return out - -def add_password_arg(cmd, psw, required=False): - """Append password switch to commandline.""" - if UNRAR_TOOL == ALT_TOOL: - return - if psw is not None: - cmd.append('-p' + psw) - else: - cmd.append('-p-') - -def check_returncode(p, out): - """Raise exception according to unrar exit code""" - - code = p.returncode - if code == 0: - return - if code == 9: - return - - # map return code to exception class - errmap = [None, - RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, - RarWriteError, RarOpenError, RarUserError, RarMemoryError, - RarCreateError, RarNoFilesError] # codes from rar.txt - if UNRAR_TOOL == ALT_TOOL: - errmap = [None] - if code > 0 and code < len(errmap): - exc = errmap[code] - elif code == 255: - exc = RarUserBreak - elif code < 0: - exc = RarSignalExit - else: - exc = RarUnknownError - - # format message - if out: - msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) - else: - msg = "%s [%d]" % (exc.__doc__, p.returncode) - - raise exc(msg) - -# -# Check if unrar works -# - -try: - # does UNRAR_TOOL work? - custom_check([UNRAR_TOOL], True) -except RarCannotExec: - try: - # does ALT_TOOL work? - custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) - # replace config - UNRAR_TOOL = ALT_TOOL - OPEN_ARGS = ALT_OPEN_ARGS - EXTRACT_ARGS = ALT_EXTRACT_ARGS - TEST_ARGS = ALT_TEST_ARGS - except RarCannotExec: - # no usable tool, only uncompressed archives work - pass - diff --git a/kindlecomicconverter/shared.py b/kindlecomicconverter/shared.py index 56f07d62..20308acd 100644 --- a/kindlecomicconverter/shared.py +++ b/kindlecomicconverter/shared.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the @@ -22,9 +22,6 @@ from hashlib import md5 from html.parser import HTMLParser from distutils.version import StrictVersion -from shutil import rmtree, copy -from tempfile import mkdtemp -from zipfile import ZipFile, ZIP_DEFLATED from re import split from traceback import format_tb @@ -50,7 +47,7 @@ def error(self, message): def getImageFileName(imgfile): name, ext = os.path.splitext(imgfile) ext = ext.lower() - if name.startswith('.') or (ext != '.png' and ext != '.jpg' and ext != '.jpeg' and ext != '.gif'): + if name.startswith('.') or ext not in ['.png', '.jpg', '.jpeg', '.gif', '.webp']: return None return [name, ext] @@ -86,38 +83,19 @@ def md5Checksum(fpath): return m.hexdigest() -def check7ZFile(fpath): - with open(fpath, 'rb') as fh: - header = fh.read(6) - return header == b"7z\xbc\xaf'\x1c" - - -def removeFromZIP(zipfname, *filenames): - tempdir = mkdtemp('', 'KCC-') - try: - tempname = os.path.join(tempdir, 'KCC.zip') - with ZipFile(zipfname, 'r') as zipread: - with ZipFile(tempname, 'w', compression=ZIP_DEFLATED) as zipwrite: - for item in zipread.infolist(): - if item.filename not in filenames: - zipwrite.writestr(item, zipread.read(item.filename)) - copy(tempname, zipfname) - finally: - rmtree(tempdir, True) - - def sanitizeTrace(traceback): return ''.join(format_tb(traceback))\ - .replace('C:/projects/kcc/', '') \ - .replace('c:/projects/kcc/', '') \ + .replace('C:/projects/kcc/', '')\ + .replace('c:/projects/kcc/', '')\ .replace('C:/python36-x64/', '')\ .replace('c:/python36-x64/', '')\ - .replace('C:\\projects\\kcc\\', '') \ - .replace('c:\\projects\\kcc\\', '') \ + .replace('C:\\projects\\kcc\\', '')\ + .replace('c:\\projects\\kcc\\', '')\ .replace('C:\\python36-x64\\', '')\ .replace('c:\\python36-x64\\', '') +# noinspection PyUnresolvedReferences def dependencyCheck(level): missing = [] if level > 2: @@ -145,11 +123,11 @@ def dependencyCheck(level): except ImportError: missing.append('python-slugify 1.2.1+') try: - from PIL import PILLOW_VERSION as pillowVersion - if StrictVersion('4.0.0') > StrictVersion(pillowVersion): - missing.append('Pillow 4.0.0+') + from PIL import __version__ as pillowVersion + if StrictVersion('5.2.0') > StrictVersion(pillowVersion): + missing.append('Pillow 5.2.0+') except ImportError: - missing.append('Pillow 4.0.0+') + missing.append('Pillow 5.2.0+') if len(missing) > 0: print('ERROR: ' + ', '.join(missing) + ' is not installed!') exit(1) diff --git a/kindlecomicconverter/startup.py b/kindlecomicconverter/startup.py index deb6313a..c92579d3 100644 --- a/kindlecomicconverter/startup.py +++ b/kindlecomicconverter/startup.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2012-2014 Ciro Mattia Gonano -# Copyright (c) 2013-2018 Pawel Jastrzebski +# Copyright (c) 2013-2019 Pawel Jastrzebski # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the @@ -30,15 +30,15 @@ def start(): os.environ['QT_AUTO_SCREEN_SCALE_FACTOR'] = "1" KCCAplication = KCC_gui.QApplicationMessaging(sys.argv) if KCCAplication.isRunning(): - if len(sys.argv) > 1: - KCCAplication.sendMessage(sys.argv[1]) + for i in range(1, len(sys.argv)): + KCCAplication.sendMessage(sys.argv[i]) else: KCCAplication.sendMessage('ARISE') else: KCCWindow = KCC_gui.QMainWindowKCC() KCCUI = KCC_gui.KCCGUI(KCCAplication, KCCWindow) - if len(sys.argv) > 1: - KCCUI.handleMessage(sys.argv[1]) + for i in range(1, len(sys.argv)): + KCCUI.handleMessage(sys.argv[i]) sys.exit(KCCAplication.exec_()) diff --git a/other/linux/kindlecomicconverter b/other/linux/kindlecomicconverter deleted file mode 100644 index c9baf045..00000000 --- a/other/linux/kindlecomicconverter +++ /dev/null @@ -1,4 +0,0 @@ -kindlecomicconverter: binary-without-manpage usr/bin/kcc -kindlecomicconverter: wrong-name-for-changelog-of-native-package usr/share/doc/kindlecomicconverter/changelog.Debian.gz -kindlecomicconverter: file-missing-in-md5sums usr/share/doc/kindlecomicconverter/changelog.Debian.gz -kindlecomicconverter: hardening-no-relro usr/bin/kcc diff --git a/other/linux/kindlecomicconverter.desktop b/other/linux/kindlecomicconverter.desktop deleted file mode 100644 index 306468b5..00000000 --- a/other/linux/kindlecomicconverter.desktop +++ /dev/null @@ -1,11 +0,0 @@ -[Desktop Entry] -Type=Application -Version=1.0 -Name=Kindle Comic Converter -GenericName=Kindle Comic Converter -Comment=Comic and Manga converter for e-book readers -Icon=/usr/share/kindlecomicconverter/comic2ebook.png -Exec=/usr/bin/kcc %f -Terminal=false -Categories=Graphics; -MimeType=application/zip;application/x-rar;application/x-7z-compressed; diff --git a/other/linux/sentry.py.enc b/other/linux/sentry.py.enc deleted file mode 100644 index c38a68ce..00000000 Binary files a/other/linux/sentry.py.enc and /dev/null differ diff --git a/other/osx/7z b/other/osx/7z new file mode 100644 index 00000000..7e7b90bc Binary files /dev/null and b/other/osx/7z differ diff --git a/other/osx/7z.so b/other/osx/7z.so new file mode 100644 index 00000000..80eb496f Binary files /dev/null and b/other/osx/7z.so differ diff --git a/other/osx/7za b/other/osx/7za deleted file mode 100755 index f92fa212..00000000 Binary files a/other/osx/7za and /dev/null differ diff --git a/other/osx/Info.plist b/other/osx/Info.plist index 4a9fa483..2cf43f18 100644 --- a/other/osx/Info.plist +++ b/other/osx/Info.plist @@ -30,7 +30,7 @@ CFBundleExecutable MacOS/Kindle Comic Converter CFBundleGetInfoString - KindleComicConverter 5.4.5, written 2012-2018 by Ciro Mattia Gonano and Pawel Jastrzebski + KindleComicConverter 5.5.0, written 2012-2019 by Ciro Mattia Gonano and Pawel Jastrzebski CFBundleIconFile comic2ebook.icns CFBundleIdentifier @@ -42,11 +42,11 @@ CFBundlePackageType APPL CFBundleShortVersionString - 5.4.5 + 5.5.0 CFBundleSignature ???? CFBundleVersion - 5.4.5 + 5.5.0 LSEnvironment PATH @@ -55,7 +55,7 @@ LSHasLocalizedDisplayName LSMinimumSystemVersion - 10.10.0 + 10.12.0 NSAppleScriptEnabled NSHumanReadableCopyright diff --git a/other/osx/unrar b/other/osx/unrar deleted file mode 100755 index 8ca581d3..00000000 Binary files a/other/osx/unrar and /dev/null differ diff --git a/other/windows/7z.dll b/other/windows/7z.dll new file mode 100644 index 00000000..be29515b Binary files /dev/null and b/other/windows/7z.dll differ diff --git a/other/windows/7z.exe b/other/windows/7z.exe new file mode 100644 index 00000000..337d4b01 Binary files /dev/null and b/other/windows/7z.exe differ diff --git a/other/windows/7za.exe b/other/windows/7za.exe deleted file mode 100644 index fc8a0bd2..00000000 Binary files a/other/windows/7za.exe and /dev/null differ diff --git a/other/windows/Additional-LICENSE.txt b/other/windows/Additional-LICENSE.txt index 891d4532..8817fbce 100644 --- a/other/windows/Additional-LICENSE.txt +++ b/other/windows/Additional-LICENSE.txt @@ -1,56 +1,22 @@ - ****** ***** ****** UnRAR - free utility for RAR archives - ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ****** ******* ****** License for use and distribution of - ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ** ** ** ** ** ** FREEWARE version - ~~~~~~~~~~~~~~~~ - - The UnRAR utility is freeware. This means: - - 1. All copyrights to RAR and the utility UnRAR are exclusively - owned by the author - Alexander Roshal. - - 2. The UnRAR utility may be freely distributed. It is allowed - to distribute UnRAR inside of other software packages. - - 3. THE RAR ARCHIVER AND THE UnRAR UTILITY ARE DISTRIBUTED "AS IS". - NO WARRANTY OF ANY KIND IS EXPRESSED OR IMPLIED. YOU USE AT - YOUR OWN RISK. THE AUTHOR WILL NOT BE LIABLE FOR DATA LOSS, - DAMAGES, LOSS OF PROFITS OR ANY OTHER KIND OF LOSS WHILE USING - OR MISUSING THIS SOFTWARE. - - 4. Neither RAR binary code, WinRAR binary code, UnRAR source or UnRAR - binary code may be used or reverse engineered to re-create the RAR - compression algorithm, which is proprietary, without written - permission of the author. - - 5. If you don't agree with terms of the license you must remove - UnRAR files from your storage devices and cease to use the - utility. - - Thank you for your interest in RAR and UnRAR. - - - Alexander L. Roshal - 7-Zip ~~~~~ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2012 Igor Pavlov. + 7-Zip Copyright (C) 1999-2018 Igor Pavlov. - Licenses for files are: + The licenses for files are: - 1) 7z.dll: GNU LGPL + unRAR restriction - 2) All other files: GNU LGPL + 1) 7z.dll: + - The "GNU LGPL" as main license for most of the code + - The "GNU LGPL" with "unRAR license restriction" for some code + - The "BSD 3-clause License" for some code + 2) All other files: the "GNU LGPL". - The GNU LGPL + unRAR restriction means that you must follow both - GNU LGPL rules and unRAR restriction rules. + Redistributions in binary form must reproduce related license information from this file. - - Note: - You can use 7-Zip on any computer, including a computer in a commercial + Note: + You can use 7-Zip on any computer, including a computer in a commercial organization. You don't need to register or pay for 7-Zip. @@ -67,21 +33,54 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. - You can receive a copy of the GNU Lesser General Public License from + You can receive a copy of the GNU Lesser General Public License from http://www.gnu.org/ - unRAR restriction - ----------------- - The decompression engine for RAR archives was developed using source + + BSD 3-clause License + -------------------- + + The "BSD 3-clause License" is used for the code in 7z.dll that implements LZFSE data decompression. + That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, + that also uses the "BSD 3-clause License": + + ---- + Copyright (c) 2015-2016, Apple Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ---- + + + + + unRAR license restriction + ------------------------- + + The decompression engine for RAR archives was developed using source code of unRAR program. All copyrights to original unRAR code are owned by Alexander Roshal. The license for original unRAR code has the following restriction: - The unRAR sources cannot be used to re-create the RAR compression algorithm, - which is proprietary. Distribution of modified unRAR sources in separate form + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form or as a part of other software is permitted, provided that it is clearly stated in the documentation and source comments that the code may not be used to develop a RAR (WinRAR) compatible archiver. diff --git a/other/windows/UnRAR.exe b/other/windows/UnRAR.exe deleted file mode 100644 index 28840c13..00000000 Binary files a/other/windows/UnRAR.exe and /dev/null differ diff --git a/requirements.txt b/requirements.txt index 0478a952..355db433 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyQt5>=5.6.0 -Pillow>=4.0.0 +Pillow>=5.2.0 psutil>=5.0.0 -python-slugify>=1.2.1 +python-slugify>=1.2.1,<3.0.0 raven>=6.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index f7d71e0f..7bc5c8d4 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ Install as Python package: python3 setup.py install -Create EXE/APP/DEB: +Create EXE/APP: python3 setup.py build_binary """ @@ -37,35 +37,18 @@ def run(self): VERSION = __version__ if sys.platform == 'darwin': os.system('pyinstaller -y -F -i icons/comic2ebook.icns -n "Kindle Comic Converter" -w -s kcc.py') - shutil.copy('other/osx/7za', 'dist/Kindle Comic Converter.app/Contents/Resources') - shutil.copy('other/osx/unrar', 'dist/Kindle Comic Converter.app/Contents/Resources') + shutil.copy('other/osx/7z', 'dist/Kindle Comic Converter.app/Contents/Resources') + shutil.copy('other/osx/7z.so', 'dist/Kindle Comic Converter.app/Contents/Resources') shutil.copy('other/osx/Info.plist', 'dist/Kindle Comic Converter.app/Contents') shutil.copy('LICENSE.txt', 'dist/Kindle Comic Converter.app/Contents/Resources') shutil.copy('other/windows/Additional-LICENSE.txt', 'dist/Kindle Comic Converter.app/Contents/Resources') - os.chmod('dist/Kindle Comic Converter.app/Contents/Resources/unrar', 0o777) - os.chmod('dist/Kindle Comic Converter.app/Contents/Resources/7za', 0o777) + os.chmod('dist/Kindle Comic Converter.app/Contents/Resources/7z', 0o777) os.system('appdmg kcc.json dist/KindleComicConverter_osx_' + VERSION + '.dmg') exit(0) elif sys.platform == 'win32': - os.system('pyinstaller -y -F -i icons\comic2ebook.ico -n KCC -w --noupx kcc.py') + os.system('pyinstaller -y -F -i icons\\comic2ebook.ico -n KCC -w --noupx kcc.py') exit(0) else: - os.system('pyinstaller -y -F kcc.py') - os.system('mkdir -p dist/usr/bin dist/usr/share/applications dist/usr/share/doc/kindlecomicconverter ' - 'dist/usr/share/kindlecomicconverter dist/usr/share/lintian/overrides') - os.system('mv dist/kcc dist/usr/bin') - os.system('cp icons/comic2ebook.png dist/usr/share/kindlecomicconverter') - os.system('cp LICENSE.txt dist/usr/share/doc/kindlecomicconverter/copyright') - os.system('cp other/linux/kindlecomicconverter.desktop dist/usr/share/applications') - os.system('cp other/linux/kindlecomicconverter dist/usr/share/lintian/overrides') - os.chdir('dist') - os.system('fpm -f -s dir -t deb -n kindlecomicconverter -v ' + VERSION + - ' -m "Pawel Jastrzebski " --license "ISC" ' - '--description "$(printf "Comic and Manga converter for e-book ' - 'readers.\nThis app allows you to transform your PNG, JPG, GIF, ' - 'CBZ, CBR and CB7 files\ninto EPUB or MOBI format e-books.")" ' - '--url "https://kcc.iosphe.re/" --deb-priority "optional" --vendor "" ' - '--category "graphics" -d "unrar | unrar-free" -d "p7zip-full" -d "libc6" usr') exit(0) @@ -93,7 +76,7 @@ def run(self): packages=['kindlecomicconverter'], install_requires=[ 'PyQt5>=5.6.0', - 'Pillow>=4.0.0', + 'Pillow>=5.2.0', 'psutil>=5.0.0', 'python-slugify>=1.2.1', 'raven>=6.0.0',