Skip to content

Commit

Permalink
updated dependencies, modified config loading
Browse files Browse the repository at this point in the history
  • Loading branch information
9FS committed May 12, 2024
1 parent a6df0ee commit 728d503
Show file tree
Hide file tree
Showing 6 changed files with 99 additions and 88 deletions.
1 change: 1 addition & 0 deletions .dockerignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ config/
doc_templates/
log/

.env
docker-image.tar
tests/test.py
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ config/
doc_templates/
log/

.env
docker-image.tar
tests/test.py
87 changes: 49 additions & 38 deletions poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,18 @@ name = ""
package-mode = false # this is an application, not library
readme = "readme.md"
repository = "https://github.com/9-FS/2021-11-15-nHentai-to-PDF"
version = "2.2.0"
version = "2.3.0"

[tool.poetry.dependencies]
kfsconfig = "^1.0.0"
kfsconfig = "^2.0.0"
kfsfstr = "^1.0.0"
kfslog = "^1.0.0"
kfsmedia = "^2.5.1"
kfslog = "^2.0.0"
kfsmedia = "^2.5.0"
python = "^3.12.0"

[tool.poetry.group.dev.dependencies]
hypothesis = "^6.0.0"
pytest = "^7.0.0"
pytest = "^8.0.0"

[build-system]
build-backend = "poetry.core.masonry.api"
Expand Down
11 changes: 6 additions & 5 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ certifi==2024.2.2 ; python_full_version >= "3.12.0" and python_full_version < "4
charset-normalizer==3.3.2 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
colorama==0.4.6 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
idna==3.7 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsconfig==1.1.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsfstr==1.1.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfslog==1.1.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsmath==1.0.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsmedia==2.5.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsconfig==2.1.4 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsfstr==1.1.2 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfslog==2.0.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsmath==1.0.2 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
kfsmedia==2.5.2 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
olefile==0.47 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
pebble==5.0.7 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
pillow==10.3.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
requests==2.31.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
result==0.16.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
urllib3==2.2.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
77 changes: 37 additions & 40 deletions src/main.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# Copyright (c) 2024 구FS, all rights reserved. Subject to the MIT licence in `licence.md`.
import gc # garbage collector, explicitly free memory
import json
from KFSconfig import KFSconfig
from KFSfstr import KFSfstr
from KFSlog import KFSlog
Expand All @@ -11,35 +10,33 @@
from Hentai import Hentai


@KFSlog.timeit
@KFSlog.timeit()
def main(DEBUG: bool):
cleanup_success: bool=True # cleanup successful
cookies: dict[str, str] # for requests.get to bypass bot protection
COOKIES_DEFAULT: str=json.dumps({ # cookies default
"cf_clearance": "",
"csrftoken": "",
}, indent=4)
DOWNLOADME_FILEPATH: str="./config/downloadme.txt" # path to file containing hentai ID to download
headers: dict[str, str] # for requests.get to bypass bot protection
HEADERS_DEFAULT: str=json.dumps({ # headers default
"User-Agent": "",
}, indent=4)
hentai: Hentai # individual hentai
hentai_ID_list: list[int] # hentai ID to download
settings: dict[str, str] # settings
SETTINGS_DEFAULT: str=json.dumps({ # settings default
"library_path": "./hentai/", # path to download hentai to
"library_split": "0", # split library into subdirectories of maximum this many hentai, 0 to disable
}, indent=4)
cleanup_success: bool=True # cleanup successful
config: dict[str, str] # config
CONFIG_DEFAULT: dict[str, str]=\
{
"DOWNLOADME_FILEPATH": "./config/downloadme.txt", # path to file containing hentai ID to download
"LIBRARY_PATH": "./hentai/", # path to download hentai to
"LIBRARY_SPLIT": "0", # split library into subdirectories of maximum this many hentai, 0 to disable
}
env: dict[str, str] # environment variables
ENV_DEFAULT: dict[str, str]=\
{
"CF_CLEARANCE": "", # for requests.get to bypass bot protection
"CSRFTOKEN": "",
"USER-AGENT": "",
}
hentai: Hentai # individual hentai
hentai_ID_list: list[int] # hentai ID to download


try:
cookies =json.loads(KFSconfig.load_config("./config/cookies.json", COOKIES_DEFAULT)) # load cookies to bypass bot protection
headers =json.loads(KFSconfig.load_config("./config/headers.json", HEADERS_DEFAULT)) # load headers to bypass bot protection
settings=json.loads(KFSconfig.load_config("./config/settings.json", SETTINGS_DEFAULT)) # load settings
except FileNotFoundError:
config=KFSconfig.load_config(env=False, config_filepaths=["./config/config.json"], config_default=CONFIG_DEFAULT) # load configuration
env =KFSconfig.load_config( config_filepaths=["./.env"], config_default=ENV_DEFAULT) # load environment variables
except ValueError:
return
hentai_ID_list=get_hentai_ID_list(DOWNLOADME_FILEPATH) # get desired hentai ID
hentai_ID_list=get_hentai_ID_list(config["DOWNLOADME_FILEPATH"]) # get desired hentai ID


for i, hentai_ID in enumerate(hentai_ID_list): # work through all desired hentai
Expand All @@ -50,35 +47,35 @@ def main(DEBUG: bool):
Hentai.save_galleries()

try:
hentai=Hentai(hentai_ID, cookies, headers) # create hentai object
except ValueError: # if hentai does not exist:
continue # skip to next hentai
hentai=Hentai(hentai_ID, {"cf_clearance": env["CF_CLEARANCE"], "csrftoken": env["CSRFTOKEN"]}, {"User-Agent": env["USER-AGENT"]}) # create hentai object
except ValueError: # if hentai does not exist:
continue # skip to next hentai
else:
logging.info(hentai)

try:
_=hentai.download(settings["library_path"], int(settings["library_split"])) # download hentai
except FileExistsError: # if hentai already exists:
continue # skip to next hentai
_=hentai.download(config["LIBRARY_PATH"], int(config["LIBRARY_SPLIT"])) # download hentai
except FileExistsError: # if hentai already exists:
continue # skip to next hentai
except KFSmedia.DownloadError:
with open("./log/FAILURES.txt", "at") as fails_file: # append in failure file
with open("./log/FAILURES.txt", "at") as fails_file: # append in failure file
fails_file.write(f"{hentai.ID}\n")
continue # skip to next hentai
continue # skip to next hentai
del _
gc.collect() # explicitly free memory, otherwise PDF may clutter memory
gc.collect() # explicitly free memory, otherwise PDF may clutter memory
logging.info("--------------------------------------------------")


Hentai.save_galleries() # save all galleries to file

logging.info("Deleting leftover image directories...")
for hentai_ID in hentai_ID_list: # attempt final cleanup
if os.path.isdir(os.path.join(settings["library_path"], str(hentai_ID))) and len(os.listdir(os.path.join(settings["library_path"], str(hentai_ID))))==0: # if cache folder still exists and is empty:
for hentai_ID in hentai_ID_list: # attempt final cleanup
if os.path.isdir(os.path.join(config["LIBRARY_PATH"], str(hentai_ID))) and len(os.listdir(os.path.join(config["LIBRARY_PATH"], str(hentai_ID))))==0: # if cache folder still exists and is empty:
try:
os.rmdir(os.path.join(settings["library_path"], str(hentai_ID))) # try to clean up
except PermissionError as e: # may fail if another process is still using directory like dropbox
logging.warning(f"Deleting \"{os.path.join(settings["library_path"], str(hentai_ID))}/\" failed with {KFSfstr.full_class_name(e)}.")
cleanup_success=False # cleanup unsuccessful
os.rmdir(os.path.join(config["LIBRARY_PATH"], str(hentai_ID))) # try to clean up
except PermissionError as e: # may fail if another process is still using directory like dropbox
logging.warning(f"Deleting \"{os.path.join(config["LIBRARY_PATH"], str(hentai_ID))}/\" failed with {KFSfstr.full_class_name(e)}.")
cleanup_success=False # cleanup unsuccessful
if cleanup_success==True:
logging.info("\rDeleted leftover image directories.")

Expand Down

0 comments on commit 728d503

Please sign in to comment.