Skip to content
This repository has been archived by the owner on Sep 6, 2024. It is now read-only.

Commit

Permalink
fixed some env stuff
Browse files Browse the repository at this point in the history
  • Loading branch information
9FS committed May 12, 2024
1 parent 27cb147 commit df8677b
Show file tree
Hide file tree
Showing 5 changed files with 23 additions and 19 deletions.
3 changes: 3 additions & 0 deletions docker-compose.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ services:
environment:
- HOST_OS=Unraid
- TZ=UTC
- CF_CLEARANCE=
- CSRFTOKEN=
- USER_AGENT=
volumes:
- /mnt/user/appdata/2022-10-23-nhentai-to-pdf-server/config/:/app/config/:rw
- /mnt/user/appdata/2022-10-23-nhentai-to-pdf-server/log/:/app/log/:rw
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ name = ""
package-mode = false # this is an application, not library
readme = "readme.md"
repository = "https://github.com/9-FS/2022-10-23-nHentai-to-PDF-Server"
version = "3.3.0"
version = "3.3.1"

[tool.poetry.dependencies]
kfsconfig = "^2.0.0"
Expand Down
4 changes: 2 additions & 2 deletions src/Hentai.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ def _get_gallery(cls, nhentai_ID: int, cookies: dict[str, str], headers: dict[st
else: # if failed 3 times: give up
raise
if gallery_page.status_code==403: # if status code 403 (forbidden): probably cookies and headers not set correctly
logging.critical(f"Downloading gallery {nhentai_ID} from \"{NHENTAI_GALLERY_API_URL}/{nhentai_ID}\" resulted in status code {gallery_page.status_code}. Have you set \"cookies.json\" and \"headers.json\" correctly?")
raise requests.HTTPError(f"Error in {Hentai._get_gallery.__name__}{inspect.signature(Hentai._get_gallery)}: Downloading gallery {nhentai_ID} from \"{NHENTAI_GALLERY_API_URL}/{nhentai_ID}\" resulted in status code {gallery_page.status_code}. Have you set \"cookies.json\" and \"headers.json\" correctly?", response=gallery_page)
logging.critical(f"Downloading gallery {nhentai_ID} from \"{NHENTAI_GALLERY_API_URL}/{nhentai_ID}\" resulted in status code {gallery_page.status_code}. Have you set cookies and headers correctly?")
raise requests.HTTPError(f"Error in {Hentai._get_gallery.__name__}{inspect.signature(Hentai._get_gallery)}: Downloading gallery {nhentai_ID} from \"{NHENTAI_GALLERY_API_URL}/{nhentai_ID}\" resulted in status code {gallery_page.status_code}. Have you set cookies and headers correctly?", response=gallery_page)
if gallery_page.status_code==404: # if status code 404 (not found): hentai does not exist (anymore?)
logging.error(f"Hentai with ID \"{nhentai_ID}\" does not exist.")
raise ValueError(f"Error in {Hentai._get_gallery.__name__}{inspect.signature(Hentai._get_gallery)}: Hentai with ID \"{nhentai_ID}\" does not exist.")
Expand Down
8 changes: 4 additions & 4 deletions src/get_hentai_ID_list.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ def _get_page_no_max_by_tag(search_request: requests.PreparedRequest) -> int:
else: # if failed 3 times: give up
raise
if search_page.status_code==403: # if status code 403 (forbidden): probably cookies and headers not set correctly
logging.critical(f"Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set \"cookies.json\" and \"headers.json\" correctly?")
raise requests.HTTPError(f"Error in {_search_hentai_ID_by_tag.__name__}{inspect.signature(_search_hentai_ID_by_tag)}: Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set \"cookies.json\" and \"headers.json\" correctly?")
logging.critical(f"Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set cookies and headers correctly?")
raise requests.HTTPError(f"Error in {_search_hentai_ID_by_tag.__name__}{inspect.signature(_search_hentai_ID_by_tag)}: Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set cookies and headers correctly?")
if search_page.ok==False:
time.sleep(1)
if attempt_no<3: # try 3 times
Expand Down Expand Up @@ -167,8 +167,8 @@ def _search_hentai_ID_by_tag(search_request: requests.PreparedRequest) -> list[s
else: # if failed 3 times: give up
raise
if search_page.status_code==403: # if status code 403 (forbidden): probably cookies and headers not set correctly
logging.critical(f"Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set \"cookies.json\" and \"headers.json\" correctly?")
raise requests.HTTPError(f"Error in {_search_hentai_ID_by_tag.__name__}{inspect.signature(_search_hentai_ID_by_tag)}: Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set \"cookies.json\" and \"headers.json\" correctly?")
logging.critical(f"Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set cookies and headers correctly?")
raise requests.HTTPError(f"Error in {_search_hentai_ID_by_tag.__name__}{inspect.signature(_search_hentai_ID_by_tag)}: Downloading tag search from \"{search_request.url}\" resulted in status code {search_page.status_code}. Have you set cookies and headers correctly?")
if search_page.status_code==404: # if status code 404 (not found): nhenati API is sus and randomly does not have some search result pages
hentai_ID_list_str=[] # just return empty list
return hentai_ID_list_str
Expand Down
25 changes: 13 additions & 12 deletions src/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,28 @@
from KFSsleep import KFSsleep
import logging
import os
import typing
from get_hentai_ID_list import get_hentai_ID_list
from Hentai import Hentai


@KFSlog.timeit()
def main(DEBUG: bool):
cleanup_success: bool=True # cleanup successful
config: dict[str, str] # config
CONFIG_DEFAULT: dict[str, str]=\
config: dict[str, typing.Any] # config
CONFIG_DEFAULT: dict[str, typing.Any]=\
{
"DOWNLOADME_FILEPATH": "./config/downloadme.txt", # path to file containing hentai ID to download
"LIBRARY_PATH": "./hentai/", # path to download hentai to
"LIBRARY_SPLIT": "0", # split library into subdirectories of maximum this many hentai, 0 to disable
"LIBRARY_SPLIT": 10000, # split library into subdirectories of maximum this many hentai, 0 to disable
"NHENTAI_TAG": "language:english", # download hentai with this tag, normal tags are in format "tag:{tag}" for example "tag:ffm-threesome"
}
env: dict[str, str] # environment variables
ENV_DEFAULT: dict[str, str]=\
{
"CF_CLEARANCE": "", # for requests.get to bypass bot protection
"CSRFTOKEN": "",
"USER-AGENT": "",
"USER_AGENT": "",
}
hentai: Hentai # individual hentai
hentai_ID_list: list[int] # hentai ID to download
Expand All @@ -41,7 +42,7 @@ def main(DEBUG: bool):


while True:
hentai_ID_list=get_hentai_ID_list(config["DOWNLOADME_FILEPATH"], {"cf_clearance": env["CF_CLEARANCE"], "csrftoken": env["CSRFTOKEN"]}, {"User-Agent": env["USER-AGENT"]}, config["NHENTAI_TAG"]) # get desired hentai ID
hentai_ID_list=get_hentai_ID_list(config["DOWNLOADME_FILEPATH"], {"cf_clearance": env["CF_CLEARANCE"], "csrftoken": env["CSRFTOKEN"]}, {"User-Agent": env["USER_AGENT"]}, config["NHENTAI_TAG"]) # get desired hentai ID


for i, hentai_ID in enumerate(hentai_ID_list): # work through all desired hentai
Expand All @@ -52,22 +53,22 @@ def main(DEBUG: bool):
Hentai.save_galleries()

try:
hentai=Hentai(hentai_ID, {"cf_clearance": env["CF_CLEARANCE"], "csrftoken": env["CSRFTOKEN"]}, {"User-Agent": env["USER-AGENT"]}) # create hentai object
hentai=Hentai(hentai_ID, {"cf_clearance": env["CF_CLEARANCE"], "csrftoken": env["CSRFTOKEN"]}, {"User-Agent": env["USER_AGENT"]}) # create hentai object
except ValueError: # if hentai does not exist:
continue # skip to next hentai
else:
logging.info(hentai)

try:
_=hentai.download(config["LIBRARY_PATH"], int(config["LIBRARY_SPLIT"])) # download hentai
except FileExistsError: # if hentai already exists:
continue # skip to next hentai
_=hentai.download(config["LIBRARY_PATH"], config["LIBRARY_SPLIT"]) # download hentai
except FileExistsError: # if hentai already exists:
continue # skip to next hentai
except KFSmedia.DownloadError:
with open("./log/FAILURES.txt", "at") as fails_file: # append in failure file
with open("./log/FAILURES.txt", "at") as fails_file: # append in failure file
fails_file.write(f"{hentai.ID}\n")
continue # skip to next hentai
continue # skip to next hentai
del _
gc.collect() # explicitly free memory, otherwise PDF may clutter memory
gc.collect() # explicitly free memory, otherwise PDF may clutter memory
logging.info("--------------------------------------------------")


Expand Down

0 comments on commit df8677b

Please sign in to comment.