upgraded to python3.12; hopefully fixed memory leak drastically reduc…

…ing RAM requirement with the reliable PIL backend; removed hentai ID list sorting except when creating downloadme.txt
9-FS · Feb 13, 2024 · e231292 · e231292
1 parent f323ad0
commit e231292
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 46 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -5,19 +5,19 @@ license     = "MIT"
 name        = "x"                                                        # can't leave empty because of bug with `poetry install` from poetry.lock file
 readme      = "readme.md"
 repository  = "https://github.com/9-FS/2022-10-23-nHentai-to-PDF-Server"
-version     = "3.0.4"
+version     = "3.1.0"
 
 [tool.poetry.dependencies]
 kfsconfig = "^1.0.0"
 kfsfstr   = "^1.0.0"
 kfslog    = "^1.0.0"
-kfsmedia  = "^2.4.0"
+kfsmedia  = "^2.5.1"
 kfssleep  = "^1.0.0"
-python    = "^3.11.0"
+python    = "^3.12.0"
 
 [tool.poetry.group.dev.dependencies]
-hypothesis = "^6.87.0"
-pytest     = "^7.4.2"
+hypothesis = "^6.0.0"
+pytest     = "^7.0.0"
 
 [build-system]
 build-backend = "poetry.core.masonry.api"

diff --git a/requirements.txt b/requirements.txt
@@ -1,15 +1,15 @@
-certifi==2024.2.2 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-charset-normalizer==3.3.2 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-colorama==0.4.6 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-idna==3.6 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-kfsconfig==1.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-kfsfstr==1.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-kfslog==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-kfsmath==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-kfsmedia==2.4.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-kfssleep==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-olefile==0.47 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-pebble==5.0.6 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-pillow==10.2.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-requests==2.31.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
-urllib3==2.2.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
+certifi==2024.2.2 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+charset-normalizer==3.3.2 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+colorama==0.4.6 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+idna==3.6 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+kfsconfig==1.1.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+kfsfstr==1.1.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+kfslog==1.1.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+kfsmath==1.0.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+kfsmedia==2.5.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+kfssleep==1.0.1 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+olefile==0.47 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+pebble==5.0.6 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+pillow==10.2.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+requests==2.31.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
+urllib3==2.2.0 ; python_full_version >= "3.12.0" and python_full_version < "4.0.0"
diff --git a/src/Hentai.py b/src/Hentai.py
@@ -1,9 +1,9 @@
 # Copyright (c) 2023 구FS, all rights reserved. Subject to the MIT licence in `licence.md`.
-import inspect
 import dataclasses
+import inspect
+import json
 from KFSfstr  import KFSfstr
 from KFSmedia import KFSmedia
-import json
 import logging
 import os
 import random
@@ -189,7 +189,7 @@ def download(self, library_path: str, library_split: int) -> bytes:
         - FileExistsError: File \"{PDF_filepath}\" already exists.
         - Hentai.DownloadError:
             - \"{PDF_filepath}\" already exists as directory.
-            - Can't generate page URL for {self} page {i+1}, because media type \"{page['t']}\" is unknown.
+            - Can't generate page URL for {self} page {i+1}, because media type \"{page["t"]}\" is unknown.
             - Tried to download and convert hentai {self} several times, but failed.
         """
 
@@ -208,11 +208,11 @@ def download(self, library_path: str, library_split: int) -> bytes:
 
         for i, page in enumerate(self._gallery["images"]["pages"]):
             if page["t"] not in MEDIA_TYPES.keys(): # if media type unknown:
-                logging.error(f"Can't generate page URL for {self} page {i+1}, because media type \"{page['t']}\" is unknown.")
-                raise KFSmedia.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: Can't generate page URL for {self} page {i+1}, because media type \"{page['t']}\" is unknown.")
+                logging.error(f"Can't generate page URL for {self} page {i+1}, because media type \"{page["t"]}\" is unknown.")
+                raise KFSmedia.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: Can't generate page URL for {self} page {i+1}, because media type \"{page["t"]}\" is unknown.")
 
-            pages_URL.append(f"https://i{random.choice(['', '2', '3', '5', '7'])}.nhentai.net/galleries/{self._gallery['media_id']}/{i+1}{MEDIA_TYPES[page['t']]}") # URL, use random image server instance to distribute load
-            images_filepath.append(os.path.join(library_path, str(self.ID), f"{self.ID}-{i+1}{MEDIA_TYPES[page['t']]}"))                                            # media filepath, but usually image filepath
+            pages_URL.append(f"https://i{random.choice(["", "2", "3", "5", "7"])}.nhentai.net/galleries/{self._gallery["media_id"]}/{i+1}{MEDIA_TYPES[page["t"]]}") # URL, use random image server instance to distribute load
+            images_filepath.append(os.path.join(library_path, str(self.ID), f"{self.ID}-{i+1}{MEDIA_TYPES[page["t"]]}"))                                            # media filepath, but usually image filepath
 
         PDF_filepath=self.title
         for c in TITLE_CHARACTERS_FORBIDDEN:                                                                                                                                        # remove forbidden characters for filenames

diff --git a/src/get_hentai_ID_list.py b/src/get_hentai_ID_list.py
@@ -50,7 +50,7 @@ def get_hentai_ID_list(downloadme_filepath: str, cookies: dict[str, str], header
 
 def _get_hentai_ID_list_from_tag_search(cookies: dict[str, str], headers: dict[str, str], nhentai_tag: str) -> list[int]:
     """
-    Tries to return hentai ID list to download by searching on nhentai.net for all hentai ID with tag nhentai_tag.
+    Tries to return sorted hentai ID list to download by searching on nhentai.net for all hentai ID with tag nhentai_tag.
 
     Arguments:
     - cookies: cookies to send with the request to bypass bot protection
@@ -85,6 +85,7 @@ def _get_hentai_ID_list_from_tag_search(cookies: dict[str, str], headers: dict[s
             page_no_current+=1
 
     hentai_ID_list=_convert_hentai_ID_list_str_to_hentai_ID_list_int(hentai_ID_list_str)    # list[str] -> list[int], clean up data
+    hentai_ID_list=sorted(hentai_ID_list)                                                   # sort list downloaded numerically
 
     return hentai_ID_list
 
@@ -189,7 +190,7 @@ def _search_hentai_ID_by_tag(search_request: requests.PreparedRequest) -> list[s
 
 def _convert_hentai_ID_list_str_to_hentai_ID_list_int(hentai_ID_list_str: list[str]) -> list[int]:
     """
-    Converts list of hentai ID from list[str] to list[int] cleans up entries.
+    Converts list of hentai ID from list[str] to list[int] cleans up entries. Does not sort to respect input order.
 
     Arguments:
     - hentai_ID_list_str: list of hentai ID in str to convert
@@ -209,7 +210,5 @@ def _convert_hentai_ID_list_str_to_hentai_ID_list_int(hentai_ID_list_str: list[s
             hentai_ID_list.append(int(hentai_ID))
         except ValueError:                                                                  # if input invalid: discard that, keep rest
             logging.error(f"Converting input \"{hentai_ID}\" to int failed. Skipping ID.")
-
-    hentai_ID_list=sorted(hentai_ID_list)                                                   # sort numerically
 
     return hentai_ID_list
diff --git a/src/main.py b/src/main.py
@@ -1,4 +1,5 @@
 # Copyright (c) 2023 구FS, all rights reserved. Subject to the MIT licence in `licence.md`.
+import gc   # garbage collector, explicitly free memory
 import json
 from KFSconfig  import KFSconfig
 from KFSfstr    import KFSfstr
@@ -68,6 +69,7 @@ def main(DEBUG: bool):
                 with open("./log/FAILURES.txt", "at") as fails_file:                        # append in failure file
                     fails_file.write(f"{hentai.ID}\n")
                 continue                                                                    # skip to next hentai
+            gc.collect()                                                                    # explicitly free memory, otherwise PDF may clutter memory
         logging.info("--------------------------------------------------")
 
 
@@ -79,7 +81,7 @@ def main(DEBUG: bool):
                 try:
                     os.rmdir(os.path.join(settings["library_path"], str(hentai_ID)))                                                                                    # try to clean up
                 except PermissionError as e:                                                                                                                            # may fail if another process is still using directory like dropbox
-                    logging.warning(f"Deleting \"{os.path.join(settings['library_path'], str(hentai_ID))}/\" failed with {KFSfstr.full_class_name(e)}.")
+                    logging.warning(f"Deleting \"{os.path.join(settings["library_path"], str(hentai_ID))}/\" failed with {KFSfstr.full_class_name(e)}.")
                     cleanup_success=False                                                                                                                               # cleanup unsuccessful
         if cleanup_success==True:
             logging.info("\rDeleted leftover image directories.")