switching gpt model, making 429s hard stop

RobokopU24 · Oct 24, 2024 · 6b6f999 · 6b6f999
1 parent 723651e
commit 6b6f999
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 5 deletions.
diff --git a/parsers/LitCoin/src/bagel/bagel.py b/parsers/LitCoin/src/bagel/bagel.py
@@ -13,8 +13,8 @@
 
 # output of parse_gpt looks like {"entity": triple["object"], "qualifier": triple["object_qualifier"]}
 session = requests.Session()
-retries = Retry(total=8,
-                backoff_factor=.75,
+retries = Retry(total=6,
+                backoff_factor=1,
                 status_forcelist=[502, 503, 504, 520, 429])
 session.mount('http://', HTTPAdapter(max_retries=retries))
 session.mount('https://', HTTPAdapter(max_retries=retries))

diff --git a/parsers/LitCoin/src/bagel/bagel_gpt.py b/parsers/LitCoin/src/bagel/bagel_gpt.py
@@ -85,7 +85,7 @@ def query(prompt, requests_session):
     }
 
     payload = {
-        "model": "gpt-4-0125-preview",
+        "model": "gpt-4o-mini",
         "messages": [
             {
                 "role": "user",

diff --git a/parsers/LitCoin/src/loadLitCoin.py b/parsers/LitCoin/src/loadLitCoin.py
@@ -2,6 +2,8 @@
 import os
 import json
 
+import requests.exceptions
+
 from Common.biolink_utils import BiolinkUtils
 from Common.loader_interface import SourceDataLoader
 from Common.biolink_constants import PUBLICATIONS
@@ -216,10 +218,12 @@ def parse_data(self) -> dict:
                                                                abstract_id=abstract_id)
                         self.bagel_results_lookup[subject_name] = bagel_results
                         bagelized_success += 1
-                    except Exception as e:
+                    except requests.exceptions.HTTPError as e:
                         self.logger.error(f'Failed Bagelization: {type(e)}:{e}')
                         skipped_records += 1
                         bagelization_errors += 1
+                        if e.response.status_code == 429:
+                            raise e
                         continue
                 else:
                     bagel_results = self.bagel_results_lookup[subject_name]
@@ -246,10 +250,12 @@ def parse_data(self) -> dict:
                                                                abstract_id=abstract_id)
                         self.bagel_results_lookup[object_name] = bagel_results
                         bagelized_success += 1
-                    except Exception as e:
+                    except requests.exceptions.HTTPError as e:
                         self.logger.error(f'Failed Bagelization: {type(e)}:{e}')
                         skipped_records += 1
                         bagelization_errors += 1
+                        if e.response.status_code == 429:
+                            raise e
                         continue
                 else:
                     bagel_results = self.bagel_results_lookup[object_name]