Skip to content

Commit

Permalink
Catch a different SSL Error
Browse files Browse the repository at this point in the history
This has been running for two hours, and seems to be working better? I
think I was retrying on the wrong error before🫰.
  • Loading branch information
edsu committed Jun 24, 2024
1 parent 95e1ba1 commit 7ce7839
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions rialto_airflow/harvest/openalex.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import time

import requests
from requests.exceptions import SSLError
from ssl import SSLEOFError
from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_random

from rialto_airflow.utils import invert_dict
Expand Down Expand Up @@ -32,7 +32,7 @@ def doi_orcids_pickle(authors_csv, pickle_file, limit=None):
@retry(
wait=wait_random(1, 5),
stop=stop_after_delay(60),
retry=retry_if_exception_type(SSLError),
retry=retry_if_exception_type(SSLEOFError)
)
def dois_from_orcid(orcid: str):
"""
Expand Down Expand Up @@ -70,6 +70,8 @@ def works_from_author_id(author_id, limit=None):
while has_more:
page += 1
params["page"] = page

logging.info(f"fetching works for {author_id} page={page}")
resp = requests.get(url, params)

if resp.status_code == 200:
Expand Down

0 comments on commit 7ce7839

Please sign in to comment.