From 39a607e5b779c7749a254c5a95838dc9437bc740 Mon Sep 17 00:00:00 2001 From: Hadrien Bossard Date: Mon, 23 Dec 2024 15:29:31 +0100 Subject: [PATCH] refactor(metrics): use simple_connection_retry --- dgv/metrics/task_functions.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/dgv/metrics/task_functions.py b/dgv/metrics/task_functions.py index 4f23b9fd..09e94d6a 100644 --- a/dgv/metrics/task_functions.py +++ b/dgv/metrics/task_functions.py @@ -17,6 +17,7 @@ AIRFLOW_DAG_TMP, MINIO_BUCKET_INFRA, ) +from datagouvfr_data_pipelines.utils.retry import simple_connection_retry from datagouvfr_data_pipelines.utils.download import download_files from datagouvfr_data_pipelines.utils.minio import MinIOClient, File as MinioFile from datagouvfr_data_pipelines.utils.postgres import ( @@ -515,6 +516,7 @@ def process_log(ti): ti.xcom_push(key="dates_processed", value=dates_processed) +@simple_connection_retry def get_matomo_outlinks(model, slug, target, metric_date): matomo_url = "https://stats.data.gouv.fr/index.php" params = { @@ -528,14 +530,7 @@ def get_matomo_outlinks(model, slug, target, metric_date): "period": "day", "date": metric_date.isoformat(), } - tries = 5 - while tries > 0: - matomo_res = requests.get(matomo_url, params=params) - # The Matomo API is sometimes unstable and required a few retries - if matomo_res.status_code in ["502"]: - time.sleep(0.5) - tries-=1 - break + matomo_res = requests.get(matomo_url, params=params) matomo_res.raise_for_status() return sum( outlink["nb_hits"]