External data (#2795)

* export sha256sum and sha1sum functions * remove console.log (#2794) * feat: add helper for campaigns url * export sha256sum and sha1sum functions * feat: add helper for campaigns url * fix geo migrations * fix type for datasets --------- Co-authored-by: Ludovic Delhomme <[email protected]>
betagouv · Feb 6, 2025 · 2f92e8b · 2f92e8b
1 parent 17d635d
commit 2f92e8b
Show file tree

Hide file tree

Showing 9 changed files with 52 additions and 15 deletions.
diff --git a/api/deno.lock b/api/deno.lock
diff --git a/api/src/db/cmd-source.ts b/api/src/db/cmd-source.ts
@@ -10,7 +10,7 @@ import { env_or_fail } from "@/lib/env/index.ts";
 const connectionString = env_or_fail("APP_POSTGRES_URL");
 const migrator = buildMigrator({
   pool: { connectionString },
-  app: { targetSchema: "geo", datasets: new Set() },
+  app: { targetSchema: "geo" },
 });
 
 await migrator.prepare();

diff --git a/api/src/db/external_data/datasets.ts b/api/src/db/external_data/datasets.ts
@@ -3,7 +3,7 @@ import { AiresCovoiturage } from "./datasets/AiresCovoiturage.ts";
 import { IncentiveCampaigns } from "./datasets/IncentiveCampaigns.ts";
 import { CreateAiresCovoiturageTable } from "./datastructures/CreateAiresCovoiturageTable.ts";
 import { CreateIncentiveCampaignsTable } from "./datastructures/CreateIncentiveCampaignsTable.ts";
-import { getAiresLastUrl } from "./helpers.ts";
+import { getAiresLastUrl, getCampaignsLastUrl } from "./helpers.ts";
 
 export const datastructures: Set<StaticMigrable> = new Set([
   CreateAiresCovoiturageTable,
@@ -13,9 +13,11 @@ export const datastructures: Set<StaticMigrable> = new Set([
 export const datasets = async () => {
   // add Aires migration
   const AiresUrl = "https://transport.data.gouv.fr/api/datasets/5d6eaffc8b4c417cdc452ac3";
-  const url = await getAiresLastUrl(AiresUrl);
+  const airesResponse = await getAiresLastUrl(AiresUrl);
+  const CampaignsUrl = "https://www.data.gouv.fr/api/1/datasets/64a436118c609995b0386541";
+  const campaignsResponse = await getCampaignsLastUrl(CampaignsUrl);
   const datasets: Set<StaticAbstractDataset> = new Set([]);
-  datasets.add(AiresCovoiturage(url));
-  datasets.add(IncentiveCampaigns("https://www.data.gouv.fr/fr/datasets/r/08f58ee3-7b3e-43d8-9e55-3c82bf406190"));
+  datasets.add(AiresCovoiturage(airesResponse));
+  datasets.add(IncentiveCampaigns(campaignsResponse));
   return datasets;
 };
diff --git a/api/src/db/external_data/datasets/AiresCovoiturage.ts b/api/src/db/external_data/datasets/AiresCovoiturage.ts
@@ -6,6 +6,8 @@ export function AiresCovoiturage(url: string): StaticAbstractDataset {
     static dataset = "aires_covoiturage";
     static year = 2024;
     static url = url;
+    static sha256 = undefined;
+    static filename = undefined;
     static table = `aires_covoiturage_temp`;
     static skipStatePersistence = true;
     readonly targetTable = "aires_covoiturage";

diff --git a/api/src/db/external_data/datasets/IncentiveCampaigns.ts b/api/src/db/external_data/datasets/IncentiveCampaigns.ts
@@ -6,6 +6,8 @@ export function IncentiveCampaigns(url: string): StaticAbstractDataset {
     static dataset = "incentive_campaigns";
     static year = 2024;
     static url = url;
+    static sha256 = undefined;
+    static filename = undefined;
     static table = `incentive_campaigns_temp`;
     static skipStatePersistence = true;
     readonly targetTable = "incentive_campaigns";

diff --git a/api/src/db/external_data/helpers.ts b/api/src/db/external_data/helpers.ts
@@ -1,12 +1,21 @@
 export async function getAiresLastUrl(url: string): Promise<string> {
   const response = await fetch(url, { method: "get" });
   const res = await response.json();
-  const list = res
-    ? res.history.filter((h) => h.payload.schema_name !== null)
-    : [];
+  if (!response.ok) {
+    console.error(res.error.data);
+  }
+  const list = res ? res.history.filter((h: any) => h.payload.schema_name !== null) : [];
   const fileUrl = list.length > 0 ? list[0].payload.permanent_url : "";
+  return fileUrl;
+}
+
+export async function getCampaignsLastUrl(url: string): Promise<string> {
+  const response = await fetch(url, { method: "get" });
+  const res = await response.json();
   if (!response.ok) {
     console.error(res.error.data);
   }
+  const list = res ? res.resources : [];
+  const fileUrl = list.length > 0 ? list[0].latest : "";
   return fileUrl;
 }
diff --git a/api/src/db/geo/interfaces/DatasetInterface.ts b/api/src/db/geo/interfaces/DatasetInterface.ts
@@ -15,8 +15,8 @@ export interface StaticMigrable {
 
 export interface StaticAbstractDataset extends StaticMigrable {
   readonly url: string;
-  readonly sha256: string | undefined;
-  readonly filename: string | undefined;
+  readonly sha256?: string;
+  readonly filename?: string;
   readonly producer: string;
   readonly dataset: string;
 }

diff --git a/api/src/deps.ts b/api/src/deps.ts
@@ -232,11 +232,12 @@ export {
 };
 export * as semver from "jsr:@std/semver@1";
 export * as collections from "jsr:@std/collections@1";
-export * as stdCrypto from "https://deno.land/[email protected]/crypto/mod.ts";
+export * as stdCrypto from "jsr:@std/crypto@1";
 export {
   decodeBase64,
   encodeBase64,
-} from "https://deno.land/[email protected]/encoding/base64.ts";
+  encodeHex,
+} from "jsr:@std/encoding@1";
 export * as log from "https://deno.land/[email protected]/log/mod.ts";
 export * as path from "https://deno.land/[email protected]/path/posix/mod.ts";
 export * as bcrypt from "https://deno.land/x/[email protected]/mod.ts";

diff --git a/api/src/lib/crypto/index.ts b/api/src/lib/crypto/index.ts
@@ -1,6 +1,5 @@
-import { bcrypt, decodeBase64, encodeBase64, stdCrypto } from "@/deps.ts";
+import { bcrypt, decodeBase64, encodeBase64, encodeHex, stdCrypto } from "@/deps.ts";
 import { exists, read } from "@/lib/file/index.ts";
-import { encodeHex } from "https://deno.land/[email protected]/encoding/hex.ts";
 
 export async function bcrypt_hash(
   plaintext: string,
@@ -72,6 +71,17 @@ export async function createHash(message: string): Promise<string> {
 }
 
 export async function sha256sum(source: string | ReadableStream<Uint8Array>): Promise<string> {
+  return shaSum(source, "SHA-256");
+}
+
+export async function sha1sum(source: string | ReadableStream<Uint8Array>): Promise<string> {
+  return shaSum(source, "SHA-1");
+}
+
+export async function shaSum(
+  source: string | ReadableStream<Uint8Array>,
+  alg: stdCrypto.DigestAlgorithm = "SHA-256",
+): Promise<string> {
   let stream;
   if (source instanceof ReadableStream) {
     stream = source;
@@ -84,6 +94,6 @@ export async function sha256sum(source: string | ReadableStream<Uint8Array>): Pr
     stream = file.readable;
   }
 
-  const hashBuffer = await stdCrypto.crypto.subtle.digest("SHA-256", stream);
+  const hashBuffer = await stdCrypto.crypto.subtle.digest(alg, stream);
   return encodeHex(hashBuffer);
 }