diff --git a/api/deno.lock b/api/deno.lock index 17d8eaf34..be7321abe 100644 --- a/api/deno.lock +++ b/api/deno.lock @@ -3,6 +3,11 @@ "packages": { "specifiers": { "jsr:@std/collections@1": "jsr:@std/collections@1.0.9", + "jsr:@std/crypto@1": "jsr:@std/crypto@1.0.4", + "jsr:@std/crypto@1.0.4": "jsr:@std/crypto@1.0.4", + "jsr:@std/encoding": "jsr:@std/encoding@1.0.7", + "jsr:@std/encoding@1": "jsr:@std/encoding@1.0.7", + "jsr:@std/encoding@1.0.7": "jsr:@std/encoding@1.0.7", "jsr:@std/semver@1": "jsr:@std/semver@1.0.3", "jsr:@superstruct/core@^2": "jsr:@superstruct/core@2.0.2", "npm:@faker-js/faker@^8.4": "npm:@faker-js/faker@8.4.1", @@ -71,6 +76,12 @@ "@std/collections@1.0.9": { "integrity": "4f58104ead08a04a2199374247f07befe50ba01d9cca8cbb23ab9a0419921e71" }, + "@std/crypto@1.0.4": { + "integrity": "cee245c453bd5366207f4d8aa25ea3e9c86cecad2be3fefcaa6cb17203d79340" + }, + "@std/encoding@1.0.7": { + "integrity": "f631247c1698fef289f2de9e2a33d571e46133b38d042905e3eac3715030a82d" + }, "@std/semver@1.0.3": { "integrity": "7c139c6076a080eeaa4252c78b95ca5302818d7eafab0470d34cafd9930c13c8" }, diff --git a/api/src/db/cmd-source.ts b/api/src/db/cmd-source.ts index b3dd78bd2..63b487a16 100644 --- a/api/src/db/cmd-source.ts +++ b/api/src/db/cmd-source.ts @@ -10,7 +10,7 @@ import { env_or_fail } from "@/lib/env/index.ts"; const connectionString = env_or_fail("APP_POSTGRES_URL"); const migrator = buildMigrator({ pool: { connectionString }, - app: { targetSchema: "geo", datasets: new Set() }, + app: { targetSchema: "geo" }, }); await migrator.prepare(); diff --git a/api/src/db/external_data/datasets.ts b/api/src/db/external_data/datasets.ts index 671db3e83..c02b5ecc6 100644 --- a/api/src/db/external_data/datasets.ts +++ b/api/src/db/external_data/datasets.ts @@ -3,7 +3,7 @@ import { AiresCovoiturage } from "./datasets/AiresCovoiturage.ts"; import { IncentiveCampaigns } from "./datasets/IncentiveCampaigns.ts"; import { CreateAiresCovoiturageTable } from "./datastructures/CreateAiresCovoiturageTable.ts"; import { CreateIncentiveCampaignsTable } from "./datastructures/CreateIncentiveCampaignsTable.ts"; -import { getAiresLastUrl } from "./helpers.ts"; +import { getAiresLastUrl, getCampaignsLastUrl } from "./helpers.ts"; export const datastructures: Set = new Set([ CreateAiresCovoiturageTable, @@ -13,9 +13,11 @@ export const datastructures: Set = new Set([ export const datasets = async () => { // add Aires migration const AiresUrl = "https://transport.data.gouv.fr/api/datasets/5d6eaffc8b4c417cdc452ac3"; - const url = await getAiresLastUrl(AiresUrl); + const airesResponse = await getAiresLastUrl(AiresUrl); + const CampaignsUrl = "https://www.data.gouv.fr/api/1/datasets/64a436118c609995b0386541"; + const campaignsResponse = await getCampaignsLastUrl(CampaignsUrl); const datasets: Set = new Set([]); - datasets.add(AiresCovoiturage(url)); - datasets.add(IncentiveCampaigns("https://www.data.gouv.fr/fr/datasets/r/08f58ee3-7b3e-43d8-9e55-3c82bf406190")); + datasets.add(AiresCovoiturage(airesResponse)); + datasets.add(IncentiveCampaigns(campaignsResponse)); return datasets; }; diff --git a/api/src/db/external_data/datasets/AiresCovoiturage.ts b/api/src/db/external_data/datasets/AiresCovoiturage.ts index 644536916..b355bdaa4 100644 --- a/api/src/db/external_data/datasets/AiresCovoiturage.ts +++ b/api/src/db/external_data/datasets/AiresCovoiturage.ts @@ -6,6 +6,8 @@ export function AiresCovoiturage(url: string): StaticAbstractDataset { static dataset = "aires_covoiturage"; static year = 2024; static url = url; + static sha256 = undefined; + static filename = undefined; static table = `aires_covoiturage_temp`; static skipStatePersistence = true; readonly targetTable = "aires_covoiturage"; diff --git a/api/src/db/external_data/datasets/IncentiveCampaigns.ts b/api/src/db/external_data/datasets/IncentiveCampaigns.ts index 53fa7e28b..3ac8aeb8c 100644 --- a/api/src/db/external_data/datasets/IncentiveCampaigns.ts +++ b/api/src/db/external_data/datasets/IncentiveCampaigns.ts @@ -6,6 +6,8 @@ export function IncentiveCampaigns(url: string): StaticAbstractDataset { static dataset = "incentive_campaigns"; static year = 2024; static url = url; + static sha256 = undefined; + static filename = undefined; static table = `incentive_campaigns_temp`; static skipStatePersistence = true; readonly targetTable = "incentive_campaigns"; diff --git a/api/src/db/external_data/helpers.ts b/api/src/db/external_data/helpers.ts index 9bc5ca23e..7f4b66dd5 100644 --- a/api/src/db/external_data/helpers.ts +++ b/api/src/db/external_data/helpers.ts @@ -1,12 +1,21 @@ export async function getAiresLastUrl(url: string): Promise { const response = await fetch(url, { method: "get" }); const res = await response.json(); - const list = res - ? res.history.filter((h) => h.payload.schema_name !== null) - : []; + if (!response.ok) { + console.error(res.error.data); + } + const list = res ? res.history.filter((h: any) => h.payload.schema_name !== null) : []; const fileUrl = list.length > 0 ? list[0].payload.permanent_url : ""; + return fileUrl; +} + +export async function getCampaignsLastUrl(url: string): Promise { + const response = await fetch(url, { method: "get" }); + const res = await response.json(); if (!response.ok) { console.error(res.error.data); } + const list = res ? res.resources : []; + const fileUrl = list.length > 0 ? list[0].latest : ""; return fileUrl; } diff --git a/api/src/db/geo/interfaces/DatasetInterface.ts b/api/src/db/geo/interfaces/DatasetInterface.ts index 05d15baa0..3347b0c67 100644 --- a/api/src/db/geo/interfaces/DatasetInterface.ts +++ b/api/src/db/geo/interfaces/DatasetInterface.ts @@ -15,8 +15,8 @@ export interface StaticMigrable { export interface StaticAbstractDataset extends StaticMigrable { readonly url: string; - readonly sha256: string | undefined; - readonly filename: string | undefined; + readonly sha256?: string; + readonly filename?: string; readonly producer: string; readonly dataset: string; } diff --git a/api/src/deps.ts b/api/src/deps.ts index 304059750..1e89a5969 100644 --- a/api/src/deps.ts +++ b/api/src/deps.ts @@ -232,11 +232,12 @@ export { }; export * as semver from "jsr:@std/semver@1"; export * as collections from "jsr:@std/collections@1"; -export * as stdCrypto from "https://deno.land/std@0.224.0/crypto/mod.ts"; +export * as stdCrypto from "jsr:@std/crypto@1"; export { decodeBase64, encodeBase64, -} from "https://deno.land/std@0.224.0/encoding/base64.ts"; + encodeHex, +} from "jsr:@std/encoding@1"; export * as log from "https://deno.land/std@0.224.0/log/mod.ts"; export * as path from "https://deno.land/std@0.224.0/path/posix/mod.ts"; export * as bcrypt from "https://deno.land/x/bcrypt@v0.4.1/mod.ts"; diff --git a/api/src/lib/crypto/index.ts b/api/src/lib/crypto/index.ts index 2d1ebb640..e57383112 100644 --- a/api/src/lib/crypto/index.ts +++ b/api/src/lib/crypto/index.ts @@ -1,6 +1,5 @@ -import { bcrypt, decodeBase64, encodeBase64, stdCrypto } from "@/deps.ts"; +import { bcrypt, decodeBase64, encodeBase64, encodeHex, stdCrypto } from "@/deps.ts"; import { exists, read } from "@/lib/file/index.ts"; -import { encodeHex } from "https://deno.land/std@0.214.0/encoding/hex.ts"; export async function bcrypt_hash( plaintext: string, @@ -72,6 +71,17 @@ export async function createHash(message: string): Promise { } export async function sha256sum(source: string | ReadableStream): Promise { + return shaSum(source, "SHA-256"); +} + +export async function sha1sum(source: string | ReadableStream): Promise { + return shaSum(source, "SHA-1"); +} + +export async function shaSum( + source: string | ReadableStream, + alg: stdCrypto.DigestAlgorithm = "SHA-256", +): Promise { let stream; if (source instanceof ReadableStream) { stream = source; @@ -84,6 +94,6 @@ export async function sha256sum(source: string | ReadableStream): Pr stream = file.readable; } - const hashBuffer = await stdCrypto.crypto.subtle.digest("SHA-256", stream); + const hashBuffer = await stdCrypto.crypto.subtle.digest(alg, stream); return encodeHex(hashBuffer); }