Seed more realistic housing adresses and coordinates

frontend/package.json

@@ -94,7 +94,6 @@
     "identity-obj-proxy": "^3.0.0",
     "jest": "^29.7.0",
     "jest-extended": "^4.0.2",
-    "jest-fetch-mock": "^3.0.3",
     "jest-watch-typeahead": "^2.2.2",
     "randomstring": "^1.3.0",
     "react-dev-utils": "^12.0.1",

server/package.json

@@ -33,6 +33,7 @@
     "@zerologementvacant/schemas": "workspace:*",
     "@zerologementvacant/utils": "workspace:*",
     "async": "^3.2.6",
+    "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
     "cli-progress": "^3.12.0",
     "commander": "^12.1.0",
@@ -66,7 +67,6 @@
     "multer": "^1.4.5-lts.1",
     "multer-s3": "^3.0.1",
     "nock": "^13.5.5",
-    "node-fetch": "^3.3.2",
     "nodemailer": "^6.9.15",
     "parse-redis-url-simple": "^1.0.2",
     "pdf-lib": "^1.17.1",
@@ -123,7 +123,6 @@
     "@types/wuzzy": "^0.1.3",
     "jest": "^29.7.0",
     "jest-extended": "^4.0.2",
-    "jest-fetch-mock": "^3.0.3",
     "jest-sorted": "^1.0.15",
     "nodemailer-mock": "^2.0.6",
     "nodemon": "^3.1.7",

server/src/controllers/datafoncierHousingController.test.ts

@@ -7,13 +7,13 @@ import {
   genDatafoncierHousing,
   genEstablishmentApi,
   genUserApi,
-  oneOf,
+  oneOf
 } from '~/test/testFixtures';
 import { DatafoncierHouses } from '~/repositories/datafoncierHousingRepository';
 import { formatUserApi, Users } from '~/repositories/userRepository';
 import {
   Establishments,
-  formatEstablishmentApi,
+  formatEstablishmentApi
 } from '~/repositories/establishmentRepository';
 
 describe('Datafoncier housing controller', () => {
@@ -55,13 +55,6 @@ describe('Datafoncier housing controller', () => {
     });
 
     it('should return "not found" otherwise', async () => {
-      fetchMock.mockIf(
-        (request) => request.url.endsWith(`/ff/locaux/missing`),
-        async () => ({
-          status: 404,
-        }),
-      );
-
       const { status } = await request(app)
         .get(testRoute('missing'))
         .use(tokenProvider(user));

server/src/infra/database/seeds/development/20240404235458_housings.ts

@@ -1,7 +1,9 @@
 import { faker } from '@faker-js/faker/locale/fr';
+import * as turf from '@turf/turf';
 import async from 'async';
 import { Knex } from 'knex';
 
+import { AddressKinds } from '@zerologementvacant/models';
 import { Establishments } from '~/repositories/establishmentRepository';
 import { HousingApi } from '~/models/HousingApi';
 import { genHousingApi, genOwnerApi } from '~/test/testFixtures';
@@ -16,23 +18,113 @@ import {
   formatHousingOwnerApi,
   housingOwnersTable
 } from '~/repositories/housingOwnerRepository';
+import { Feature, MultiPolygon, Polygon, Position } from 'geojson';
+import { createBanAPI } from '~/services/ban/ban-api';
+import { MarkRequired } from 'ts-essentials';
+import {
+  banAddressesTable,
+  formatAddressApi
+} from '~/repositories/banAddressesRepository';
+import { AddressApi } from '~/models/AddressApi';
 
 export async function seed(knex: Knex): Promise<void> {
-  const establishments = await Establishments(knex).where({ available: true });
+  const ban = createBanAPI();
+
+  await knex.raw(`TRUNCATE TABLE ${housingOwnersTable} CASCADE`);
+  await knex.raw(`TRUNCATE TABLE ${housingTable} CASCADE`);
+  await knex.raw(`TRUNCATE TABLE ${ownerTable} CASCADE`);
 
+  const establishments = await Establishments(knex).where({ available: true });
   await async.forEachSeries(establishments, async (establishment) => {
-    const housings: ReadonlyArray<HousingApi> = faker.helpers.multiple(
-      () =>
-        genHousingApi(
-          faker.helpers.arrayElement(establishment.localities_geo_code)
-        ),
-      {
-        count: {
-          min: 100,
-          max: 10000
+    const id =
+      establishment.kind === 'Commune'
+        ? establishment.localities_geo_code[0]
+        : establishment.siren;
+    const kind = establishment.kind === 'Commune' ? 'communes' : 'epcis';
+    const response = await fetch(
+      `https://geo.api.gouv.fr/${kind}/${id}?format=geojson&geometry=contour`
+    );
+    if (!response.ok) {
+      const error = await response.json();
+      console.log(error);
+      throw new Error('Failed to fetch geojson');
+    }
+
+    const epci = await response.json();
+    const contour = (epci as Feature<Polygon | MultiPolygon>).geometry;
+
+    const baseHousings: ReadonlyArray<
+      MarkRequired<HousingApi, 'longitude' | 'latitude'>
+    > = faker.helpers
+      .multiple(
+        () =>
+          genHousingApi(
+            faker.helpers.arrayElement(establishment.localities_geo_code)
+          ),
+        {
+          count: {
+            min: 100,
+            max: 10000
+          }
         }
-      }
+      )
+      // Put the housing inside the establishment perimeter
+      .map((housing) => {
+        const point = generatePointInside(contour);
+        return {
+          ...housing,
+          longitude: point[0],
+          latitude: point[1]
+        };
+      });
+
+    // Infer housing addresses using the generated coordinates
+    const points = baseHousings.map((housing) => ({
+      refId: housing.id,
+      geoCode: housing.geoCode,
+      longitude: housing.longitude,
+      latitude: housing.latitude
+    }));
+    const addresses = await ban.reverseMany(points).then((addresses) => {
+      return addresses.filter((address) => !!address.label);
+    });
+    const housings = baseHousings
+      .filter((housing) => {
+        return addresses.some(
+          (address) =>
+            address.refId === housing.id && address.geoCode === housing.geoCode
+        );
+      })
+      .map<HousingApi>((housing, i) => {
+        const address = addresses[i];
+        if (address.refId !== housing.id) {
+          throw new Error('Should never happen');
+        }
+        return {
+          ...housing,
+          rawAddress: [address.label]
+        };
+      });
+
+    // Insert housings
+    console.log(`Inserting ${housings.length} housings...`, {
+      establishment: establishment.name
+    });
+    await knex.batchInsert(housingTable, housings.map(formatHousingRecordApi));
+
+    // Insert BAN housing addresses
+    const housingAddresses: ReadonlyArray<AddressApi> = addresses.map(
+      (address) => ({ ...address, addressKind: AddressKinds.Housing })
+    );
+    console.log(
+      `Inserting ${housingAddresses.length} BAN housing addresses...`,
+      { establishment: establishment.name }
+    );
+    await knex.batchInsert(
+      banAddressesTable,
+      housingAddresses.map(formatAddressApi)
     );
+
     const housingOwners: ReadonlyArray<HousingOwnerApi> = housings.flatMap(
       (housing) => {
         const owners = faker.helpers.multiple(() => genOwnerApi(), {
@@ -55,12 +147,26 @@ export async function seed(knex: Knex): Promise<void> {
       }
     );
     const owners: ReadonlyArray<OwnerApi> = housingOwners.flat();
-
-    await knex.batchInsert(housingTable, housings.map(formatHousingRecordApi));
+    console.log(`Inserting ${owners.length} owners...`, {
+      establishment: establishment.name
+    });
     await knex.batchInsert(ownerTable, owners.map(formatOwnerApi));
+
+    console.log(`Inserting ${housingOwners.length} housing owners...`, {
+      establishment: establishment.name
+    });
     await knex.batchInsert(
       housingOwnersTable,
       housingOwners.map(formatHousingOwnerApi)
     );
   });
 }
+
+function generatePointInside(perimeter: Polygon | MultiPolygon): Position {
+  function generate(): Position {
+    const bbox = turf.bbox(perimeter);
+    const point = turf.randomPosition(bbox);
+    return turf.booleanPointInPolygon(point, perimeter) ? point : generate();
+  }
+  return generate();
+}

server/src/services/ban/ban-api.ts

@@ -0,0 +1,142 @@
+import axios from 'axios';
+import { parse as fromCSV } from 'csv-parse/sync';
+import { stringify as toCSV } from 'csv-stringify/sync';
+
+import { Address, AddressQuery, BAN, Point } from '~/services/ban/ban';
+
+class BanAPI implements BAN {
+  readonly http = axios.create({
+    baseURL: 'https://api-adresse.data.gouv.fr'
+  });
+
+  async searchMany<Q extends AddressQuery>(
+    queries: ReadonlyArray<Q>
+  ): Promise<ReadonlyArray<Address & Q>> {
+    if (!queries.length) {
+      throw new Error('Must contain at least one query');
+    }
+
+    const [query] = queries;
+    const columns = listColumns(query);
+    const csv = toCSV(queries as Q[], {
+      header: true,
+      columns
+    });
+    const blob = new Blob([csv], { type: 'text/csv' });
+    const form = new FormData();
+    form.append('data', blob, 'search.csv');
+
+    // Add columns
+    columns.forEach((column) => {
+      form.append('columns', column);
+    });
+
+    // Tells the API that geoCode is an INSEE code
+    form.append('citycode', 'geoCode');
+    form.append('result_columns', 'result_id');
+    form.append('result_columns', 'result_label');
+    form.append('result_columns', 'result_type');
+    form.append('result_columns', 'result_housenumber');
+    form.append('result_columns', 'result_street');
+    form.append('result_columns', 'result_postcode');
+    form.append('result_columns', 'result_city');
+    form.append('result_columns', 'latitude');
+    form.append('result_columns', 'longitude');
+    form.append('result_columns', 'result_score');
+
+    const { data } = await this.http.post<string>('/search/csv', form);
+    const records: ReadonlyArray<BanAddress & Q> = fromCSV(data, {
+      columns: true
+    });
+    return records.filter((record) => !!record.result_id).map<Address & Q>(map);
+  }
+
+  async reverseMany<P extends Point>(
+    points: ReadonlyArray<P>
+  ): Promise<ReadonlyArray<Address & P>> {
+    if (!points.length) {
+      throw new Error('Must contain at least one point');
+    }
+
+    const columns = listColumns(points[0]);
+    const csv = toCSV(points as P[], {
+      header: true,
+      columns
+    });
+    const blob = new Blob([csv], { type: 'text/csv' });
+    const form = new FormData();
+    form.append('data', blob, 'reverse.csv');
+
+    // Add columns
+    columns.forEach((column) => {
+      form.append('columns', column);
+    });
+
+    // Tells the API that geoCode is an INSEE code
+
+    form.append('citycode', 'geoCode');
+    form.append('result_columns', 'result_id');
+    form.append('result_columns', 'result_label');
+    form.append('result_columns', 'result_type');
+    form.append('result_columns', 'result_housenumber');
+    form.append('result_columns', 'result_street');
+    form.append('result_columns', 'result_postcode');
+    form.append('result_columns', 'result_city');
+    form.append('result_columns', 'latitude');
+    form.append('result_columns', 'longitude');
+    form.append('result_columns', 'result_score');
+
+    const { data } = await this.http.post<string>('/reverse/csv', form);
+    const records: ReadonlyArray<BanAddress & P> = fromCSV(data, {
+      columns: true
+    });
+    return records.map<Address & P>(map);
+  }
+}
+
+interface BanAddress {
+  result_id: string;
+  result_label: string;
+  result_housenumber: string;
+  result_street: string;
+  result_postcode: string;
+  result_city: string;
+  latitude: string;
+  longitude: string;
+  result_score: string;
+}
+
+function listColumns<A extends object>(a: A): ReadonlyArray<string> {
+  return Object.keys(a);
+}
+
+function map<A>(address: BanAddress & A): Address & A {
+  const {
+    result_id,
+    result_label,
+    result_housenumber,
+    result_street,
+    result_postcode,
+    result_city,
+    result_score,
+    latitude,
+    longitude,
+    ...rest
+  } = address;
+  return {
+    ...(rest as unknown as A),
+    id: result_id,
+    label: result_label,
+    houseNumber: result_housenumber,
+    street: result_street,
+    postalCode: result_postcode,
+    city: result_city,
+    latitude: Number(latitude),
+    longitude: Number(longitude),
+    score: Number(result_score)
+  };
+}
+
+export function createBanAPI(): BAN {
+  return new BanAPI();
+}

server/src/services/ban/ban.ts

@@ -0,0 +1,29 @@
+export interface AddressQuery {
+  label: string;
+}
+
+export interface Address {
+  id: string;
+  label: string;
+  houseNumber: string;
+  street: string;
+  postalCode: string;
+  city: string;
+  latitude: number;
+  longitude: number;
+  score: number;
+}
+
+export interface Point {
+  longitude: number;
+  latitude: number;
+}
+
+export interface BAN {
+  searchMany<Q extends AddressQuery>(
+    addresses: ReadonlyArray<Q>
+  ): Promise<ReadonlyArray<Address & Q>>;
+  reverseMany<P extends Point>(
+    points: ReadonlyArray<P>
+  ): Promise<ReadonlyArray<Address & P>>;
+}