Merge pull request #765 from MTES-MCT/main

Mise en production 17/06/2024

.clever.json

@@ -47,6 +47,14 @@
       "git_ssh_url": "git+ssh://[email protected]/app_7abbd260-6a91-4262-9c2f-3107f1194821.git",
       "name": "Queue staging",
       "alias": "queue-staging"
+    },
+    {
+      "app_id": "app_0a820d50-269c-4d96-9b5e-81b112b0c608",
+      "org_id": "orga_31f13631-abb3-48d2-a58b-24b736f02a9d",
+      "deploy_url": "https://push-n3-par-clevercloud-customers.services.clever-cloud.com/app_0a820d50-269c-4d96-9b5e-81b112b0c608.git",
+      "git_ssh_url": "git+ssh://[email protected]/app_0a820d50-269c-4d96-9b5e-81b112b0c608.git",
+      "name": "Dagster (Production)",
+      "alias": "dagster-production"
     }
   ]
 }

.docker/dagster/Dockerfile

@@ -0,0 +1,34 @@
+FROM python:3.10-slim
+
+# ARG DAGSTER_PASSWORD=zlv
+# ARG DAGSTER_USERNAME=zlv
+
+# ENV DAGSTER_PASSWORD=$DAGSTER_PASSWORD
+# ENV DAGSTER_USERNAME=$DAGSTER_USERNAME
+
+RUN apt-get update && apt-get install -y nginx apache2-utils
+
+RUN pip install dagster-webserver dagster-postgres dagster-aws
+
+COPY analytics/dagster/requirements.txt .
+
+
+RUN pip install -r requirements.txt
+
+ENV DAGSTER_HOME=/opt/dagster/dagster_home/
+
+RUN mkdir -p $DAGSTER_HOME
+
+COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME
+
+COPY analytics/dagster/src $DAGSTER_HOME/src
+
+WORKDIR $DAGSTER_HOME
+
+# Setup Nginx configuration
+RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv
+COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default
+
+EXPOSE 8080 3000
+
+CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000

.github/workflows/github-actions-data-stack.yml

@@ -0,0 +1,14 @@
+name: Data Stack CI
+
+on:
+  push:
+    paths:
+      - analytics/**
+
+jobs:
+  deploy-dagster:
+    uses: ./.github/workflows/deploy.yml
+    with:
+      app: dagster-production
+      branch: main
+    secrets: inherit

.github/workflows/review-app.yml

@@ -66,6 +66,7 @@ jobs:
         run: |
           clever env set AUTH_SECRET "secret" -a $APP_ALIAS
           clever env set CC_HEALTH_CHECK_PATH "/" -a $APP_ALIAS
+          clever env set CC_OVERRIDE_BUILDCACHE '.:../.cache/puppeteer' -a $APP_ALIAS
           clever env set CC_PRE_RUN_HOOK 'corepack yarn workspace $WORKSPACE migrate && corepack yarn workspace $WORKSPACE seed' -a $APP_ALIAS
           clever env set CEREMA_API "https://api.cerema.fr" -a $APP_ALIAS
           clever env set CEREMA_ENABLED "false" -a $APP_ALIAS
@@ -135,6 +136,7 @@ jobs:
           clever env set API_HOST https://$PR_NAME-api.cleverapps.io/api -a $APP_ALIAS
           clever env set AUTH_SECRET "secret" -a $APP_ALIAS
           clever env set CC_HEALTH_CHECK_PATH "/" -a $APP_ALIAS
+          clever env set CC_OVERRIDE_BUILDCACHE '.:../.cache/puppeteer' -a $APP_ALIAS
           clever env set DATABASE_URL $POSTGRESQL_ADDON_URI -a $APP_ALIAS
           clever env set S3_ACCESS_KEY_ID $CELLAR_ADDON_KEY_ID -a $APP_ALIAS
           clever env set S3_BUCKET zerologementvacant -a $APP_ALIAS

.gitignore

@@ -33,3 +33,8 @@ yarn-error.log*
 
 /talisman_report
 .clever.json
+
+# Data Stack
+.pyc
+__pycache__
+analytics/dagster/storage/

.talismanrc

@@ -38,12 +38,6 @@ fileignoreconfig:
   checksum: a9dcff1f35295ed617ba50e44cc95c57c63130e80a43e9303c20f160c3b3165d
 - filename: frontend/src/views/Login/LoginView.test.tsx
   checksum: a1fab3ca0b731331cc1e0945e238f60ba319da577710ebeae502813623985a39
-- filename: packages/api-sdk/src/infra/token-provider.ts
-  checksum: 941d4fabe1280d748cb22341e8a4a9a5a405673ab1f2ff84bac942f38f5664aa
-- filename: packages/schemas/src/index.ts
-  checksum: 45473f39f4b07b3a33b6138a7e6fc3268690b59439d9f539eb1ce2c1abd430dc
-- filename: packages/schemas/src/password.ts
-  checksum: 716d8d646fd8a00af8d4225e15190810f9e8ab082fbb14e7430be109fa676611
 - filename: packages/api-sdk/src/infra/test/token-provider.test.ts
   checksum: c3ea87f28ca17a354e1dadf58e2832088503f258327f4620e0a23a24bb79241a
 - filename: packages/api-sdk/src/infra/token-provider.ts
@@ -54,8 +48,12 @@ fileignoreconfig:
   checksum: e2cabb9dd76c168622f38b4d446907fedcb6f690c86d1c9d1eccbf17a6fe0044
 - filename: packages/api-sdk/src/test/housing-api.test.ts
   checksum: fb256466ac0b962dee535b787282de895f60fb04274b994bf9f821a9851deecb
+- filename: packages/schemas/src/index.ts
+  checksum: 45473f39f4b07b3a33b6138a7e6fc3268690b59439d9f539eb1ce2c1abd430dc
+- filename: packages/schemas/src/password.ts
+  checksum: 716d8d646fd8a00af8d4225e15190810f9e8ab082fbb14e7430be109fa676611
 - filename: queue/.env.example
-  checksum: 30e4ee5808230342b969d42d9315fc7744900410a5797d7f7b2e984c678be2e7
+  checksum: df71affe33700cb0a36ee0b4824ef60e7d8e5a08f3d4d16ca53c31e9085b45ae
 - filename: queue/README.md
   checksum: fe2ac5277a42e6bf8dfc1e0fe53cc3308bba9019d9091f4b8c291e4a2e4a4eb8
 - filename: queue/src/config.ts
@@ -76,6 +74,8 @@ fileignoreconfig:
   checksum: ce84d57e61bfdf4168938e114113005b0961659ea299a3871415ade48ba7b310
 - filename: server/src/index.ts
   checksum: 88aec4113a92083b1401a101a673a2cf4457345c258bb8572dccbc713d22a94c
+- filename: server/src/infra/database/scripts/002-load-establishments_direction_territoriale.sql
+  checksum: 875d4c93a998a6ed9c35a905816b599d086da095406c0e282474e8239a7efc94
 - filename: server/src/infra/database/seeds/development/20240404235457_users.ts
   checksum: f3fabfbe62c13a614236fd7c9474c78de726c44515e8f116f3532765907c218c
 - filename: server/src/routers/unprotected.ts
@@ -91,3 +91,9 @@ scopeconfig:
 allowed_patterns:
 - key=\{.+\}
 version: "1.0"
+9978f136699921498fe90d02c495d4c1d27f47c96e64b646b219f29204c1
+scopeconfig:
+- scope: node
+allowed_patterns:
+- key=\{.+\}
+version: "1.0"

analytics/dagster/.nux/nux.yaml

@@ -0,0 +1 @@
+seen: 1

analytics/dagster/Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+
+RUN pip install dagster-webserver dagster-postgres dagster-aws
+
+RUN apt-get update && apt-get install -y nginx apache2-utils
+
+RUN ls 
+# Install dependencies
+COPY requirements.txt .
+
+
+RUN pip install -r requirements.txt
+
+ENV DAGSTER_HOME=/opt/dagster/dagster_home/
+
+RUN mkdir -p $DAGSTER_HOME
+
+COPY dagster.yaml workspace.yaml $DAGSTER_HOME
+
+COPY src/ $DAGSTER_HOME
+
+WORKDIR $DAGSTER_HOME
+
+# Setup Nginx configuration
+RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv
+COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default
+
+# Expose port 80 for the Nginx web server
+EXPOSE 8080
+
+CMD nginx -g 'daemon off;' & dagster-webserver -h 0.0.0.0 -p 3000
+

analytics/dagster/dagster.yaml

@@ -0,0 +1,16 @@
+storage:
+  postgres:
+    postgres_db:
+      username:
+        env: DAGSTER_PG_USERNAME
+      password:
+        env: DAGSTER_PG_PASSWORD
+      hostname:
+        env: DAGSTER_PG_HOST
+      db_name:
+        env: DAGSTER_PG_DB
+      port: 
+        env: DAGSTER_PG_PORT
+
+telemetry:
+  enabled: false

analytics/dagster/data/most_frequent_words.json

@@ -0,0 +1 @@
+{"\u2013": 9, "new": 6, "hn": 5, "why": 5, "ai": 4, "show": 4, "from": 4, "macos": 3, "language": 3, "server": 3, "at": 3, "video": 3, "may": 3, "be": 3, "using": 3, "pdf": 3, "vision": 2, "sequoia": 2, "virtual": 2, "arm": 2, "exploring": 2, "apple's": 2, "models": 2, "just": 2, "fast": 2}