From 68c8063e5bfccf2ff4c9d31495ca549729cdc0e8 Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Thu, 30 Jan 2025 15:56:15 +0100
Subject: [PATCH 1/7] add script to CI to check ecobalyse-data sync

---
 .github/workflows/node.js.yml |  3 ++
 check-ecobalyse-data-sync.sh  | 62 +++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100755 check-ecobalyse-data-sync.sh

diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
index 1f63099f9..b3f2a374f 100644
--- a/.github/workflows/node.js.yml
+++ b/.github/workflows/node.js.yml
@@ -63,6 +63,9 @@ jobs:
     - name: Check JSON db formating
       run: npm run db:validate
 
+    - name: Check synchronization with ecobalyse-data for generated JSON
+      run: ./scripts/check-ecobalyse-data-sync.sh
+
     - name: Install Python dependencies
       run: pip install pipenv && pipenv install -d
 
diff --git a/check-ecobalyse-data-sync.sh b/check-ecobalyse-data-sync.sh
new file mode 100755
index 000000000..381d83d69
--- /dev/null
+++ b/check-ecobalyse-data-sync.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+set -euo pipefail
+
+# Directory where ecobalyse-data will be cloned
+TEMP_DIR=$(mktemp -d)
+ECOBALYSE_DATA_REPO="https://github.com/MTES-MCT/ecobalyse-data.git"
+ECOBALYSE_DATA_BRANCH="${ECOBALYSE_DATA_BRANCH:-main}"
+
+# Cleanup function
+cleanup() {
+    rm -rf "$TEMP_DIR"
+}
+trap cleanup EXIT
+
+# Clone ecobalyse-data repository
+echo "Cloning ecobalyse-data repository (branch: ${ECOBALYSE_DATA_BRANCH})..."
+git clone -q -b "${ECOBALYSE_DATA_BRANCH}" "$ECOBALYSE_DATA_REPO" "$TEMP_DIR"
+
+# Files to check
+FILES=(
+    "public/data/food/ingredients.json"
+    "public/data/food/processes.json"
+    "public/data/textile/materials.json"
+    "public/data/textile/processes.json"
+    "public/data/object/processes.json"
+)
+
+# Compare specified JSON files
+echo "Comparing JSON files between ecobalyse-data and ecobalyse repositories..."
+DIFFERENCES_FOUND=0
+
+for file in "${FILES[@]}"; do
+    if [ -f "$file" ]; then
+        other_file="$TEMP_DIR/$file"
+
+        if [ -f "$other_file" ]; then
+            # Compare files and store the diff output
+            diff_output=$(diff -u "$file" "$other_file" || true)
+            if [ -n "$diff_output" ]; then
+                echo "--❌ $file is different:"
+                echo "$diff_output"
+                DIFFERENCES_FOUND=1
+            else
+                echo "--✅ $file is synchronized"
+            fi
+        else
+            echo "--⚠️  $file does not exist in ecobalyse-data repository"
+            DIFFERENCES_FOUND=1
+        fi
+    else
+        echo "--⚠️  $file does not exist in ecobalyse repository"
+        DIFFERENCES_FOUND=1
+    fi
+done
+
+if [ $DIFFERENCES_FOUND -eq 1 ]; then
+    echo "❌ Differences found between repositories ecobalyse-data and ecobalyse"
+    exit 1
+else
+    echo "✅ All JSON files are synchronized"
+    exit 0
+fi

From c64c88473c853628caafb0067e18d7a726c081db Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Thu, 30 Jan 2025 16:12:20 +0100
Subject: [PATCH 2/7] run sync check only on PR

---
 .github/workflows/node.js.yml | 12 +++++++++---
 check-ecobalyse-data-sync.sh  |  2 --
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
index b3f2a374f..b3f02629c 100644
--- a/.github/workflows/node.js.yml
+++ b/.github/workflows/node.js.yml
@@ -63,9 +63,6 @@ jobs:
     - name: Check JSON db formating
       run: npm run db:validate
 
-    - name: Check synchronization with ecobalyse-data for generated JSON
-      run: ./scripts/check-ecobalyse-data-sync.sh
-
     - name: Install Python dependencies
       run: pip install pipenv && pipenv install -d
 
@@ -88,3 +85,12 @@ jobs:
 
     - name: Run server tests
       run: pipenv run backend/update.sh && npm run test:server-ci && npm run test:backend
+
+  check-ecobalyse-data-sync:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Check synchronization with ecobalyse-data for generated JSON
+      run: ./scripts/check-ecobalyse-data-sync.sh
diff --git a/check-ecobalyse-data-sync.sh b/check-ecobalyse-data-sync.sh
index 381d83d69..f9b1e51ed 100755
--- a/check-ecobalyse-data-sync.sh
+++ b/check-ecobalyse-data-sync.sh
@@ -12,7 +12,6 @@ cleanup() {
 }
 trap cleanup EXIT
 
-# Clone ecobalyse-data repository
 echo "Cloning ecobalyse-data repository (branch: ${ECOBALYSE_DATA_BRANCH})..."
 git clone -q -b "${ECOBALYSE_DATA_BRANCH}" "$ECOBALYSE_DATA_REPO" "$TEMP_DIR"
 
@@ -25,7 +24,6 @@ FILES=(
     "public/data/object/processes.json"
 )
 
-# Compare specified JSON files
 echo "Comparing JSON files between ecobalyse-data and ecobalyse repositories..."
 DIFFERENCES_FOUND=0
 

From 2826f22996e5aa460847ad6bb6d6d4d3ff78a06b Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Thu, 30 Jan 2025 16:17:03 +0100
Subject: [PATCH 3/7] update path to script

---
 .github/workflows/node.js.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
index b3f02629c..4469320d7 100644
--- a/.github/workflows/node.js.yml
+++ b/.github/workflows/node.js.yml
@@ -93,4 +93,4 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Check synchronization with ecobalyse-data for generated JSON
-      run: ./scripts/check-ecobalyse-data-sync.sh
+      run: ./check-ecobalyse-data-sync.sh

From d46b0dff1f782f197a7310de0cc06a3781b59095 Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Mon, 3 Feb 2025 14:55:36 +0100
Subject: [PATCH 4/7] use wget instead of git clone

---
 check-ecobalyse-data-sync.sh | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/check-ecobalyse-data-sync.sh b/check-ecobalyse-data-sync.sh
index f9b1e51ed..55cffe648 100755
--- a/check-ecobalyse-data-sync.sh
+++ b/check-ecobalyse-data-sync.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 set -euo pipefail
 
-# Directory where ecobalyse-data will be cloned
+# Directory where files will be downloaded
 TEMP_DIR=$(mktemp -d)
-ECOBALYSE_DATA_REPO="https://github.com/MTES-MCT/ecobalyse-data.git"
 ECOBALYSE_DATA_BRANCH="${ECOBALYSE_DATA_BRANCH:-main}"
+RAW_GITHUB_URL="https://raw.githubusercontent.com/MTES-MCT/ecobalyse-data/${ECOBALYSE_DATA_BRANCH}"
 
 # Cleanup function
 cleanup() {
@@ -12,9 +12,6 @@ cleanup() {
 }
 trap cleanup EXIT
 
-echo "Cloning ecobalyse-data repository (branch: ${ECOBALYSE_DATA_BRANCH})..."
-git clone -q -b "${ECOBALYSE_DATA_BRANCH}" "$ECOBALYSE_DATA_REPO" "$TEMP_DIR"
-
 # Files to check
 FILES=(
     "public/data/food/ingredients.json"
@@ -24,6 +21,22 @@ FILES=(
     "public/data/object/processes.json"
 )
 
+echo "Downloading files from ecobalyse-data repository (branch: ${ECOBALYSE_DATA_BRANCH})..."
+
+# Create necessary directories in TEMP_DIR
+for file in "${FILES[@]}"; do
+    mkdir -p "$TEMP_DIR/$(dirname "$file")"
+done
+
+# Download each file
+for file in "${FILES[@]}"; do
+    wget -q "$RAW_GITHUB_URL/$file" -O "$TEMP_DIR/$file" || {
+        echo "--⚠️  Failed to download $file from ecobalyse-data repository"
+        DIFFERENCES_FOUND=1
+        continue
+    }
+done
+
 echo "Comparing JSON files between ecobalyse-data and ecobalyse repositories..."
 DIFFERENCES_FOUND=0
 

From 047da31d3d40db8dd335fcdb3951659fb6d1a9c2 Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Mon, 3 Feb 2025 15:01:13 +0100
Subject: [PATCH 5/7] separate into different worfklow

---
 .github/workflows/ecobalyse-data-sync.yml | 20 ++++++++++++++++++++
 .github/workflows/node.js.yml             |  9 ---------
 2 files changed, 20 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/ecobalyse-data-sync.yml

diff --git a/.github/workflows/ecobalyse-data-sync.yml b/.github/workflows/ecobalyse-data-sync.yml
new file mode 100644
index 000000000..64af1f92f
--- /dev/null
+++ b/.github/workflows/ecobalyse-data-sync.yml
@@ -0,0 +1,20 @@
+name: ecobalyse-data sync
+
+on:
+  pull_request:
+    branches: [ master, staging ]
+  workflow_dispatch:
+
+jobs:
+  check-ecobalyse-data-sync:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install wget
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y wget
+
+    - name: Check synchronization with ecobalyse-data for generated JSON
+      run: ./check-ecobalyse-data-sync.sh
diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
index 4469320d7..1f63099f9 100644
--- a/.github/workflows/node.js.yml
+++ b/.github/workflows/node.js.yml
@@ -85,12 +85,3 @@ jobs:
 
     - name: Run server tests
       run: pipenv run backend/update.sh && npm run test:server-ci && npm run test:backend
-
-  check-ecobalyse-data-sync:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Check synchronization with ecobalyse-data for generated JSON
-      run: ./check-ecobalyse-data-sync.sh

From 96b0ddb51e3709fe64bed061264d02ef09b141a1 Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Tue, 4 Feb 2025 11:18:08 +0100
Subject: [PATCH 6/7] suggested improvements following review

---
 .github/workflows/ecobalyse-data-sync.yml                 | 5 -----
 .../check-ecobalyse-data-sync.sh                          | 8 ++++----
 2 files changed, 4 insertions(+), 9 deletions(-)
 rename check-ecobalyse-data-sync.sh => bin/check-ecobalyse-data-sync.sh (89%)

diff --git a/.github/workflows/ecobalyse-data-sync.yml b/.github/workflows/ecobalyse-data-sync.yml
index 64af1f92f..c0446c198 100644
--- a/.github/workflows/ecobalyse-data-sync.yml
+++ b/.github/workflows/ecobalyse-data-sync.yml
@@ -11,10 +11,5 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Install wget
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y wget
-
     - name: Check synchronization with ecobalyse-data for generated JSON
       run: ./check-ecobalyse-data-sync.sh
diff --git a/check-ecobalyse-data-sync.sh b/bin/check-ecobalyse-data-sync.sh
similarity index 89%
rename from check-ecobalyse-data-sync.sh
rename to bin/check-ecobalyse-data-sync.sh
index 55cffe648..f8705316f 100755
--- a/check-ecobalyse-data-sync.sh
+++ b/bin/check-ecobalyse-data-sync.sh
@@ -4,7 +4,9 @@ set -euo pipefail
 # Directory where files will be downloaded
 TEMP_DIR=$(mktemp -d)
 ECOBALYSE_DATA_BRANCH="${ECOBALYSE_DATA_BRANCH:-main}"
-RAW_GITHUB_URL="https://raw.githubusercontent.com/MTES-MCT/ecobalyse-data/${ECOBALYSE_DATA_BRANCH}"
+ECOBALYSE_DATA_REPO="${ECOBALYSE_DATA_REPO:-MTES-MCT/ecobalyse-data}"
+RAW_GITHUB_URL="https://raw.githubusercontent.com/${ECOBALYSE_DATA_REPO}/${ECOBALYSE_DATA_BRANCH}"
+DIFFERENCES_FOUND=0
 
 # Cleanup function
 cleanup() {
@@ -30,15 +32,13 @@ done
 
 # Download each file
 for file in "${FILES[@]}"; do
-    wget -q "$RAW_GITHUB_URL/$file" -O "$TEMP_DIR/$file" || {
+    curl -s "$RAW_GITHUB_URL/$file" -o "$TEMP_DIR/$file" || {
         echo "--⚠️  Failed to download $file from ecobalyse-data repository"
-        DIFFERENCES_FOUND=1
         continue
     }
 done
 
 echo "Comparing JSON files between ecobalyse-data and ecobalyse repositories..."
-DIFFERENCES_FOUND=0
 
 for file in "${FILES[@]}"; do
     if [ -f "$file" ]; then

From cfb73b463d1793d53a9c72a001438f5f2803d901 Mon Sep 17 00:00:00 2001
From: paulboosz <paulboosz@gmail.com>
Date: Tue, 4 Feb 2025 11:19:53 +0100
Subject: [PATCH 7/7] fix path to bash script

---
 .github/workflows/ecobalyse-data-sync.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ecobalyse-data-sync.yml b/.github/workflows/ecobalyse-data-sync.yml
index c0446c198..f54c49260 100644
--- a/.github/workflows/ecobalyse-data-sync.yml
+++ b/.github/workflows/ecobalyse-data-sync.yml
@@ -12,4 +12,4 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Check synchronization with ecobalyse-data for generated JSON
-      run: ./check-ecobalyse-data-sync.sh
+      run: ./bin/check-ecobalyse-data-sync.sh