OONI Data, OONI Pipeline split

.github/workflows/test_oonidata.yml

@@ -0,0 +1,48 @@
+name: test oonidata
+on: push
+jobs:
+  run_tests:
+    name: ${{ matrix.os }} / ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      matrix:
+        os: [Ubuntu]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Set up datadir cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/datadir/
+          key: oonidata-tests-data-datadir
+
+      - name: Set up measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/measurements/
+          key: oonidata-tests-data-measurements
+
+      - name: Set up raw_measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/raw_measurements/
+          key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+      - name: Run all tests
+        run: hatch run cov
+        working-directory: ./oonidata/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonidata
+          working-directory: ./oonidata/

.github/workflows/test_oonipipeline.yml

@@ -0,0 +1,55 @@
+name: test oonipipeline
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      # Contrary to oonidata, which is a tool for end users, oonipipeline
+      # targets a single python version to minimize the likelyhood of breakage
+      # in production.
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Set up datadir cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/datadir/
+          key: oonipipeline-tests-data-datadir
+
+      - name: Set up measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/measurements/
+          key: oonipipeline-tests-data-measurements
+
+      - name: Set up raw_measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/raw_measurements/
+          key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+      - name: Install clickhouse
+        run: |
+          sudo apt-get install -y apt-transport-https ca-certificates dirmngr
+          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
+          echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
+              /etc/apt/sources.list.d/clickhouse.list
+          sudo apt-get update
+          sudo apt-get install -y clickhouse-server clickhouse-client
+
+      - name: Run all tests
+        run: hatch run cov
+        working-directory: ./oonipipeline/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonipipeline
+          working-directory: ./oonipipeline/

.gitignore

@@ -1,12 +1,11 @@
 __pycache__
-/.coverage
-/coverage.xml
-/tests/data/datadir/*
-/tests/data/raw_measurements/*
-/tests/data/measurements/*
+.coverage
+.coverage.*
+coverage.xml
+
+.ipynb_checkpoints/
+
 /dist
-/datadir
 /output
 /attic
 /prof
-.ipynb_checkpoints/

.vscode/settings.json

@@ -3,6 +3,7 @@
   "python.testing.pytestArgs": ["tests"],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true,
+  "python.defaultInterpreterPath": "${workspaceFolder}/oonipipeline/.venv",
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   }