OONI Run v2 refactor

.github/workflows/build_dataapi.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - oonirunv2
     paths:
       - "api/fastapi/**"
       - ".github/workflows/build_dataapi.yml"
@@ -13,8 +14,28 @@ env:
   IMAGE_NAME: ooni/dataapi
 
 jobs:
+  test:
+    uses: ./.github/workflows/test_dataapi.yml
+
   build_and_push:
+    name: Build and push
+    needs: [test]
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: oonipg
+          POSTGRES_PASSWORD: oonipg
+          POSTGRES_DB: oonipg
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -36,6 +57,7 @@ jobs:
           echo "version_number=$VERSION_NUMBER" >> "$GITHUB_OUTPUT"
 
       - name: Build and Push Docker Image
+        id: dockerbuild
         env:
           DOCKERFILE_PATH: ${{ env.oonidataapi_dir }}
         run: |
@@ -44,43 +66,91 @@ jobs:
           TAG_BUILD_LABEL=$IMAGE_NAME:${{ steps.version.outputs.build_label }}
           TAG_VERSION=$IMAGE_NAME:v${{ steps.version.outputs.version_number }}
 
+          echo "tag_latest=$TAG_LATEST" >> $GITHUB_OUTPUT
+          echo "tag_environment=$TAG_ENVIRONMENT" >> $GITHUB_OUTPUT
+          echo "tag_build_label=$TAG_BUILD_LABEL" >> $GITHUB_OUTPUT
+          echo "tag_version=$TAG_VERSION" >> $GITHUB_OUTPUT
+
           # Build Docker image with multiple tags
           docker build --build-arg BUILD_LABEL=${{ steps.version.outputs.build_label }} \
                        -t $TAG_BUILD_LABEL \
                        -t $TAG_ENVIRONMENT \
                        -t $TAG_LATEST \
                        -t $TAG_VERSION \
                        $DOCKERFILE_PATH
+      # Setup python
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
 
-          # Push all tags
-          docker push $TAG_BUILD_LABEL
-          docker push $TAG_ENVIRONMENT
-          docker push $TAG_LATEST
-          docker push $TAG_VERSION
+      - name: Install poetry
+        run: |
+          curl -fsS https://install.python-poetry.org | python - --preview -y
 
-      - name: Checkout ooni/devops
-        uses: actions/checkout@v2
+      - name: Add poetry to PATH
+        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Set up poetry cache
+        uses: actions/cache@v3
         with:
-          repository: "ooni/devops" # Replace with your repository's name
-          ssh-key: ${{ secrets.OONI_DEVOPS_DEPLOYKEY }}
-          path: "ooni-devops"
-          ref: "tf-actions"
+          path: "$HOME/.cache/pypoetry/virtualenvs"
+          key: venv-${{ runner.os }}-${{ hashFiles('**/api/fastapi/poetry.lock') }}
 
-      - name: Bump version of dataapi
-        run: |
-          jq --arg value "v${{ steps.version.outputs.version_number }}" \
-              '(.ooni_service_config.dataapi_version) = $value' \
-              ${BASE_DIR}/terraform.tfvars.json > ${BASE_DIR}/terraform.tfvars.json.tmp \
-          && mv ${BASE_DIR}/terraform.tfvars.json.tmp ${BASE_DIR}/terraform.tfvars.json
+      - name: Install dependencies
+        run: poetry install
+        working-directory: ./api/fastapi/
+
+      # Configure database and docker
+      - name: Run alembic migrations
         env:
-          BASE_DIR: "ooni-devops/tf/environments/production"
+          OONI_PG_PASSWORD: oonipg
+          OONI_PG_HOST: localhost
+        run: poetry run alembic upgrade head
+        working-directory: ./api/fastapi/oonidataapi/
+
+      - name: Start Docker container with PostgreSQL
+        run: |
+          docker run -d --name oonidataapi -p 8000:80 \
+            -e POSTGRESQL_URL="postgresql://oonipg:oonipg@localhost/oonipg" \
+            ${{ steps.dockerbuild.outputs.tag_version }}
+
+      # Run smoke test
+      #- name: Run smoketest against the built docker image
+      #  run: poetry run python oonidataapi/tests/run_smoketest.py --backend-base-url=http://localhost:8000/
+      #  working-directory: ./api/fastapi/
 
-      - name: Commit changes
-        id: commit
+      - name: Push docker tags
         run: |
-          cd ooni-devops
-          git config --global user.email "[email protected]"
-          git config --global user.name "OONI Github Actions Bot"
-          git add .
-          git commit -m "auto: update oonidataapi package version to v${{ steps.version.outputs.version_number }}" || echo "No changes to commit"
-          git push origin
+          # Push all tags
+          docker push ${{ steps.dockerbuild.outputs.tag_latest }}
+          docker push ${{ steps.dockerbuild.outputs.tag_environment }}
+          docker push ${{ steps.dockerbuild.outputs.tag_build_label }}
+          docker push ${{ steps.dockerbuild.outputs.tag_version }}
+
+      #- name: Checkout ooni/devops
+      #  uses: actions/checkout@v2
+      #  with:
+      #    repository: "ooni/devops" # Replace with your repository's name
+      #    ssh-key: ${{ secrets.OONI_DEVOPS_DEPLOYKEY }}
+      #    path: "ooni-devops"
+
+      #- name: Bump version of dataapi
+      #  run: |
+      #    jq --arg value "v${{ steps.version.outputs.version_number }}" \
+      #        '(.ooni_service_config.dataapi_version) = $value' \
+      #        ${BASE_DIR}/terraform.tfvars.json > ${BASE_DIR}/terraform.tfvars.json.tmp \
+      #    && mv ${BASE_DIR}/terraform.tfvars.json.tmp ${BASE_DIR}/terraform.tfvars.json
+      #  env:
+      #    BASE_DIR: "ooni-devops/tf/environments/prod"
+
+      #- name: Commit changes
+      #  id: commit
+      #  run: |
+      #    cd ooni-devops
+      #    git config --global user.email "[email protected]"
+      #    git config --global user.name "OONI Github Actions Bot"
+      #    git checkout -b bump-api
+      #    git add .
+      #    git commit -m "auto: update oonidataapi package version to v${{ steps.version.outputs.version_number }}" || echo "No changes to commit"
+      #    git push origin

.github/workflows/test_dataapi.yml

@@ -0,0 +1,42 @@
+name: Tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - "*"
+  workflow_call:
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install poetry
+        run: |
+          curl -fsS https://install.python-poetry.org | python - --preview -y
+
+      - name: Add poetry to PATH
+        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Set up poetry cache
+        uses: actions/cache@v3
+        with:
+          path: "$HOME/.cache/pypoetry/virtualenvs"
+          key: venv-${{ runner.os }}-${{ hashFiles('**/api/fastapi/poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+        working-directory: ./api/fastapi/
+
+      - name: Run all tests
+        env:
+          PYTHONUNBUFFERED: "1"
+        run: poetry run pytest -s --full-trace --log-level=INFO  --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=term oonidataapi/tests
+        working-directory: ./api/fastapi/

.github/workflows/test_new_api.yml → .github/workflows/test_legacy_ooniapi.py

@@ -1,4 +1,4 @@
-name: Test API
+name: Test Legacy API
 on:
   pull_request:
   workflow_dispatch:
@@ -9,6 +9,29 @@
         default: false
 
 jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    container: debian:11
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Setup APT
+        run: |
+          apt-get update
+          apt-get install --no-install-recommends -y ca-certificates gnupg
+          echo "deb http://deb-ci.ooni.org unstable main" >> /etc/apt/sources.list
+          apt-key adv --verbose --keyserver hkp://keyserver.ubuntu.com --recv-keys "B5A08F01796E7F521861B449372D1FF271F2DD50"
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install --no-install-recommends -qy mypy
+
+      - name: Run tests
+        # see the mypy.ini file
+        run: cd api && mypy **/*.py
+
   integration_test:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -17,3 +17,4 @@ af/oometa/tsvdump.dir/
 .tox/
 af/fastpath/var/
 af/fastpath/etc/
+.coverage

api/fastapi/Readme.md

@@ -1,6 +1,18 @@
-Running:
+## OONI Data API
+
+Setup:
 
 ```
 poetry install
+```
+
+To run tests
+
+```
+poetry run pytests oonidataapi/tests
+```
+
+To run the backend:
+```
 poetry run uvicorn oonidataapi.main:app
 ```

api/fastapi/buildspec.yml

@@ -0,0 +1,55 @@
+version: 0.2
+env:
+  variables:
+    OONI_CODE_PATH: api/fastapi/
+    IMAGE_NAME: ooni/dataapi
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+    commands:
+      - echo "Installing Poetry"
+      - curl -fsS https://install.python-poetry.org | python - --preview -y
+      - export PATH="$HOME/.local/bin:$PATH"
+
+  pre_build:
+    commands:
+      - aws --version
+      - echo "Logging in to ECR"
+      - aws secretsmanager get-secret-value --secret-id DOCKER_HUB_PASSWORD --query SecretString --output text | docker login --username ooni --password-stdin
+      - echo "Formatting version information..."
+      - DATE=$(date +'%Y%m%d')
+      - SHORT_SHA=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c1-8)
+      - BUILD_LABEL="${DATE}-${SHORT_SHA}"
+      - VERSION_NUMBER=$(cat your_project_dir/pyproject.toml | grep 'version =' | awk -F '"' '{print $2}')
+
+  build:
+    commands:
+      - PROJECT_ROOT=$(pwd)
+      - cd $OONI_CODE_PATH
+      - echo "Installing project dependencies with poetry..."
+      - poetry install --no-root
+      - poetry run pytest -s --full-trace --log-level=INFO  --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=term oonidataapi/tests
+
+      - echo "Building and tagging Docker image..."
+      - |
+        docker build --build-arg BUILD_LABEL=${BUILD_LABEL} \
+                     -t $IMAGE_NAME:$BUILD_LABEL \
+                     -t $IMAGE_NAME:production \
+                     -t $IMAGE_NAME:latest \
+                     -t $IMAGE_NAME:v$VERSION_NUMBER \
+                     .
+
+  post_build:
+    commands:
+      - echo "Build complete at $(date)"
+      - echo "Pushing Docker images..."
+      - docker push $IMAGE_NAME:$BUILD_LABEL
+      - docker push $IMAGE_NAME:production
+      - docker push $IMAGE_NAME:latest
+      - docker push $IMAGE_NAME:v$VERSION_NUMBER
+      - printf '[{"name":"ooni_dataapi","imageUri":"%s"}]' $IMAGE_NAME:$BUILD_LABEL > ${PROJECT_ROOT}/imagedefinitions.json
+
+artifacts:
+  files: imagedefinitions.json