diff --git a/.github/workflows/build_dataapi.yml b/.github/workflows/build_dataapi.yml index 16fabd63..848d8032 100644 --- a/.github/workflows/build_dataapi.yml +++ b/.github/workflows/build_dataapi.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - oonirunv2 paths: - "api/fastapi/**" - ".github/workflows/build_dataapi.yml" @@ -13,8 +14,28 @@ env: IMAGE_NAME: ooni/dataapi jobs: + test: + uses: ./.github/workflows/test_dataapi.yml + build_and_push: + name: Build and push + needs: [test] runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_USER: oonipg + POSTGRES_PASSWORD: oonipg + POSTGRES_DB: oonipg + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Checkout Repository uses: actions/checkout@v2 @@ -36,6 +57,7 @@ jobs: echo "version_number=$VERSION_NUMBER" >> "$GITHUB_OUTPUT" - name: Build and Push Docker Image + id: dockerbuild env: DOCKERFILE_PATH: ${{ env.oonidataapi_dir }} run: | @@ -44,6 +66,11 @@ jobs: TAG_BUILD_LABEL=$IMAGE_NAME:${{ steps.version.outputs.build_label }} TAG_VERSION=$IMAGE_NAME:v${{ steps.version.outputs.version_number }} + echo "tag_latest=$TAG_LATEST" >> $GITHUB_OUTPUT + echo "tag_environment=$TAG_ENVIRONMENT" >> $GITHUB_OUTPUT + echo "tag_build_label=$TAG_BUILD_LABEL" >> $GITHUB_OUTPUT + echo "tag_version=$TAG_VERSION" >> $GITHUB_OUTPUT + # Build Docker image with multiple tags docker build --build-arg BUILD_LABEL=${{ steps.version.outputs.build_label }} \ -t $TAG_BUILD_LABEL \ @@ -51,36 +78,79 @@ jobs: -t $TAG_LATEST \ -t $TAG_VERSION \ $DOCKERFILE_PATH + # Setup python + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 - # Push all tags - docker push $TAG_BUILD_LABEL - docker push $TAG_ENVIRONMENT - docker push $TAG_LATEST - docker push $TAG_VERSION + - name: Install poetry + run: | + curl -fsS https://install.python-poetry.org | python - --preview -y - - name: Checkout ooni/devops - uses: actions/checkout@v2 + - name: Add poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Set up poetry cache + uses: actions/cache@v3 with: - repository: "ooni/devops" # Replace with your repository's name - ssh-key: ${{ secrets.OONI_DEVOPS_DEPLOYKEY }} - path: "ooni-devops" - ref: "tf-actions" + path: "$HOME/.cache/pypoetry/virtualenvs" + key: venv-${{ runner.os }}-${{ hashFiles('**/api/fastapi/poetry.lock') }} - - name: Bump version of dataapi - run: | - jq --arg value "v${{ steps.version.outputs.version_number }}" \ - '(.ooni_service_config.dataapi_version) = $value' \ - ${BASE_DIR}/terraform.tfvars.json > ${BASE_DIR}/terraform.tfvars.json.tmp \ - && mv ${BASE_DIR}/terraform.tfvars.json.tmp ${BASE_DIR}/terraform.tfvars.json + - name: Install dependencies + run: poetry install + working-directory: ./api/fastapi/ + + # Configure database and docker + - name: Run alembic migrations env: - BASE_DIR: "ooni-devops/tf/environments/production" + OONI_PG_PASSWORD: oonipg + OONI_PG_HOST: localhost + run: poetry run alembic upgrade head + working-directory: ./api/fastapi/oonidataapi/ + + - name: Start Docker container with PostgreSQL + run: | + docker run -d --name oonidataapi -p 8000:80 \ + -e POSTGRESQL_URL="postgresql://oonipg:oonipg@localhost/oonipg" \ + ${{ steps.dockerbuild.outputs.tag_version }} + + # Run smoke test + #- name: Run smoketest against the built docker image + # run: poetry run python oonidataapi/tests/run_smoketest.py --backend-base-url=http://localhost:8000/ + # working-directory: ./api/fastapi/ - - name: Commit changes - id: commit + - name: Push docker tags run: | - cd ooni-devops - git config --global user.email "nothuman@ooni.org" - git config --global user.name "OONI Github Actions Bot" - git add . - git commit -m "auto: update oonidataapi package version to v${{ steps.version.outputs.version_number }}" || echo "No changes to commit" - git push origin + # Push all tags + docker push ${{ steps.dockerbuild.outputs.tag_latest }} + docker push ${{ steps.dockerbuild.outputs.tag_environment }} + docker push ${{ steps.dockerbuild.outputs.tag_build_label }} + docker push ${{ steps.dockerbuild.outputs.tag_version }} + + #- name: Checkout ooni/devops + # uses: actions/checkout@v2 + # with: + # repository: "ooni/devops" # Replace with your repository's name + # ssh-key: ${{ secrets.OONI_DEVOPS_DEPLOYKEY }} + # path: "ooni-devops" + + #- name: Bump version of dataapi + # run: | + # jq --arg value "v${{ steps.version.outputs.version_number }}" \ + # '(.ooni_service_config.dataapi_version) = $value' \ + # ${BASE_DIR}/terraform.tfvars.json > ${BASE_DIR}/terraform.tfvars.json.tmp \ + # && mv ${BASE_DIR}/terraform.tfvars.json.tmp ${BASE_DIR}/terraform.tfvars.json + # env: + # BASE_DIR: "ooni-devops/tf/environments/prod" + + #- name: Commit changes + # id: commit + # run: | + # cd ooni-devops + # git config --global user.email "nothuman@ooni.org" + # git config --global user.name "OONI Github Actions Bot" + # git checkout -b bump-api + # git add . + # git commit -m "auto: update oonidataapi package version to v${{ steps.version.outputs.version_number }}" || echo "No changes to commit" + # git push origin diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml deleted file mode 100644 index e851f5a9..00000000 --- a/.github/workflows/mypy.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Run mypy on API -on: - pull_request: - paths: - - 'api/**' - -jobs: - test: - runs-on: ubuntu-latest - container: debian:11 - - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - - name: Setup APT - run: | - apt-get update - apt-get install --no-install-recommends -y ca-certificates gnupg - echo "deb http://deb-ci.ooni.org unstable main" >> /etc/apt/sources.list - apt-key adv --verbose --keyserver hkp://keyserver.ubuntu.com --recv-keys "B5A08F01796E7F521861B449372D1FF271F2DD50" - - - name: Install dependencies - run: | - apt-get update - apt-get install --no-install-recommends -qy mypy - - - name: Run tests - # see the mypy.ini file - run: cd api && mypy **/*.py diff --git a/.github/workflows/test_dataapi.yml b/.github/workflows/test_dataapi.yml new file mode 100644 index 00000000..ea95c5a7 --- /dev/null +++ b/.github/workflows/test_dataapi.yml @@ -0,0 +1,42 @@ +name: Tests +on: + push: + branches: + - main + pull_request: + branches: + - "*" + workflow_call: +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install poetry + run: | + curl -fsS https://install.python-poetry.org | python - --preview -y + + - name: Add poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Set up poetry cache + uses: actions/cache@v3 + with: + path: "$HOME/.cache/pypoetry/virtualenvs" + key: venv-${{ runner.os }}-${{ hashFiles('**/api/fastapi/poetry.lock') }} + + - name: Install dependencies + run: poetry install + working-directory: ./api/fastapi/ + + - name: Run all tests + env: + PYTHONUNBUFFERED: "1" + run: poetry run pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=term oonidataapi/tests + working-directory: ./api/fastapi/ diff --git a/.github/workflows/test_new_api.yml b/.github/workflows/test_legacy_ooniapi.py similarity index 70% rename from .github/workflows/test_new_api.yml rename to .github/workflows/test_legacy_ooniapi.py index 69757324..a355a0b6 100644 --- a/.github/workflows/test_new_api.yml +++ b/.github/workflows/test_legacy_ooniapi.py @@ -1,4 +1,4 @@ -name: Test API +name: Test Legacy API on: pull_request: workflow_dispatch: @@ -9,6 +9,29 @@ default: false jobs: + mypy: + runs-on: ubuntu-latest + container: debian:11 + steps: + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Setup APT + run: | + apt-get update + apt-get install --no-install-recommends -y ca-certificates gnupg + echo "deb http://deb-ci.ooni.org unstable main" >> /etc/apt/sources.list + apt-key adv --verbose --keyserver hkp://keyserver.ubuntu.com --recv-keys "B5A08F01796E7F521861B449372D1FF271F2DD50" + + - name: Install dependencies + run: | + apt-get update + apt-get install --no-install-recommends -qy mypy + + - name: Run tests + # see the mypy.ini file + run: cd api && mypy **/*.py + integration_test: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index b3c2cc7a..a572b9f6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ af/oometa/tsvdump.dir/ .tox/ af/fastpath/var/ af/fastpath/etc/ +.coverage diff --git a/api/fastapi/Readme.md b/api/fastapi/Readme.md index 19bdf4d7..15dc9f45 100644 --- a/api/fastapi/Readme.md +++ b/api/fastapi/Readme.md @@ -1,6 +1,18 @@ -Running: +## OONI Data API + +Setup: ``` poetry install +``` + +To run tests + +``` +poetry run pytests oonidataapi/tests +``` + +To run the backend: +``` poetry run uvicorn oonidataapi.main:app ``` diff --git a/api/fastapi/buildspec.yml b/api/fastapi/buildspec.yml new file mode 100644 index 00000000..23a02cf7 --- /dev/null +++ b/api/fastapi/buildspec.yml @@ -0,0 +1,55 @@ +version: 0.2 +env: + variables: + OONI_CODE_PATH: api/fastapi/ + IMAGE_NAME: ooni/dataapi + +phases: + install: + runtime-versions: + python: 3.11 + commands: + - echo "Installing Poetry" + - curl -fsS https://install.python-poetry.org | python - --preview -y + - export PATH="$HOME/.local/bin:$PATH" + + pre_build: + commands: + - aws --version + - echo "Logging in to ECR" + - aws secretsmanager get-secret-value --secret-id DOCKER_HUB_PASSWORD --query SecretString --output text | docker login --username ooni --password-stdin + - echo "Formatting version information..." + - DATE=$(date +'%Y%m%d') + - SHORT_SHA=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c1-8) + - BUILD_LABEL="${DATE}-${SHORT_SHA}" + - VERSION_NUMBER=$(cat your_project_dir/pyproject.toml | grep 'version =' | awk -F '"' '{print $2}') + + build: + commands: + - PROJECT_ROOT=$(pwd) + - cd $OONI_CODE_PATH + - echo "Installing project dependencies with poetry..." + - poetry install --no-root + - poetry run pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=term oonidataapi/tests + + - echo "Building and tagging Docker image..." + - | + docker build --build-arg BUILD_LABEL=${BUILD_LABEL} \ + -t $IMAGE_NAME:$BUILD_LABEL \ + -t $IMAGE_NAME:production \ + -t $IMAGE_NAME:latest \ + -t $IMAGE_NAME:v$VERSION_NUMBER \ + . + + post_build: + commands: + - echo "Build complete at $(date)" + - echo "Pushing Docker images..." + - docker push $IMAGE_NAME:$BUILD_LABEL + - docker push $IMAGE_NAME:production + - docker push $IMAGE_NAME:latest + - docker push $IMAGE_NAME:v$VERSION_NUMBER + - printf '[{"name":"ooni_dataapi","imageUri":"%s"}]' $IMAGE_NAME:$BUILD_LABEL > ${PROJECT_ROOT}/imagedefinitions.json + +artifacts: + files: imagedefinitions.json diff --git a/api/fastapi/oonidataapi/alembic.ini b/api/fastapi/oonidataapi/alembic.ini new file mode 100644 index 00000000..ce134c2f --- /dev/null +++ b/api/fastapi/oonidataapi/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql://oonipg:%(OONI_PG_PASSWORD)s@%(OONI_PG_HOST)s/oonipg + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/api/fastapi/oonidataapi/alembic/Readme.md b/api/fastapi/oonidataapi/alembic/Readme.md new file mode 100644 index 00000000..ad30644d --- /dev/null +++ b/api/fastapi/oonidataapi/alembic/Readme.md @@ -0,0 +1,17 @@ +# Alembic database migrations + +When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. + +This is how you do it: + +1. Create the template migration script +``` +poetry run alembic revision -m "name of the revision" +``` +2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits +3. You can now run the migration like so: +``` +OONI_PG_PASSWORD=XXXX poetry run alembic upgrade head +``` + + diff --git a/api/fastapi/oonidataapi/alembic/env.py b/api/fastapi/oonidataapi/alembic/env.py new file mode 100644 index 00000000..a99ece64 --- /dev/null +++ b/api/fastapi/oonidataapi/alembic/env.py @@ -0,0 +1,88 @@ +import os + +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from oonidataapi import models + +target_metadata = models.Base.metadata + +section = config.config_ini_section +config.set_section_option( + section, "OONI_PG_PASSWORD", os.environ.get("OONI_PG_PASSWORD", "") +) +config.set_section_option( + section, "OONI_PG_HOST", os.environ.get("OONI_PG_HOST", "postgres.tier0.prod.ooni.nu") +) + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/api/fastapi/oonidataapi/alembic/script.py.mako b/api/fastapi/oonidataapi/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/api/fastapi/oonidataapi/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/api/fastapi/oonidataapi/alembic/versions/7d5841cb9549_make_oonirun_link_id_a_string.py b/api/fastapi/oonidataapi/alembic/versions/7d5841cb9549_make_oonirun_link_id_a_string.py new file mode 100644 index 00000000..f7dccf63 --- /dev/null +++ b/api/fastapi/oonidataapi/alembic/versions/7d5841cb9549_make_oonirun_link_id_a_string.py @@ -0,0 +1,37 @@ +"""make oonirun link id a string + +Revision ID: 7d5841cb9549 +Revises: 836b3451a168 +Create Date: 2024-02-28 15:41:53.811746 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "7d5841cb9549" +down_revision: Union[str, None] = "836b3451a168" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE oonirun + ALTER COLUMN oonirun_link_id TYPE TEXT USING oonirun_link_id::TEXT + """ + ) + + +def downgrade() -> None: + op.execute( + """ + ALTER TABLE oonirun + ALTER COLUMN oonirun TYPE INTEGER USING oonirun::INTEGER + """ + ) diff --git a/api/fastapi/oonidataapi/alembic/versions/836b3451a168_add_expiration_date_color_columns_drop_.py b/api/fastapi/oonidataapi/alembic/versions/836b3451a168_add_expiration_date_color_columns_drop_.py new file mode 100644 index 00000000..2ac09b5c --- /dev/null +++ b/api/fastapi/oonidataapi/alembic/versions/836b3451a168_add_expiration_date_color_columns_drop_.py @@ -0,0 +1,32 @@ +"""Add expiration_date, color columns. Drop is_archived column. + +Revision ID: 836b3451a168 +Revises: f96cf47f2791 +Create Date: 2024-02-27 09:44:26.833238 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "836b3451a168" +down_revision: Union[str, None] = "f96cf47f2791" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "oonirun", sa.Column("expiration_date", sa.DateTime(), nullable=False) + ) + op.add_column("oonirun", sa.Column("color", sa.String(), nullable=True)) + op.drop_column("oonirun", "is_archived") + + +def downgrade() -> None: + op.drop_column("oonirun", "expiration_date") + op.drop_column("oonirun", "color") diff --git a/api/fastapi/oonidataapi/alembic/versions/f96cf47f2791_create_oonirun_db.py b/api/fastapi/oonidataapi/alembic/versions/f96cf47f2791_create_oonirun_db.py new file mode 100644 index 00000000..5dcded74 --- /dev/null +++ b/api/fastapi/oonidataapi/alembic/versions/f96cf47f2791_create_oonirun_db.py @@ -0,0 +1,44 @@ +"""create oonirun db + +Revision ID: f96cf47f2791 +Revises: +Create Date: 2024-02-15 14:39:47.867136 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "f96cf47f2791" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "oonirun", + sa.Column("oonirun_link_id", sa.Integer, primary_key=True), + sa.Column("revision", sa.Integer(), nullable=False, primary_key=True), + sa.Column("date_created", sa.DateTime(), nullable=False), + sa.Column("date_updated", sa.DateTime(), nullable=False), + sa.Column("creator_account_id", sa.String(), nullable=False), + sa.Column("name", sa.String()), + sa.Column("name_intl", sa.JSON()), + sa.Column("short_description", sa.String()), + sa.Column("short_description_intl", sa.JSON()), + sa.Column("description", sa.String()), + sa.Column("description_intl", sa.JSON()), + sa.Column("author", sa.String()), + sa.Column("icon", sa.String()), + sa.Column("nettests", sa.JSON(), nullable=False), + sa.Column("is_archived", sa.Boolean()), + ) + + +def downgrade() -> None: + op.drop_table("oonirun") diff --git a/api/fastapi/oonidataapi/config.py b/api/fastapi/oonidataapi/config.py index d4aba5a0..989d9a31 100644 --- a/api/fastapi/oonidataapi/config.py +++ b/api/fastapi/oonidataapi/config.py @@ -8,12 +8,15 @@ class Settings(BaseSettings): app_name: str = "OONI Data API" base_url: str = "https://api.ooni.io" clickhouse_url: str = "clickhouse://localhost" + # In production you want to set this to: postgresql://user:password@postgresserver/db + postgresql_url: str = "sqlite:///./testdb.sqlite3" log_level: str = "info" s3_bucket_name: str = "oonidata-eufra" other_collectors: List[str] = [] statsd_host: str = "localhost" statsd_port: int = 8125 statsd_prefix: str = "ooniapi" + jwt_encryption_key: str = "CHANGEME" settings = Settings() diff --git a/api/fastapi/oonidataapi/dependencies.py b/api/fastapi/oonidataapi/dependencies.py index 2107793d..ca24f99f 100644 --- a/api/fastapi/oonidataapi/dependencies.py +++ b/api/fastapi/oonidataapi/dependencies.py @@ -1,7 +1,19 @@ +from typing import Generator +from sqlalchemy.orm.session import Session + from clickhouse_driver import Client as ClickhouseClient +from .postgresql import SessionLocal from .config import settings def get_clickhouse_client() -> ClickhouseClient: return ClickhouseClient.from_url(settings.clickhouse_url) + + +def get_postgresql_session(): + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/api/fastapi/oonidataapi/main.py b/api/fastapi/oonidataapi/main.py index 098bd709..93678fd3 100644 --- a/api/fastapi/oonidataapi/main.py +++ b/api/fastapi/oonidataapi/main.py @@ -2,23 +2,44 @@ from .routers import measurements from .routers import aggregation +from .routers import oonirun from .config import settings +from fastapi.middleware.cors import CORSMiddleware import logging logging.basicConfig(level=getattr(logging, settings.log_level.upper())) app = FastAPI() +# TODO: temporarily enable all +origins = [ + "*" +] +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + app.include_router(measurements.router, prefix="/api") app.include_router(aggregation.router, prefix="/api") +app.include_router(oonirun.router, prefix="/api") + from importlib.metadata import version as importlib_version from importlib.resources import files as importlib_files pkg_name = "oonidataapi" -pkg_version = importlib_version(pkg_name) +try: + pkg_version = importlib_version(pkg_name) +except: + # This happens when we are not installed, for example in development + pkg_version = None + try: with importlib_files(pkg_name).joinpath("BUILD_LABEL").open("r") as in_file: build_label = in_file.read().strip() diff --git a/api/fastapi/oonidataapi/models.py b/api/fastapi/oonidataapi/models.py new file mode 100644 index 00000000..b8ef1ddc --- /dev/null +++ b/api/fastapi/oonidataapi/models.py @@ -0,0 +1,39 @@ +from datetime import timezone +from sqlalchemy import Boolean, Column, Integer, String, DateTime, JSON + +from .postgresql import Base + + +class OONIRunLink(Base): + __tablename__ = "oonirun" + + oonirun_link_id = Column(String, primary_key=True) + revision = Column(Integer, default=1, primary_key=True) + date_updated = Column(DateTime) + date_created = Column(DateTime) + creator_account_id = Column(String) + + expiration_date = Column(DateTime, nullable=False) + + # Timezones are kind of tricky. We assume everything is always in UTC, + # but python, rightfully complains, if that encoding is not specified in + # the object itself since more modern versions of python. + # To avoid making this a DB specific change, we don't introduce the + # TIMESTAMP column which would allow us to retrieve timezone native + # objects, but instead do casting to the timezone native equivalent in + # the code. + # See: https://stackoverflow.com/questions/414952/sqlalchemy-datetime-timezone + @property + def expiration_date_dt_native(self): + return self.expiration_date.replace(tzinfo=timezone.utc) + + name = Column(String) + name_intl = Column(JSON, nullable=True) + short_description = Column(String) + short_description_intl = Column(JSON, nullable=True) + description = Column(String) + description_intl = Column(JSON, nullable=True) + author = Column(String) + icon = Column(String) + color = Column(String) + nettests = Column(JSON) diff --git a/api/fastapi/oonidataapi/postgresql.py b/api/fastapi/oonidataapi/postgresql.py new file mode 100644 index 00000000..159518a0 --- /dev/null +++ b/api/fastapi/oonidataapi/postgresql.py @@ -0,0 +1,12 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import sessionmaker + +from .config import settings + +engine = create_engine(settings.postgresql_url) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + +class Base(DeclarativeBase): + pass diff --git a/api/fastapi/oonidataapi/routers/aggregation.py b/api/fastapi/oonidataapi/routers/aggregation.py index d10adb9d..5b12439a 100644 --- a/api/fastapi/oonidataapi/routers/aggregation.py +++ b/api/fastapi/oonidataapi/routers/aggregation.py @@ -362,7 +362,7 @@ async def get_measurements( ) where_expr = and_(*where) - query = select(cols).where(where_expr).select_from(table) # type: ignore + query = select(*cols).where(where_expr).select_from(table) # Add group-by for g in group_by: @@ -386,17 +386,17 @@ async def get_measurements( if resp_format == "CSV": csv_data = convert_to_csv(r) if download: - headers[ - "Content-Disposition" - ] = f"attachment; filename=ooni-aggregate-data.csv" + headers["Content-Disposition"] = ( + f"attachment; filename=ooni-aggregate-data.csv" + ) return Response(content=csv_data, media_type="text/csv", headers=headers) else: if download: - headers[ - "Content-Disposition" - ] = f"attachment; filename=ooni-aggregate-data.csv" + headers["Content-Disposition"] = ( + f"attachment; filename=ooni-aggregate-data.csv" + ) set_dload(response, "ooni-aggregate-data.json") return MeasurementAggregation( v=0, diff --git a/api/fastapi/oonidataapi/routers/measurements.py b/api/fastapi/oonidataapi/routers/measurements.py index 20965667..021c0904 100644 --- a/api/fastapi/oonidataapi/routers/measurements.py +++ b/api/fastapi/oonidataapi/routers/measurements.py @@ -17,7 +17,7 @@ from fastapi import APIRouter, Depends, Query, HTTPException, Header, Request from fastapi.responses import Response, JSONResponse -from pydantic import BaseModel +from pydantic import BaseModel, validator from typing_extensions import Annotated # debdeps: python3-sqlalchemy @@ -115,9 +115,9 @@ def get_measurement( headers = {"Cache-Control": "max-age=3600"} if download: - headers[ - "Content-Disposition" - ] = f"attachment; filename=ooni_measurement-{measurement_uid}.json" + headers["Content-Disposition"] = ( + f"attachment; filename=ooni_measurement-{measurement_uid}.json" + ) return Response(content=body, media_type="application/json", headers=headers) @@ -608,13 +608,13 @@ async def list_measurements( Optional[str], Query(description="Category code from the citizenlab list") ] = None, since: Annotated[ - Optional[datetime], + Optional[str], Query( description='Start date of when measurements were run (ex. "2016-10-20T10:30:00")' ), ] = None, until: Annotated[ - Optional[datetime], + Optional[str], Query( description='End date of when measurement were run (ex. "2016-10-20T10:30:00")' ), @@ -729,19 +729,27 @@ async def list_measurements( # # Prepare query parameters + until_dt = None + if until is not None: + until_dt = datetime.strptime(until, "%Y-%m-%d") + # Set reasonable since/until ranges if not specified. try: if until is None: if report_id is None: t = datetime.utcnow() + timedelta(days=1) - until = datetime(t.year, t.month, t.day) + until_dt = datetime(t.year, t.month, t.day) except ValueError: raise HTTPException(status_code=400, detail="Invalid until") + since_dt = None + if since is not None: + since_dt = datetime.strptime(since, "%Y-%m-%d") + try: - if since is None: - if report_id is None and until is not None: - since = until - timedelta(days=30) + if since_dt is None: + if report_id is None and until_dt is not None: + since_dt = until_dt - timedelta(days=30) except ValueError: raise HTTPException(status_code=400, detail="Invalid since") @@ -760,11 +768,11 @@ async def list_measurements( # Populate WHERE clauses and query_params dict if since is not None: - query_params["since"] = since + query_params["since"] = since_dt fpwhere.append(sql_text("measurement_start_time > :since")) if until is not None: - query_params["until"] = until + query_params["until"] = until_dt fpwhere.append(sql_text("measurement_start_time <= :until")) if report_id: @@ -981,13 +989,6 @@ async def get_torsf_stats( """ cacheable = False - cols = [ - sql_text("toDate(measurement_start_time) AS measurement_start_day"), - column("probe_cc"), - sql_text("countIf(anomaly = 't') AS anomaly_count"), - sql_text("countIf(confirmed = 't') AS confirmed_count"), - sql_text("countIf(msm_failure = 't') AS failure_count"), - ] table = sql_table("fastpath") where = [sql_text("test_name = 'torsf'")] query_params: Dict[str, Any] = {} @@ -1007,7 +1008,17 @@ async def get_torsf_stats( # Assemble query where_expr = and_(*where) - query = select(cols).where(where_expr).select_from(table) # type: ignore + query = ( + select( + sql_text("toDate(measurement_start_time) AS measurement_start_day"), + column("probe_cc"), + sql_text("countIf(anomaly = 't') AS anomaly_count"), + sql_text("countIf(confirmed = 't') AS confirmed_count"), + sql_text("countIf(msm_failure = 't') AS failure_count"), + ) + .where(where_expr) + .select_from(table) + ) query = query.group_by(column("measurement_start_day"), column("probe_cc")) query = query.order_by(column("measurement_start_day"), column("probe_cc")) diff --git a/api/fastapi/oonidataapi/routers/oonirun.py b/api/fastapi/oonidataapi/routers/oonirun.py new file mode 100644 index 00000000..ea213c1c --- /dev/null +++ b/api/fastapi/oonidataapi/routers/oonirun.py @@ -0,0 +1,378 @@ +""" +OONIRun link management + +https://github.com/ooni/spec/blob/master/backends/bk-005-ooni-run-v2.md +""" + +from datetime import datetime, timedelta, timezone +from os import urandom +from sys import byteorder +from typing import Dict, Any, List, Optional +import json +import logging + +from fastapi import APIRouter, Depends, Query, HTTPException, Header +from pydantic import computed_field, constr, Field, validator +from pydantic import BaseModel as PydandicBaseModel +from typing_extensions import Annotated + +import sqlalchemy + +from ..config import metrics +from .. import models + +from ..utils import ( + commasplit, + role_required, + get_client_role, + get_account_id_or_raise, + get_account_id_or_none, +) +from ..dependencies import get_postgresql_session + + +ISO_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class BaseModel(PydandicBaseModel): + class Config: + json_encoders = {datetime: lambda v: v.strftime(ISO_FORMAT)} + + +log = logging.getLogger(__name__) + +router = APIRouter() + + +class OONIRunLinkBase(BaseModel): + name: str = Field( + default="", title="name of the ooni run link", min_length=2, max_length=50 + ) + short_description: str = Field( + default="", + title="short description of the ooni run link", + min_length=2, + max_length=200, + ) + + description: str = Field( + default="", title="full description of the ooni run link", min_length=2 + ) + author: str = Field( + default="", + title="public email address of the author name of the ooni run link", + min_length=2, + max_length=100, + ) + + nettests: List[Dict] + + name_intl: Optional[Dict[str, str]] = Field( + default=None, + description="name of the ooni run link in different languages", + ) + short_description_intl: Optional[Dict[str, str]] = Field( + default=None, + description="short description of the ooni run link in different languages", + ) + description_intl: Optional[Dict[str, str]] = Field( + default=None, + description="full description of the ooni run link in different languages", + ) + + @validator("name_intl", "short_description_intl", "description_intl") + def validate_intl(cls, v): + # None is also a valid type + if v is None: + return v + for value in v.values(): + if len(value) < 2: + raise ValueError("must be at least 2 characters") + return v + + icon: Optional[str] = Field( + default=None, + description="icon to use for the ooni run link", + ) + color: Optional[str] = Field( + default=None, + description="color to use for the ooni run link as a hex value prefixed with #", + pattern="^#(?:[0-9a-fA-F]{6})$", + ) + expiration_date: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc) + timedelta(days=30 * 6), + description="future date after which the ooni run link will be considered expired and no longer editable or usable (defaults to 6 months from now)", + ) + + +class OONIRunLink(OONIRunLinkBase): + oonirun_link_id: int + date_created: datetime + date_updated: datetime + revision: int + is_mine: Optional[bool] = False + + @computed_field + @property + def is_expired(self) -> bool: + # See docstring of models.OONIRunLink.expiration_date_dt_native + return self.expiration_date.replace(tzinfo=timezone.utc) < datetime.now( + timezone.utc + ) + + class Config: + orm_mode = True + + +class OONIRunLinkCreateEdit(OONIRunLinkBase): + pass + + +def generate_random_intuid() -> int: + collector_id = 0 + randint = int.from_bytes(urandom(4), byteorder) + return randint * 100 + collector_id + + +@router.post( + "/v2/oonirun", + tags=["oonirun"], + dependencies=[Depends(role_required(["admin", "user"]))], + response_model=OONIRunLink, +) +def create_oonirun_link( + create_request: OONIRunLinkCreateEdit, + authorization: str = Header("authorization"), + db=Depends(get_postgresql_session), +): + """Create a new oonirun link or a new version for an existing one.""" + log.debug("creating oonirun") + account_id = get_account_id_or_raise(authorization) + assert create_request + + now = datetime.now(timezone.utc).replace(microsecond=0) + + oonirun_link = models.OONIRunLink( + oonirun_link_id=generate_random_intuid(), + creator_account_id=account_id, + name=create_request.name, + name_intl=create_request.name_intl, + short_description=create_request.short_description, + short_description_intl=create_request.short_description_intl, + description=create_request.description, + description_intl=create_request.description_intl, + author=create_request.author, + nettests=create_request.nettests, + icon=create_request.icon, + color=create_request.color, + expiration_date=create_request.expiration_date, + date_created=now, + date_updated=now, + ) + + db.add(oonirun_link) + db.commit() + db.refresh(oonirun_link) + + return oonirun_link + + +@router.put( + "/v2/oonirun/{oonirun_link_id}", + dependencies=[Depends(role_required(["admin", "user"]))], + tags=["oonirun"], + response_model=OONIRunLink, +) +def edit_oonirun_link( + oonirun_link_id: str, + edit_request: OONIRunLinkCreateEdit, + authorization: str = Header("authorization"), + db=Depends(get_postgresql_session), +): + """Edit an existing OONI Run link""" + log.debug(f"edit oonirun {oonirun_link_id}") + account_id = get_account_id_or_raise(authorization) + + now = datetime.now(timezone.utc).replace(microsecond=0) + + q = db.query(models.OONIRunLink).filter( + models.OONIRunLink.oonirun_link_id == oonirun_link_id + ) + if get_client_role(authorization) != "admin": + q = q.filter(models.OONIRunLink.creator_account_id == account_id) + oonirun_link = q.order_by(models.OONIRunLink.revision.desc()).first() + if not oonirun_link: + raise HTTPException(status_code=404, detail="OONI Run link not found") + + if oonirun_link.expiration_date_dt_native < now: + raise HTTPException( + status_code=403, + detail="OONI Run link has expired and cannot be edited", + ) + + if edit_request.expiration_date is not None: + q = db.query(models.OONIRunLink).filter( + models.OONIRunLink.oonirun_link_id == oonirun_link_id, + # Timezones in python are a mess... + models.OONIRunLink.expiration_date > now.replace(tzinfo=None), + ) + if get_client_role(authorization) != "admin": + q = q.filter(models.OONIRunLink.creator_account_id == account_id) + + q.update({"expiration_date": edit_request.expiration_date}) + db.commit() + + current_nettests = oonirun_link.nettests + if current_nettests != edit_request.nettests: + new_oonirun_link = models.OONIRunLink( + oonirun_link_id=oonirun_link.oonirun_link_id, + creator_account_id=account_id, + name=edit_request.name, + name_intl=edit_request.name_intl, + short_description=edit_request.short_description, + short_description_intl=edit_request.short_description_intl, + description=edit_request.description, + description_intl=edit_request.description_intl, + author=edit_request.author, + nettests=edit_request.nettests, + icon=edit_request.icon, + color=edit_request.color, + expiration_date=edit_request.expiration_date, + revision=int(oonirun_link.revision + 1), + date_created=now, + date_updated=now, + ) + db.add(new_oonirun_link) + db.commit() + return new_oonirun_link + + oonirun_link.name = edit_request.name + oonirun_link.name_intl = edit_request.name_intl + oonirun_link.short_description = edit_request.short_description + oonirun_link.short_description_intl = edit_request.short_description_intl + oonirun_link.description = edit_request.description + oonirun_link.description_intl = edit_request.description_intl + oonirun_link.author = edit_request.author + oonirun_link.nettests = edit_request.nettests + oonirun_link.icon = edit_request.icon + oonirun_link.color = edit_request.color + oonirun_link.expiration_date = edit_request.expiration_date + oonirun_link.date_updated = now + db.commit() + return oonirun_link + + +@metrics.timer("fetch_oonirun_link") +@router.get( + "/v2/oonirun/{oonirun_link_id}", tags=["oonirun"], response_model=OONIRunLink +) +def fetch_oonirun_link( + oonirun_link_id: str, + revision: Annotated[ + Optional[int], + Query( + description="specificy which revision of the run link descriptor you wish to fetch" + ), + ] = None, + authorization: str = Header("authorization"), + db=Depends(get_postgresql_session), +): + """Fetch OONIRun descriptor by creation time or the newest one""" + # Return the latest version of the translations + log.debug("fetching oonirun") + account_id = get_account_id_or_none(authorization) + + q = db.query(models.OONIRunLink).filter( + models.OONIRunLink.oonirun_link_id == oonirun_link_id + ) + if revision is not None: + q = q.filter(models.OONIRunLink.revision == revision) + oonirun_link = q.order_by(models.OONIRunLink.revision.desc()).first() + + if oonirun_link is None: + raise HTTPException(status_code=404, detail=f"OONI Run link not found") + + oonirun_link.is_mine = account_id == oonirun_link.creator_account_id + return oonirun_link + + +class OONIRunLinkList(BaseModel): + links: List[OONIRunLink] + + class Config: + orm_mode = True + + +@router.get("/v2/oonirun_links", tags=["oonirun"]) +def list_oonirun_links( + oonirun_link_id: Annotated[ + Optional[str], + Query(description="OONI Run descriptors comma separated"), + ] = None, + only_latest: Annotated[ + Optional[bool], + Query(description="List only the latest versions"), + ] = None, + only_mine: Annotated[ + Optional[bool], + Query(description="List only the my descriptors"), + ] = None, + include_expired: Annotated[ + Optional[bool], + Query(description="List also expired descriptors"), + ] = None, + authorization: str = Header("authorization"), + db=Depends(get_postgresql_session), +) -> OONIRunLinkList: + """List OONIRun descriptors""" + log.debug("list oonirun") + account_id = get_account_id_or_none(authorization) + + q = db.query(models.OONIRunLink) + if only_latest: + subquery = ( + db.query( + models.OONIRunLink.oonirun_link_id, + sqlalchemy.func.max(models.OONIRunLink.revision).label("revision"), + ) + .group_by(models.OONIRunLink.oonirun_link_id) + .subquery("latest_link") + ) + q = q.filter( + sqlalchemy.tuple_( + models.OONIRunLink.oonirun_link_id, + models.OONIRunLink.revision, + ).in_(subquery) + ) + if not include_expired: + q = q.filter(models.OONIRunLink.expiration_date > datetime.now(timezone.utc)) + if only_mine: + q = q.filter(models.OONIRunLink.creator_account_id == account_id) + + if oonirun_link_id: + q = q.filter( + models.OONIRunLink.oonirun_link_id.in_(commasplit(oonirun_link_id)) + ) + + links = [] + for row in q.all(): + oonirun_link = OONIRunLink( + oonirun_link_id=row.oonirun_link_id, + name=row.name, + name_intl=row.name_intl, + short_description=row.short_description, + short_description_intl=row.short_description_intl, + description=row.description, + description_intl=row.description_intl, + author=row.author, + nettests=row.nettests, + icon=row.icon, + expiration_date=row.expiration_date, + revision=row.revision, + date_created=row.date_created, + date_updated=row.date_updated, + is_mine=account_id == row.creator_account_id, + ) + links.append(oonirun_link) + log.debug(f"Returning {len(links)} ooni run links") + return OONIRunLinkList(links=links) diff --git a/api/fastapi/oonidataapi/tests/conftest.py b/api/fastapi/oonidataapi/tests/conftest.py new file mode 100644 index 00000000..4f01cabb --- /dev/null +++ b/api/fastapi/oonidataapi/tests/conftest.py @@ -0,0 +1,105 @@ +import pytest + +import time +import jwt +from pathlib import Path + +from fastapi.testclient import TestClient + +from ..config import settings +from ..main import app +from ..dependencies import get_postgresql_session +from .. import models + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + + +def setup_db_alembic(db_url): + from alembic import command + from alembic.config import Config + + migrations_path = (Path(__file__).parent.parent / "alembic").resolve() + + alembic_cfg = Config() + alembic_cfg.set_main_option("script_location", str(migrations_path)) + alembic_cfg.set_main_option("sqlalchemy.url", db_url) + + ret = command.upgrade(alembic_cfg, "head") + print(ret) + + +def setup_db(db_url): + engine = create_engine(db_url, connect_args={"check_same_thread": False}) + metadata = models.OONIRunLink.metadata + metadata.create_all(engine) + + +def override_pg(db_url): + def f(): + engine = create_engine(db_url, connect_args={"check_same_thread": False}) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + db = SessionLocal() + try: + yield db + finally: + db.close() + + return f + + +@pytest.fixture +def postgresql(tmp_path_factory): + db_path = tmp_path_factory.mktemp("oonidb") / "db.sqlite3" + db_url = f"sqlite:///{db_path}" + + setup_db(db_url) + app.dependency_overrides[get_postgresql_session] = override_pg(db_url) + yield + + +@pytest.fixture +def client(postgresql): + client = TestClient(app) + return client + + +def create_jwt(payload: dict) -> str: + key = settings.jwt_encryption_key + token = jwt.encode(payload, key, algorithm="HS256") + if isinstance(token, bytes): + return token.decode() + else: + return token + + +def create_session_token(account_id: str, role: str, login_time=None) -> str: + now = int(time.time()) + if login_time is None: + login_time = now + payload = { + "nbf": now, + "iat": now, + "exp": now + 10 * 86400, + "aud": "user_auth", + "account_id": account_id, + "login_time": login_time, + "role": role, + } + return create_jwt(payload) + + +@pytest.fixture +def client_with_user_role(client): + client = TestClient(app) + jwt_token = create_session_token("0" * 16, "user") + client.headers = {"Authorization": f"Bearer {jwt_token}"} + yield client + + +@pytest.fixture +def client_with_admin_role(client): + client = TestClient(app) + jwt_token = create_session_token("0" * 16, "admin") + client.headers = {"Authorization": f"Bearer {jwt_token}"} + yield client diff --git a/api/fastapi/oonidataapi/tests/integ/conftest.py b/api/fastapi/oonidataapi/tests/integ/conftest.py index 68a386cd..5b9f0e30 100644 --- a/api/fastapi/oonidataapi/tests/integ/conftest.py +++ b/api/fastapi/oonidataapi/tests/integ/conftest.py @@ -13,14 +13,6 @@ THIS_DIR = Path(__file__).parent -def pytest_addoption(parser): - parser.addoption("--proddb", action="store_true", help="uses data from prod DB") - - -def pytest_configure(config): - pytest.proddb = config.getoption("--proddb") - - def run_clickhouse_sql_scripts(clickhouse_url): click = Clickhouse.from_url(clickhouse_url) tables = click.execute("SHOW TABLES") diff --git a/api/fastapi/oonidataapi/tests/integ/test_aggregation.py b/api/fastapi/oonidataapi/tests/integ/test_aggregation.py index f93b598d..78697569 100644 --- a/api/fastapi/oonidataapi/tests/integ/test_aggregation.py +++ b/api/fastapi/oonidataapi/tests/integ/test_aggregation.py @@ -4,11 +4,15 @@ from urllib.parse import urlencode import json +pytest.skip( + "currently broken tests, should be upgraded to work in new CI", + allow_module_level=True, +) -from ...main import app def is_json(resp): - return resp.headers.get('content-type') == 'application/json' + return resp.headers.get("content-type") == "application/json" + def fjd(o): # non-indented JSON dump @@ -23,7 +27,7 @@ def api(client, subpath, **kw): response = client.get(url) assert response.status_code == 200, response.data - assert response.is_json + assert is_json(response) return response.json @@ -401,7 +405,6 @@ def test_aggregation_x_axis_only_probe_cc(client): assert len(r["result"]) == 33 -@pytest.mark.skipif(not pytest.proddb, reason="use --proddb to run") def test_aggregation_x_axis_only_category_code(client): # 1-dimensional data url = "aggregation?probe_cc=IE&category_code=HACK&since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day" @@ -429,7 +432,6 @@ def test_aggregation_x_axis_only_category_code(client): assert r == expected, fjd(r) -@pytest.mark.skipif(not pytest.proddb, reason="use --proddb to run") def test_aggregation_x_axis_only_csv(client): # 1-dimensional data url = "aggregation?probe_cc=BR&probe_asn=AS8167&since=2021-07-09&until=2021-07-10&format=CSV&axis_x=measurement_start_day" @@ -452,7 +454,6 @@ def test_aggregation_x_axis_only_csv(client): assert r.replace("\r", "") == expected -@pytest.mark.skipif(not pytest.proddb, reason="use --proddb to run") def test_aggregation_x_axis_y_axis(client): # 2-dimensional data url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day&axis_y=probe_cc&test_name=web_connectivity" @@ -470,7 +471,6 @@ def test_aggregation_x_axis_y_axis_are_the_same(client): assert r == {"error": "Axis X and Y cannot be the same", "v": 0} -@pytest.mark.skipif(not pytest.proddb, reason="use --proddb to run") def test_aggregation_two_axis_too_big(client, log): url = "aggregation?since=2021-10-14&until=2021-10-15&test_name=web_connectivity&axis_x=measurement_start_day&axis_y=input" r = api(client, url) @@ -560,7 +560,6 @@ def test_aggregation_x_axis_category_code(client): assert r["result"][:3] == aggreg_over_category_code_expected, fjd(r) -# @pytest.mark.skipif(not pytest.proddb, reason="use --proddb to run") @pytest.mark.skip("FIXME citizenlab") def test_aggregation_y_axis_category_code(client): # 1d data over a special column: category_code diff --git a/api/fastapi/oonidataapi/tests/run_smoketest.py b/api/fastapi/oonidataapi/tests/run_smoketest.py new file mode 100644 index 00000000..95e0bc40 --- /dev/null +++ b/api/fastapi/oonidataapi/tests/run_smoketest.py @@ -0,0 +1,51 @@ +import httpx +import time +import click +import random + + +def test_oonirun(client): + r = client.get("/api/v2/oonirun_links") + j = r.json() + assert r.status_code == 200, j + desc = j["links"] + assert isinstance(desc, list) + if len(desc) > 0: + for _ in range(5): + d = random.choice(desc) + client.get(f'/api/v2/oonirun/{d["oonirun_link_id"]}').raise_for_status() + + +def wait_for_backend(backend_base_url, timeout=10): + start_time = time.time() + + while True: + try: + with httpx.Client(base_url=backend_base_url) as client: + r = client.get("/version") + if r.status_code == 200: + print("Service ready") + break + except Exception as e: + print(f"Connection failed: {e}") + + if time.time() - start_time > timeout: + raise TimeoutError("Service did not become available in time") + + time.sleep(1) + +@click.command() +@click.option( + "--backend-base-url", + default="http://localhost:8000", + help="Base URL of the backend", +) +def smoketest(backend_base_url): + """Run a smoke test against a running backend""" + wait_for_backend(backend_base_url) + + with httpx.Client(base_url=backend_base_url) as client: + test_oonirun(client) + +if __name__ == "__main__": + smoketest() diff --git a/api/fastapi/oonidataapi/tests/test_measurements.py b/api/fastapi/oonidataapi/tests/test_measurements.py index c3942e42..24dc3127 100644 --- a/api/fastapi/oonidataapi/tests/test_measurements.py +++ b/api/fastapi/oonidataapi/tests/test_measurements.py @@ -47,7 +47,7 @@ def execute(self, sql, query_params=(), *arg, **kwargs): yield MockClick(conn) conn.close() os.close(fd) - # os.remove(path) + os.remove(path) @pytest.fixture(name="client") @@ -63,13 +63,14 @@ def get_clickhouse_override(): def test_list_measurements(client, clickhouse): - clickhouse.execute("SELECT * FROM fastpath") - response = client.get("/api/v1/measurements") + response = client.get("/api/v1/measurements?since=2024-01-01&until=2024-02-01") assert response.status_code == 200 j = response.json() assert len(j["results"]) == 100 - response = client.get("/api/v1/measurements?probe_cc=IT") + response = client.get( + "/api/v1/measurements?probe_cc=IT&since=2024-01-01&until=2024-02-01" + ) assert response.status_code == 200 j = response.json() for res in j["results"]: diff --git a/api/fastapi/oonidataapi/tests/test_oonirun.py b/api/fastapi/oonidataapi/tests/test_oonirun.py new file mode 100644 index 00000000..45a68e74 --- /dev/null +++ b/api/fastapi/oonidataapi/tests/test_oonirun.py @@ -0,0 +1,469 @@ +""" +Integration test for OONIRn API +""" + +from copy import deepcopy +from datetime import datetime, timedelta, timezone +import time + +SAMPLE_OONIRUN = { + "name": "", + "name_intl": {}, + "description": "integ-test description in English", + "description_intl": { + "es": "integ-test descripción en español", + }, + "short_description": "integ-test short description in English", + "short_description_intl": { + "it": "integ-test descrizione breve in italiano", + }, + "icon": "myicon", + "author": "integ-test author", + "nettests": [ + { + "inputs": ["https://example.com/", "https://ooni.org/"], + "options": { + "HTTP3Enabled": True, + }, + "test_name": "web_connectivity", + }, + {"test_name": "dnscheck"}, + ], +} + +EXPECTED_OONIRUN_LINK_PUBLIC_KEYS = [ + "oonirun_link_id", + "date_created", + "date_updated", + "revision", + "is_mine", + "is_expired", + "name", + "short_description", + "description", + "author", + "nettests", + "name_intl", + "short_description_intl", + "description_intl", + "icon", + "color", + "expiration_date", +] + + +def test_oonirun_validation(client, client_with_user_role, client_with_admin_role): + z = deepcopy(SAMPLE_OONIRUN) + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 422, "empty name should be rejected" + + z["name"] = "integ-test name in English" + z["name_intl"] = {"it": ""} + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 422, "empty name_intl should be rejected" + + z = deepcopy(SAMPLE_OONIRUN) + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 422, "empty name should be rejected" + + z["name"] = "integ-test name in English" + z["name_intl"] = None + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, "name_intl can be None" + + +def test_oonirun_not_found(client, client_with_user_role, client_with_admin_role): + z = deepcopy(SAMPLE_OONIRUN) + ### Create descriptor as user + z["name"] = "integ-test name in English" + z["name_intl"]["it"] = "integ-test nome in italiano" + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, r.json() + j = r.json() + assert str(j["oonirun_link_id"]).endswith("00") + oonirun_link_id = int(r.json()["oonirun_link_id"]) + + j["expiration_date"] = ( + datetime.now(timezone.utc) + timedelta(minutes=-1) + ).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=j) + assert r.status_code == 200, r.json() + + not_existing_link_id = "1234676871672836187" + r = client_with_user_role.put(f"/api/v2/oonirun/{not_existing_link_id}", json=j) + assert r.status_code == 404, r.json() + + r = client.get(f"/api/v2/oonirun/{not_existing_link_id}") + assert r.status_code == 404, r.json() + + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=j) + assert r.status_code == 403, "expired link cannot be edited" + + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}" + ) + j = r.json() + assert r.status_code == 200, r.json() + assert j["links"] == [] + + +def test_oonirun_full_workflow(client, client_with_user_role, client_with_admin_role): + z = deepcopy(SAMPLE_OONIRUN) + ### Create 2 descriptors as user + z["name"] = "integ-test name in English" + z["name_intl"]["it"] = "integ-test nome in italiano" + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, r.json() + assert str(r.json()["oonirun_link_id"]).endswith("00") + oonirun_link_id = int(r.json()["oonirun_link_id"]) + + z["name"] = "second descriptor in English" + z["name_intl"]["it"] = "second integ-test nome in italiano" + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, r.json() + assert str(r.json()["oonirun_link_id"]).endswith("00") + oonirun_link_id = int(r.json()["oonirun_link_id"]) + + r = client_with_user_role.get(f"/api/v2/oonirun/{oonirun_link_id}") + assert r.status_code == 200, r.json() + + j = r.json() + assert j["name"] == z["name"] + assert j["name_intl"] == z["name_intl"] + assert j["description"] == z["description"] + assert j["nettests"] == z["nettests"] + date_created = datetime.strptime( + j["date_created"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + assert date_created < datetime.now(timezone.utc) + assert date_created > datetime.now(timezone.utc) + timedelta(hours=-1) + + date_updated = datetime.strptime( + j["date_updated"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + assert date_updated < datetime.now(timezone.utc) + assert date_updated > datetime.now(timezone.utc) + timedelta(hours=-1) + + assert j["is_mine"] == True + assert j["revision"] == 1 + + ## Fetch by revision + r = client_with_user_role.get(f"/api/v2/oonirun/{oonirun_link_id}?revision=1") + assert r.status_code == 200, r.json() + + j = r.json() + assert j["name"] == z["name"] + assert j["name_intl"] == z["name_intl"] + assert j["author"] == z["author"] + assert j["description"] == z["description"] + assert j["nettests"] == z["nettests"] + + date_created = datetime.strptime( + j["date_created"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + assert date_created < datetime.now(timezone.utc) + assert date_created > datetime.now(timezone.utc) + timedelta(hours=-1) + + date_updated = datetime.strptime( + j["date_updated"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + assert date_updated < datetime.now(timezone.utc) + assert date_updated > datetime.now(timezone.utc) + timedelta(hours=-1) + + assert j["is_mine"] == True + assert j["revision"] == 1 + + r = client_with_user_role.get("/api/v2/oonirun_links") + assert r.status_code == 200, r.json() + + j = r.json() + assert len(j["links"]) > 0 + + found = False + for d in j["links"]: + if d["oonirun_link_id"] == oonirun_link_id: + found = True + assert sorted(d.keys()) == sorted(EXPECTED_OONIRUN_LINK_PUBLIC_KEYS) + assert found == True + + ## list all items as admin + r = client_with_admin_role.get("/api/v2/oonirun_links") + assert r.status_code == 200, r.json() + + j = r.json() + assert len(j["links"]) > 0 + + found = False + for d in j["links"]: + if d["oonirun_link_id"] == oonirun_link_id: + found = True + assert sorted(d.keys()) == sorted(EXPECTED_OONIRUN_LINK_PUBLIC_KEYS) + assert found == True + + ## find the item created by client_with_user_role above + # fixme + # assert desc[0]["name_intl"] == "integ-test" + + ## list all items as anonymous + r = client.get("/api/v2/oonirun_links") + assert r.status_code == 200, r.json() + + j = r.json() + assert len(j["links"]) > 0 + + found = False + for d in j["links"]: + if d["oonirun_link_id"] == oonirun_link_id: + found = True + assert d["is_mine"] == False + assert d["is_expired"] == False + + assert sorted(d.keys()) == sorted(EXPECTED_OONIRUN_LINK_PUBLIC_KEYS) + assert found == True + + ### "update" the oonirun by creating a new version, changing the inputs + z["nettests"][0]["inputs"].append("https://foo.net/") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=z) + assert r.status_code == 200, r.json() + assert r.json()["oonirun_link_id"] == oonirun_link_id + + ## Fetch it back + r = client_with_user_role.get(f"/api/v2/oonirun/{oonirun_link_id}") + assert r.status_code == 200, r.json() + + j = r.json() + assert j["is_mine"] is True, r.json() + assert j["is_expired"] is False, r.json() + assert j["revision"] > 1, r.json() + + ## List descriptors as admin and find we have 2 versions now + r = client_with_admin_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}" + ) + assert r.status_code == 200, r.json() + descs = r.json()["links"] + assert len(descs) == 2, r.json() + + ## List descriptors using more params + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}&only_mine=True" + ) + assert r.status_code == 200, r.json() + descs = r.json()["links"] + assert len(descs) == 2, r.json() + for d in descs: + assert d["is_mine"] is True + assert d["is_expired"] is False + + # XXX this is wrong. Admin can do everything. + # TODO(art): add test for trying to edit from a non-admin account + # say("Fail to update the oonirun using the wrong account") + # r = client_with_admin_role.put(f"/api/v2/oonirun/{ooni_run_link_id}", json=z) + # assert r.status_code == 400, r.json() + # assert r.json() == {"error": "OONIRun descriptor not found"} + + # Update translations without changing descriptor_creation_time + + # We need to pause 1 second for the update time to be different + time.sleep(1) + z["description_intl"]["it"] = "integ-test *nuova* descrizione in italiano" + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=z) + assert r.status_code == 200, r.json() + + ## previous id and descriptor_creation_time, not changed + assert r.json()["oonirun_link_id"] == oonirun_link_id + # assert creation_time == r.json()["descriptor_creation_time"] + + ## Fetch latest and find descriptor_creation_time has not changed + r = client_with_user_role.get(f"/api/v2/oonirun/{oonirun_link_id}") + assert r.status_code == 200, r.json() + + j = r.json() + + assert sorted(j.keys()) == sorted(EXPECTED_OONIRUN_LINK_PUBLIC_KEYS) + + date_created = datetime.strptime( + j["date_created"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + assert date_created < datetime.now(timezone.utc) + assert date_created > datetime.now(timezone.utc) + timedelta(hours=-1) + + date_updated = datetime.strptime( + j["date_updated"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + assert date_updated < datetime.now(timezone.utc) + assert date_updated > datetime.now(timezone.utc) + timedelta(hours=-1) + + assert date_updated > date_created + + assert j["description_intl"]["it"] == "integ-test *nuova* descrizione in italiano" + assert j["is_mine"] == True + + # Archive it + edit_req = deepcopy(j) + edit_req["expiration_date"] = ( + datetime.now(timezone.utc) + timedelta(minutes=-1) + ).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=edit_req) + j = r.json() + assert r.status_code == 200, r.json() + assert j["is_expired"] == True + + ## List descriptors after expiration filtering by ID + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}&include_expired=True" + ) + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 2, r.json() + + ## List descriptors after expiration NOT filtering by ID + r = client_with_user_role.get(f"/api/v2/oonirun_links?include_expired=True") + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 3, r.json() + + ## List descriptors filtered by ID + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}" + ) + assert r.status_code == 200, r.json() + descs = r.json()["links"] + assert len(descs) == 0, r.json() + + ## List descriptors unfiltered by ID + r = client_with_user_role.get(f"/api/v2/oonirun_links") + assert r.status_code == 200, r.json() + descs = r.json()["links"] + assert len(descs) == 1, r.json() + + ## Fetch latest and find that it's archived + r = client_with_user_role.get(f"/api/v2/oonirun/{oonirun_link_id}") + assert r.status_code == 200, r.json() + assert r.json()["is_expired"] == True, r.json() + + +def test_oonirun_expiration(client, client_with_user_role): + z = deepcopy(SAMPLE_OONIRUN) + ### Create descriptor as user + z["name"] = "integ-test name in English" + z["name_intl"]["it"] = "integ-test nome in italiano" + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, r.json() + assert str(r.json()["oonirun_link_id"]).endswith("00") + oonirun_link_id = int(r.json()["oonirun_link_id"]) + + ## Fetch anonymously and check it's not expired + r = client.get(f"/api/v2/oonirun/{oonirun_link_id}") + j = r.json() + assert r.status_code == 200, r.json() + assert j["is_expired"] == False, r.json() + + ## Create new revision + j["nettests"][0]["inputs"].append("https://foo.net/") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=j) + assert r.status_code == 200, r.json() + + ## Fetch anonymously and check it's got the new revision + r = client.get(f"/api/v2/oonirun/{oonirun_link_id}") + j = r.json() + assert j["revision"] == 2, "revision did not change" + + ## Update expiry time + j["expiration_date"] = ( + datetime.now(timezone.utc) + timedelta(minutes=-1) + ).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id}", json=j) + assert r.status_code == 200, r.json() + assert r.json()["is_expired"] == True, r.json() + + ## Fetch anonymously and check it's expired + r = client.get(f"/api/v2/oonirun/{oonirun_link_id}") + assert r.status_code == 200, r.json() + assert r.json()["is_expired"] == True, r.json() + + ## List descriptors after expiration + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}" + ) + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 0, r.json() + + ## List descriptors after expiration + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}&include_expired=True" + ) + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 2, r.json() + for d in descs: + assert d["is_expired"] == True, "is_expired should be True" + + r = client_with_user_role.get( + f"/api/v2/oonirun_links?oonirun_link_id={oonirun_link_id}&include_expired=True&only_latest=True" + ) + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 1, r.json() + for d in descs: + assert d["is_expired"] == True, "is_expired should be True" + + +def test_oonirun_revisions(client, client_with_user_role): + z = deepcopy(SAMPLE_OONIRUN) + ### Create descriptor as user + z["name"] = "first descriptor" + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, r.json() + j = r.json() + oonirun_link_id_one = int(j["oonirun_link_id"]) + + ## Create two new revisions + j["nettests"][0]["inputs"].append("https://foo.net/") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id_one}", json=j) + assert r.status_code == 200, r.json() + j = r.json() + j["nettests"][0]["inputs"].append("https://foo2.net/") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id_one}", json=j) + assert r.status_code == 200, r.json() + j = r.json() + + ### Create another descriptor as user + z["name"] = "second descriptor" + r = client_with_user_role.post("/api/v2/oonirun", json=z) + assert r.status_code == 200, r.json() + j = r.json() + oonirun_link_id_two = int(j["oonirun_link_id"]) + + ## Create new revision + j["nettests"][0]["inputs"].append("https://foo.net/") + r = client_with_user_role.put(f"/api/v2/oonirun/{oonirun_link_id_two}", json=j) + assert r.status_code == 200, r.json() + + ## Fetch anonymously and check it's got the new revision + r = client.get(f"/api/v2/oonirun/{oonirun_link_id_one}") + j = r.json() + assert j["revision"] == 3, "revision is 3" + + r = client_with_user_role.get(f"/api/v2/oonirun_links") + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 5, r.json() + + r = client_with_user_role.get(f"/api/v2/oonirun_links?only_latest=True") + j = r.json() + assert r.status_code == 200, r.json() + descs = j["links"] + assert len(descs) == 2, r.json() + for d in descs: + if d["oonirun_link_id"] == oonirun_link_id_one: + assert d["revision"] == 3, "revision is 3" + if d["oonirun_link_id"] == oonirun_link_id_two: + assert d["revision"] == 2, "revision is 2" diff --git a/api/fastapi/oonidataapi/utils.py b/api/fastapi/oonidataapi/utils.py index 6738302d..d4cbc325 100644 --- a/api/fastapi/oonidataapi/utils.py +++ b/api/fastapi/oonidataapi/utils.py @@ -1,9 +1,11 @@ from csv import DictWriter from io import StringIO import logging -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union +from fastapi import HTTPException, Header from fastapi.responses import JSONResponse +import jwt import clickhouse_driver import clickhouse_driver.errors @@ -103,3 +105,108 @@ def query_click_one_row( return dict(zip(colnames, row)) # type: ignore return None + + +def insert_click(db: clickhouse_driver.Client, query: Query, rows: list) -> int: + assert isinstance(rows, list) + settings = {"priority": 1, "max_execution_time": 300} # query_prio + return db.execute(query, rows, types_check=True, settings=settings) # type: ignore + + +def optimize_table(db: clickhouse_driver.Client, tblname: str) -> None: + settings = {"priority": 1, "max_execution_time": 300} # query_prio + sql = f"OPTIMIZE TABLE {tblname} FINAL" + db.execute(sql, {}, settings=settings) + + +def raw_query( + db: clickhouse_driver.Client, query: Query, query_params: dict, query_prio=1 +): + settings = {"priority": query_prio, "max_execution_time": 300} + q = db.execute(query, query_params, with_column_types=True, settings=settings) + return q + + +def decode_jwt(token: str, **kw) -> Dict[str, Any]: + # raises ExpiredSignatureError on expiration + key = settings.jwt_encryption_key + tok = jwt.decode(token, key, algorithms=["HS256"], **kw) + return tok + + +def get_client_token(authorization: str): + try: + assert authorization.startswith("Bearer ") + token = authorization[7:] + return decode_jwt(token, audience="user_auth") + except: + return None + + +def role_required(roles): + """Wrapped function requiring user to be logged in and have the right role.""" + # Also: + # explicitely set no-cache headers + # apply the cross_origin decorator to: + # - set CORS header to a trusted URL + # - enable credentials (cookies) + # + if isinstance(roles, str): + roles = [roles] + + async def verify_jwt(authorization: str = Header("authorization")): + tok = get_client_token(authorization) + if tok is None: + raise HTTPException(detail="Authentication required", status_code=401) + if tok["role"] not in roles: + raise HTTPException(detail="Role not authorized", status_code=401) + + # TODO(art): we don't check for the session_expunge table yet. It's empty so the impact is none + # query = """SELECT threshold + # FROM session_expunge + # WHERE account_id = :account_id """ + # account_id = tok["account_id"] + # query_params = dict(account_id=account_id) + # row = query_click_one_row(sql.text(query), query_params) + # if row: + # threshold = row["threshold"] + # iat = datetime.utcfromtimestamp(tok["iat"]) + # if iat < threshold: + # return jerror("Authentication token expired", 401) + + # If needed we can add here a 2-tier expiration time: long for + # /api/v1/user_refresh_token and short for everything else + + return verify_jwt + + +def get_client_role(authorization: str) -> str: + """Raise exception for unlogged users""" + tok = get_client_token(authorization) + assert tok + return tok["role"] + + +def get_account_id_or_none(authorization: str) -> Optional[str]: + """Returns None for unlogged users""" + tok = get_client_token(authorization) + if tok: + return tok["account_id"] + return None + + +def get_account_id_or_raise(authorization: str) -> str: + """Raise exception for unlogged users""" + tok = get_client_token(authorization) + if tok: + return tok["account_id"] + raise Exception + + +def get_account_id(authorization: str): + # TODO: switch to get_account_id_or_none + tok = get_client_token(authorization) + if not tok: + return jerror("Authentication required", 401) + + return tok["account_id"] diff --git a/api/fastapi/poetry.lock b/api/fastapi/poetry.lock index 43b9651f..850c5f74 100644 --- a/api/fastapi/poetry.lock +++ b/api/fastapi/poetry.lock @@ -1,5 +1,24 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "alembic" +version = "1.13.1" +description = "A database migration tool for SQLAlchemy." +optional = false +python-versions = ">=3.8" +files = [ + {file = "alembic-1.13.1-py3-none-any.whl", hash = "sha256:2edcc97bed0bd3272611ce3a98d98279e9c209e7186e43e75bbb1b2bdfdbcc43"}, + {file = "alembic-1.13.1.tar.gz", hash = "sha256:4932c8558bf68f2ee92b9bbcb8218671c627064d5b08939437af6d77dc05e595"}, +] + +[package.dependencies] +Mako = "*" +SQLAlchemy = ">=1.3.0" +typing-extensions = ">=4" + +[package.extras] +tz = ["backports.zoneinfo"] + [[package]] name = "annotated-types" version = "0.6.0" @@ -283,6 +302,70 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coverage" +version = "7.4.3" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8580b827d4746d47294c0e0b92854c85a92c2227927433998f0d3320ae8a71b6"}, + {file = "coverage-7.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:718187eeb9849fc6cc23e0d9b092bc2348821c5e1a901c9f8975df0bc785bfd4"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:767b35c3a246bcb55b8044fd3a43b8cd553dd1f9f2c1eeb87a302b1f8daa0524"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae7f19afe0cce50039e2c782bff379c7e347cba335429678450b8fe81c4ef96d"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba3a8aaed13770e970b3df46980cb068d1c24af1a1968b7818b69af8c4347efb"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ee866acc0861caebb4f2ab79f0b94dbfbdbfadc19f82e6e9c93930f74e11d7a0"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:506edb1dd49e13a2d4cac6a5173317b82a23c9d6e8df63efb4f0380de0fbccbc"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd6545d97c98a192c5ac995d21c894b581f1fd14cf389be90724d21808b657e2"}, + {file = "coverage-7.4.3-cp310-cp310-win32.whl", hash = "sha256:f6a09b360d67e589236a44f0c39218a8efba2593b6abdccc300a8862cffc2f94"}, + {file = "coverage-7.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:18d90523ce7553dd0b7e23cbb28865db23cddfd683a38fb224115f7826de78d0"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbbe5e739d45a52f3200a771c6d2c7acf89eb2524890a4a3aa1a7fa0695d2a47"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:489763b2d037b164846ebac0cbd368b8a4ca56385c4090807ff9fad817de4113"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:451f433ad901b3bb00184d83fd83d135fb682d780b38af7944c9faeecb1e0bfe"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcc66e222cf4c719fe7722a403888b1f5e1682d1679bd780e2b26c18bb648cdc"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ec74cfef2d985e145baae90d9b1b32f85e1741b04cd967aaf9cfa84c1334f3"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:abbbd8093c5229c72d4c2926afaee0e6e3140de69d5dcd918b2921f2f0c8baba"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:35eb581efdacf7b7422af677b92170da4ef34500467381e805944a3201df2079"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8249b1c7334be8f8c3abcaaa996e1e4927b0e5a23b65f5bf6cfe3180d8ca7840"}, + {file = "coverage-7.4.3-cp311-cp311-win32.whl", hash = "sha256:cf30900aa1ba595312ae41978b95e256e419d8a823af79ce670835409fc02ad3"}, + {file = "coverage-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:18c7320695c949de11a351742ee001849912fd57e62a706d83dfc1581897fa2e"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b51bfc348925e92a9bd9b2e48dad13431b57011fd1038f08316e6bf1df107d10"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cdecaedea1ea9e033d8adf6a0ab11107b49571bbb9737175444cea6eb72328"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b2eccb883368f9e972e216c7b4c7c06cabda925b5f06dde0650281cb7666a30"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c00cdc8fa4e50e1cc1f941a7f2e3e0f26cb2a1233c9696f26963ff58445bac7"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9a4a8dd3dcf4cbd3165737358e4d7dfbd9d59902ad11e3b15eebb6393b0446e"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:062b0a75d9261e2f9c6d071753f7eef0fc9caf3a2c82d36d76667ba7b6470003"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ebe7c9e67a2d15fa97b77ea6571ce5e1e1f6b0db71d1d5e96f8d2bf134303c1d"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0a120238dd71c68484f02562f6d446d736adcc6ca0993712289b102705a9a3a"}, + {file = "coverage-7.4.3-cp312-cp312-win32.whl", hash = "sha256:37389611ba54fd6d278fde86eb2c013c8e50232e38f5c68235d09d0a3f8aa352"}, + {file = "coverage-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:d25b937a5d9ffa857d41be042b4238dd61db888533b53bc76dc082cb5a15e914"}, + {file = "coverage-7.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:28ca2098939eabab044ad68850aac8f8db6bf0b29bc7f2887d05889b17346454"}, + {file = "coverage-7.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:280459f0a03cecbe8800786cdc23067a8fc64c0bd51dc614008d9c36e1659d7e"}, + {file = "coverage-7.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c0cdedd3500e0511eac1517bf560149764b7d8e65cb800d8bf1c63ebf39edd2"}, + {file = "coverage-7.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a9babb9466fe1da12417a4aed923e90124a534736de6201794a3aea9d98484e"}, + {file = "coverage-7.4.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dec9de46a33cf2dd87a5254af095a409ea3bf952d85ad339751e7de6d962cde6"}, + {file = "coverage-7.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:16bae383a9cc5abab9bb05c10a3e5a52e0a788325dc9ba8499e821885928968c"}, + {file = "coverage-7.4.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2c854ce44e1ee31bda4e318af1dbcfc929026d12c5ed030095ad98197eeeaed0"}, + {file = "coverage-7.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ce8c50520f57ec57aa21a63ea4f325c7b657386b3f02ccaedeccf9ebe27686e1"}, + {file = "coverage-7.4.3-cp38-cp38-win32.whl", hash = "sha256:708a3369dcf055c00ddeeaa2b20f0dd1ce664eeabde6623e516c5228b753654f"}, + {file = "coverage-7.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:1bf25fbca0c8d121a3e92a2a0555c7e5bc981aee5c3fdaf4bb7809f410f696b9"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b253094dbe1b431d3a4ac2f053b6d7ede2664ac559705a704f621742e034f1f"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77fbfc5720cceac9c200054b9fab50cb2a7d79660609200ab83f5db96162d20c"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6679060424faa9c11808598504c3ab472de4531c571ab2befa32f4971835788e"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4af154d617c875b52651dd8dd17a31270c495082f3d55f6128e7629658d63765"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8640f1fde5e1b8e3439fe482cdc2b0bb6c329f4bb161927c28d2e8879c6029ee"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:69b9f6f66c0af29642e73a520b6fed25ff9fd69a25975ebe6acb297234eda501"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0842571634f39016a6c03e9d4aba502be652a6e4455fadb73cd3a3a49173e38f"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a78ed23b08e8ab524551f52953a8a05d61c3a760781762aac49f8de6eede8c45"}, + {file = "coverage-7.4.3-cp39-cp39-win32.whl", hash = "sha256:c0524de3ff096e15fcbfe8f056fdb4ea0bf497d584454f344d59fce069d3e6e9"}, + {file = "coverage-7.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0209a6369ccce576b43bb227dc8322d8ef9e323d089c6f3f26a597b09cb4d2aa"}, + {file = "coverage-7.4.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:7cbde573904625509a3f37b6fecea974e363460b556a627c60dc2f47e2fffa51"}, + {file = "coverage-7.4.3.tar.gz", hash = "sha256:276f6077a5c61447a48d133ed13e759c09e62aff0dc84274a68dc18660104d52"}, +] + +[package.extras] +toml = ["tomli"] + [[package]] name = "docker" version = "7.0.0" @@ -472,6 +555,94 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "mako" +version = "1.3.2" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Mako-1.3.2-py3-none-any.whl", hash = "sha256:32a99d70754dfce237019d17ffe4a282d2d3351b9c476e90d8a60e63f133b80c"}, + {file = "Mako-1.3.2.tar.gz", hash = "sha256:2a0c8ad7f6274271b3bb7467dd37cf9cc6dab4bc19cb69a4ef10669402de698e"}, +] + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["Babel"] +lingua = ["lingua"] +testing = ["pytest"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + [[package]] name = "packaging" version = "23.2" @@ -671,6 +842,23 @@ files = [ pydantic = ">=2.3.0" python-dotenv = ">=0.21.0" +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pytest" version = "7.4.4" @@ -691,6 +879,24 @@ pluggy = ">=0.12,<2.0" [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-cov" +version = "4.1.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -798,81 +1004,89 @@ files = [ [[package]] name = "sqlalchemy" -version = "1.4.51" +version = "2.0.27" description = "Database Abstraction Library" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-1.4.51-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b"}, - {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1"}, - {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340"}, - {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678"}, - {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072"}, - {file = "SQLAlchemy-1.4.51-cp310-cp310-win32.whl", hash = "sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee"}, - {file = "SQLAlchemy-1.4.51-cp310-cp310-win_amd64.whl", hash = "sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c"}, - {file = "SQLAlchemy-1.4.51-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d"}, - {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f"}, - {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e"}, - {file = "SQLAlchemy-1.4.51-cp311-cp311-win32.whl", hash = "sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81"}, - {file = "SQLAlchemy-1.4.51-cp311-cp311-win_amd64.whl", hash = "sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171"}, - {file = "SQLAlchemy-1.4.51-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d"}, - {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916"}, - {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb"}, - {file = "SQLAlchemy-1.4.51-cp312-cp312-win32.whl", hash = "sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87"}, - {file = "SQLAlchemy-1.4.51-cp312-cp312-win_amd64.whl", hash = "sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-win32.whl", hash = "sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054"}, - {file = "SQLAlchemy-1.4.51-cp36-cp36m-win_amd64.whl", hash = "sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-win32.whl", hash = "sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3"}, - {file = "SQLAlchemy-1.4.51-cp37-cp37m-win_amd64.whl", hash = "sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-win32.whl", hash = "sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b"}, - {file = "SQLAlchemy-1.4.51-cp38-cp38-win_amd64.whl", hash = "sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-win32.whl", hash = "sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee"}, - {file = "SQLAlchemy-1.4.51-cp39-cp39-win_amd64.whl", hash = "sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba"}, - {file = "SQLAlchemy-1.4.51.tar.gz", hash = "sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d04e579e911562f1055d26dab1868d3e0bb905db3bccf664ee8ad109f035618a"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa67d821c1fd268a5a87922ef4940442513b4e6c377553506b9db3b83beebbd8"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c7a596d0be71b7baa037f4ac10d5e057d276f65a9a611c46970f012752ebf2d"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:954d9735ee9c3fa74874c830d089a815b7b48df6f6b6e357a74130e478dbd951"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5cd20f58c29bbf2680039ff9f569fa6d21453fbd2fa84dbdb4092f006424c2e6"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:03f448ffb731b48323bda68bcc93152f751436ad6037f18a42b7e16af9e91c07"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-win32.whl", hash = "sha256:d997c5938a08b5e172c30583ba6b8aad657ed9901fc24caf3a7152eeccb2f1b4"}, + {file = "SQLAlchemy-2.0.27-cp310-cp310-win_amd64.whl", hash = "sha256:eb15ef40b833f5b2f19eeae65d65e191f039e71790dd565c2af2a3783f72262f"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c5bad7c60a392850d2f0fee8f355953abaec878c483dd7c3836e0089f046bf6"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3012ab65ea42de1be81fff5fb28d6db893ef978950afc8130ba707179b4284a"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dbcd77c4d94b23e0753c5ed8deba8c69f331d4fd83f68bfc9db58bc8983f49cd"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d177b7e82f6dd5e1aebd24d9c3297c70ce09cd1d5d37b43e53f39514379c029c"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:680b9a36029b30cf063698755d277885d4a0eab70a2c7c6e71aab601323cba45"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1306102f6d9e625cebaca3d4c9c8f10588735ef877f0360b5cdb4fdfd3fd7131"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-win32.whl", hash = "sha256:5b78aa9f4f68212248aaf8943d84c0ff0f74efc65a661c2fc68b82d498311fd5"}, + {file = "SQLAlchemy-2.0.27-cp311-cp311-win_amd64.whl", hash = "sha256:15e19a84b84528f52a68143439d0c7a3a69befcd4f50b8ef9b7b69d2628ae7c4"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0de1263aac858f288a80b2071990f02082c51d88335a1db0d589237a3435fe71"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce850db091bf7d2a1f2fdb615220b968aeff3849007b1204bf6e3e50a57b3d32"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8dfc936870507da96aebb43e664ae3a71a7b96278382bcfe84d277b88e379b18"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4fbe6a766301f2e8a4519f4500fe74ef0a8509a59e07a4085458f26228cd7cc"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4535c49d961fe9a77392e3a630a626af5baa967172d42732b7a43496c8b28876"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0fb3bffc0ced37e5aa4ac2416f56d6d858f46d4da70c09bb731a246e70bff4d5"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-win32.whl", hash = "sha256:7f470327d06400a0aa7926b375b8e8c3c31d335e0884f509fe272b3c700a7254"}, + {file = "SQLAlchemy-2.0.27-cp312-cp312-win_amd64.whl", hash = "sha256:f9374e270e2553653d710ece397df67db9d19c60d2647bcd35bfc616f1622dcd"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e97cf143d74a7a5a0f143aa34039b4fecf11343eed66538610debc438685db4a"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7b5a3e2120982b8b6bd1d5d99e3025339f7fb8b8267551c679afb39e9c7c7f1"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e36aa62b765cf9f43a003233a8c2d7ffdeb55bc62eaa0a0380475b228663a38f"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5ada0438f5b74c3952d916c199367c29ee4d6858edff18eab783b3978d0db16d"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b1d9d1bfd96eef3c3faedb73f486c89e44e64e40e5bfec304ee163de01cf996f"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-win32.whl", hash = "sha256:ca891af9f3289d24a490a5fde664ea04fe2f4984cd97e26de7442a4251bd4b7c"}, + {file = "SQLAlchemy-2.0.27-cp37-cp37m-win_amd64.whl", hash = "sha256:fd8aafda7cdff03b905d4426b714601c0978725a19efc39f5f207b86d188ba01"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec1f5a328464daf7a1e4e385e4f5652dd9b1d12405075ccba1df842f7774b4fc"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ad862295ad3f644e3c2c0d8b10a988e1600d3123ecb48702d2c0f26771f1c396"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48217be1de7d29a5600b5c513f3f7664b21d32e596d69582be0a94e36b8309cb"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e56afce6431450442f3ab5973156289bd5ec33dd618941283847c9fd5ff06bf"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:611068511b5531304137bcd7fe8117c985d1b828eb86043bd944cebb7fae3910"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b86abba762ecfeea359112b2bb4490802b340850bbee1948f785141a5e020de8"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-win32.whl", hash = "sha256:30d81cc1192dc693d49d5671cd40cdec596b885b0ce3b72f323888ab1c3863d5"}, + {file = "SQLAlchemy-2.0.27-cp38-cp38-win_amd64.whl", hash = "sha256:120af1e49d614d2525ac247f6123841589b029c318b9afbfc9e2b70e22e1827d"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d07ee7793f2aeb9b80ec8ceb96bc8cc08a2aec8a1b152da1955d64e4825fcbac"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cb0845e934647232b6ff5150df37ceffd0b67b754b9fdbb095233deebcddbd4a"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc19ae2e07a067663dd24fca55f8ed06a288384f0e6e3910420bf4b1270cc51"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b90053be91973a6fb6020a6e44382c97739736a5a9d74e08cc29b196639eb979"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2f5c9dfb0b9ab5e3a8a00249534bdd838d943ec4cfb9abe176a6c33408430230"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:33e8bde8fff203de50399b9039c4e14e42d4d227759155c21f8da4a47fc8053c"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-win32.whl", hash = "sha256:d873c21b356bfaf1589b89090a4011e6532582b3a8ea568a00e0c3aab09399dd"}, + {file = "SQLAlchemy-2.0.27-cp39-cp39-win_amd64.whl", hash = "sha256:ff2f1b7c963961d41403b650842dc2039175b906ab2093635d8319bef0b7d620"}, + {file = "SQLAlchemy-2.0.27-py3-none-any.whl", hash = "sha256:1ab4e0448018d01b142c916cc7119ca573803a4745cfe341b8f95657812700ac"}, + {file = "SQLAlchemy-2.0.27.tar.gz", hash = "sha256:86a6ed69a71fe6b88bf9331594fa390a2adda4a49b5c06f98e47bf0d392534f8"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +typing-extensions = ">=4.6.0" [package.extras] aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] mssql = ["pyodbc"] mssql-pymssql = ["pymssql"] mssql-pyodbc = ["pyodbc"] -mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] -mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql", "pymysql (<1)"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] sqlcipher = ["sqlcipher3_binary"] [[package]] @@ -1053,4 +1267,4 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "a6b91a958aabf9e75cade9a6aefb58d22c42379a8e09bda8aec8738d41df0f43" +content-hash = "83e123671a74e164919c325c59ddaeafb0f720216c713e851d343694264af300" diff --git a/api/fastapi/pyproject.toml b/api/fastapi/pyproject.toml index 2cf544e7..a82705f2 100644 --- a/api/fastapi/pyproject.toml +++ b/api/fastapi/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "oonidataapi" -version = "0.2.0.dev1" +version = "0.4.3.dev1" description = "" authors = ["OONI "] readme = "Readme.md" @@ -9,7 +9,7 @@ readme = "Readme.md" python = "^3.11" fastapi = "^0.108.0" clickhouse-driver = "^0.2.6" -sqlalchemy = "1.4.51" +sqlalchemy = "^2.0.27" ujson = "^5.9.0" urllib3 = "^2.1.0" python-dateutil = "^2.8.2" @@ -18,11 +18,15 @@ statsd = "^4.0.1" uvicorn = "^0.25.0" psycopg2 = "^2.9.9" httpx = "^0.26.0" +pyjwt = "^2.8.0" +alembic = "^1.13.1" [tool.poetry.group.dev.dependencies] pytest = "^7.4.4" docker = "^7.0.0" +pytest-cov = "^4.1.0" +click = "^8.1.7" [build-system] requires = ["poetry-core"] diff --git a/api/ooniapi/auth.py b/api/ooniapi/auth.py index ce729a56..a4e15aed 100644 --- a/api/ooniapi/auth.py +++ b/api/ooniapi/auth.py @@ -1,6 +1,7 @@ """ Authentication API """ + from datetime import datetime, timedelta from email.message import EmailMessage from functools import wraps