From e981bca0722fb3373db6a29930902416a43f45ee Mon Sep 17 00:00:00 2001 From: Ryan Rymarczyk Date: Tue, 14 Jan 2025 14:34:13 -0500 Subject: [PATCH] initial github ci jobs --- .envrc | 15 ++++-- .github/actions/python_deps/action.yaml | 30 +++++++++++ .github/actions/tools/action.yaml | 24 +++++++++ .github/workflows/ci_python.yaml | 67 +++++++++++++++++++++++++ README.md | 2 + src/odin/run.py | 4 +- src/odin/utils/aws/s3.py | 24 ++++----- 7 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 .github/actions/python_deps/action.yaml create mode 100644 .github/actions/tools/action.yaml create mode 100644 .github/workflows/ci_python.yaml diff --git a/.envrc b/.envrc index c2852d1..81c2133 100644 --- a/.envrc +++ b/.envrc @@ -8,9 +8,11 @@ layout python3 # this fuction allows project dependencies to be managed by pip-tools # a 'pyproject.toml' file is required in the root of the project function use_pip-tools() { + local python_version + python_version=$(python3 -c "import platform; print(platform.python_version())") # if VIRTUAL_ENV is not set to direnv default, delete direnv default dir as clean-up - if [[ "${VIRTUAL_ENV}" != *"$(direnv_layout_dir)/python-$python_version"* ]]; then - echo "custom VIRTUAL_ENV set, deleting $(direnv_layout_dir)" + if [[ "${VIRTUAL_ENV}" != *"$(direnv_layout_dir)/python-$python_version"* && -d $(direnv_layout_dir) ]]; then + echo "[use pip-tools] custom VIRTUAL_ENV set ($VIRTUAL_ENV), deleting ($(direnv_layout_dir))" rm -rf $(direnv_layout_dir) fi @@ -31,20 +33,23 @@ function use_pip-tools() { fi if [ ! -f "pyproject.toml" ]; then - echo "[use pip-tools] ERROR: No 'pyproject.toml' file exists, please create..." + echo "[use pip-tools] ERROR: No 'pyproject.toml' file exists, please create one!" return 1 fi - echo "[use pip-tools] resyncing (dev)requirements" + echo "[use pip-tools] syncing Python (dev)requirements..." # create requirements files pip-compile --quiet --strip-extras -o requirements.txt pip-compile --quiet --strip-extras --extra=dev -o requirements-dev.txt # install all pinned requirements to virtual environment pip-sync requirements.txt requirements-dev.txt # install local project from /src as editable, so code changes are reflected without re-install + echo "[use pip-tools] installing project to Python virtual environment ($VIRTUAL_ENV)..." pip install --editable . watch_file pyproject.toml + echo "[use pip-tools] Python virtual environment sucessfully created, updated and activated!" + echo "[use pip-tools] Running 'python --version' should show 'Python $python_version'" } -use pip-tools \ No newline at end of file +use pip-tools diff --git a/.github/actions/python_deps/action.yaml b/.github/actions/python_deps/action.yaml new file mode 100644 index 0000000..2c7745e --- /dev/null +++ b/.github/actions/python_deps/action.yaml @@ -0,0 +1,30 @@ +name: Setup Python Dependencies +description: Loads python dependencies for a CI/CD job, install them if not cached + +runs: + using: composite + steps: + - name: ASDF Tools Install + uses: ./.github/actions/tools + + - name: Python Deps Cache + uses: actions/cache@v3 + id: python-cache + with: + path: ./.venv + key: ${{ runner.os }}-python-deps-${{ hashFiles('**/requirements*.txt') }} + + - run: python3 -m venv .venv + if: "!steps.python-cache.outputs.cache-hit" + shell: bash + + - run: | + source .venv/bin/activate + echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV + echo "${VIRTUAL_ENV}/bin" >> $GITHUB_PATH + shell: bash + + - run: pip install -r requirements.txt -r requirements-dev.txt + if: "!steps.python-cache.outputs.cache-hit" + shell: bash + diff --git a/.github/actions/tools/action.yaml b/.github/actions/tools/action.yaml new file mode 100644 index 0000000..9e771cc --- /dev/null +++ b/.github/actions/tools/action.yaml @@ -0,0 +1,24 @@ +name: Setup ASDF Tools +description: Loads ASDF tools for for a CI/CD job, installing them if not cached +outputs: + cache-hit: + description: "Whether the ASDF cache was hit" + value: ${{ steps.asdf-cache.outputs-cache-hit }} +runs: + using: composite + steps: + # cache the ASDF directory, using values from .tool-versions + - name: ASDF Tools Cache + uses: actions/cache@v3 + id: asdf-cache + with: + path: ~/.asdf + # runner.os vs CACHE_UUID secret + key: ${{ runner.os}}-asdf-${{ hashFiles('**/.tool-versions') }} + + - name: Install ASDF Tools + uses: asdf-vm/actions/install@v2 + if: steps.asdf-cache.outputs.cache-hit != 'true' + + - name: Re-shim ASDF Install + uses: mbta/actions/reshim-asdf@v1 diff --git a/.github/workflows/ci_python.yaml b/.github/workflows/ci_python.yaml new file mode 100644 index 0000000..bb76b0c --- /dev/null +++ b/.github/workflows/ci_python.yaml @@ -0,0 +1,67 @@ +name: Continuous Integration (Python) + +on: + push: + branches: + - main + paths: + - 'src/**' + - 'pyproject.toml' + - 'requirements-dev.txt' + - '.github/workflows/ci_python.yaml' + - '.github/python_deps/action.yaml' + pull_request: + paths: + - 'src/**' + - 'pyproject.toml' + - 'requirements-dev.txt' + - '.github/workflows/ci_python.yaml' + - '.github/python_deps/action.yaml' + +defaults: + run: + shell: bash + working-directory: . + +concurrency: + group: python-ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + setup: + name: Python Setup + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/python_deps + + format: + name: Format + runs-on: ubuntu-22.04 + needs: setup + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/python_deps + + - run: ruff format --diff . + + typing: + name: Type Check + runs-on: ubuntu-22.04 + needs: setup + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/python_deps + + - run: mypy . + + lint: + name: Lint + runs-on: ubuntu-22.04 + needs: setup + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/python_deps + + - run: ruff check --diff . + \ No newline at end of file diff --git a/README.md b/README.md index 01bca47..4ad40ea 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ asdf install `pip-tools` is used by to manage python dependencies. > `direnv` will automatically create (and activate) a python virutal environemnt in the project folder for local development. `pip-tools` is also automatically run (by `direnv`) to install project dependencies when moving into the project directory. +> +> `pip-tools` creates [requirements.txt](requirements.txt) and [requirements-dev.txt](requirements-dev.txt) files containing versions and hashes of all python library dependencies for the project. `docker` is required to run containerized version of application for local development. diff --git a/src/odin/run.py b/src/odin/run.py index d55b063..906dcde 100644 --- a/src/odin/run.py +++ b/src/odin/run.py @@ -3,7 +3,9 @@ from odin.utils.logger import ProcessLog from odin.utils.runtime import validate_env_vars + def start(): + """Application Entry""" os.environ["SERVICE_NAME"] = "odin" validate_env_vars( required=[], @@ -13,4 +15,4 @@ def start(): if __name__ == "main": - start() \ No newline at end of file + start() diff --git a/src/odin/utils/aws/s3.py b/src/odin/utils/aws/s3.py index 0de3926..6bdebef 100644 --- a/src/odin/utils/aws/s3.py +++ b/src/odin/utils/aws/s3.py @@ -27,7 +27,7 @@ def get_client() -> BaseClient: def split_object(object: str) -> Tuple[str, str]: """ - Split s3 object as "s3://bucket/object_key" into Tuple[bucket, key]. + Split S3 object as "s3://bucket/object_key" into Tuple[bucket, key]. :param object: s3 object as "s3://bucket/object_key" or "bucket/object_key" @@ -39,16 +39,14 @@ def split_object(object: str) -> Tuple[str, str]: def list_objects( - bucket: str, - prefix: str, + partition: str, max_objects: int = 1_000_000, in_filter: Optional[str] = None, ) -> List[str]: """ - Get list of S3 objects in 'bucket' starting with 'prefix' + Get list of S3 objects starting with 'partition'. - :param bucket: the name of the bucket with objects - :param prefix: prefix for objs to return + :param partition: S3 partition as "s3://bucket/prefix" or "bucket/prefix" :param max_objects: maximum number of objects to return :param in_filter: will filter for objects containing string @@ -56,11 +54,11 @@ def list_objects( """ logger = ProcessLog( "list_objects", - bucket=bucket, - prefix=prefix, + partition=partition, max_objects=max_objects, in_filter=in_filter, ) + bucket, prefix = split_object(partition) try: client = get_client() paginator = client.get_paginator("list_objects_v2") @@ -111,7 +109,7 @@ def object_exists(object: str) -> bool: def upload_file(file_name: str, object: str, extra_args: Optional[Dict] = None) -> bool: """ - Upload a local file to an S3 Bucket + Upload a local file to S3 as an object. :param file_name: local file path to upload :param object: S3 object path as 's3://bucket/object' or 'bucket/object' @@ -173,9 +171,9 @@ def download_object(object: str, local_path: str) -> bool: return False -def get_object(object: str) -> StreamingBody: +def stream_object(object: str) -> StreamingBody: """ - Get an S3 object as StreamingBody + Stream an S3 object as StreamingBody. :param object: S3 object path as 's3://bucket/object' or 'bucket/object' @@ -196,7 +194,7 @@ def get_object(object: str) -> StreamingBody: def delete_object(object: str) -> bool: """ - Delete s3 object + Delete an S3 object. :param object: S3 object to delete as 's3://bucket/object' or 'bucket/object' @@ -217,7 +215,7 @@ def delete_object(object: str) -> bool: def rename_object(from_object: str, to_object: str) -> bool: """ - Rename from_object TO to_object as copy and delete operation. + Rename an S3 object as copy and delete operation. :param from_object: COPY from as 's3://bucket/object' or 'bucket/object' :param to_object: COPY to as 's3://bucket/object' or 'bucket/object'