diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..bb92e4e7 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,136 @@ +name: Build and Publish Docker to ghcr + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + push: + branches: + - dev + # Publish semver tags as releases. + #tags: [ 'v*.*.*' ] + pull_request: + branches: + - build-actions + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build-test-publish-dev-backend: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + services: + mongodb: + image: mongo:latest + env: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: password + ports: + - 27017:27017 + + steps: + + - name: set datetime env + run: echo "NOW=$(date +'%Y.%m.%d.%H%M%S')" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - --version 1.7.1 + + - name: Configure Poetry + run: | + cd backend + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + + - name: Install backend dependencies + run: cd backend && poetry install + + - name: Run tests + run: | + cd backend + poetry run bash -c "cd .. && MONGO_URL=mongodb://root:password@localhost:27017 make test" + + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./backend + file: ./backend/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ghcr.io/promptsail/prompt_sail_dev:${{ env.NOW }} #${{ steps.meta.outputs.tags }} #"ghcr.io/promptsail/dev-${{ env.NOW }}" + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/backend/Dockerfile b/backend/Dockerfile index dfd451c0..ae90a2b1 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,57 +1,108 @@ -FROM python:3.10.2-slim-buster AS base +# syntax=docker/dockerfile:1 +# Keep this syntax directive! It's used to enable Docker BuildKit -ENV PYTHONUNBUFFERED=1 \ - POETRY_VERSION=1.1.13 - -WORKDIR /src - -# Install curl and Poetry in a single step, and clean up the apt cache to keep the image small -RUN apt-get update && apt-get install -y --no-install-recommends curl \ - && curl -sSL https://install.python-poetry.org | python3 - --version 1.7.1 \ - && ln -s /root/.local/bin/poetry /usr/local/bin/poetry \ - && apt-get purge -y --auto-remove curl \ - && rm -rf /var/lib/apt/lists/* - - -# Copy the Python dependency files into the image -COPY pyproject.toml poetry.lock ./ - - -# Create the virtual environment and export the dependency lists -RUN poetry config virtualenvs.create false \ - && poetry install --no-dev --no-interaction --no-ansi \ - && poetry export -f requirements.txt --without-hashes --output production.txt \ - && poetry export -f requirements.txt --dev --without-hashes --output all.txt \ - && comm -23 all.txt production.txt > development.txt - -# Set up a non-root user for security purposes -RUN useradd --create-home promptsail - -# Ensure the user has access to the workdir -RUN chown -R promptsail /src +# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865 +# but I try to keep it updated (see history) -# Switch to non-root user -USER promptsail +################################ +# PYTHON-BASE +# Sets up all our shared environment variables +################################ +FROM python:3.10.2-slim-buster as python-base -# Copy only the necessary files for running the application -COPY --chown=promptsail src /src/ -COPY --chown=promptsail static /static/ - -# Continue as new stage to minimize production image size -FROM base as production - -# ARGs are only available in the build stage they were declared in, so re-declare BUILD_SHA here -ARG BUILD_SHA - -# Set build SHA environment variable -ENV BUILD_SHA=${BUILD_SHA} - -# Copy the production requirements and install them -COPY --from=base --chown=promptsail src/production.txt ./ -RUN pip install --no-cache-dir -r production.txt - -# Expose the port the app runs on + # python +ENV PYTHONUNBUFFERED=1 \ + # prevents python creating .pyc files + PYTHONDONTWRITEBYTECODE=1 \ + \ + # pip + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + \ + # poetry + # https://python-poetry.org/docs/configuration/#using-environment-variables + POETRY_VERSION=1.7.1 \ + # make poetry install to this location + POETRY_HOME="/opt/poetry" \ + # make poetry create the virtual environment in the project's root + # it gets named `.venv` + POETRY_VIRTUALENVS_IN_PROJECT=true \ + # do not ask any interactive question + POETRY_NO_INTERACTION=1 \ + \ + # paths + # this is where our requirements + virtual environment will live + PYSETUP_PATH="/opt/pysetup" \ + VENV_PATH="/opt/pysetup/.venv" + + +# prepend poetry and venv to path +ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" + + +################################ +# BUILDER-BASE +# Used to build deps + create our virtual environment +################################ +FROM python-base as builder-base +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + # deps for installing poetry + curl \ + # deps for building python deps + build-essential + +# install poetry - respects $POETRY_VERSION & $POETRY_HOME +# The --mount will mount the buildx cache directory to where +# Poetry and Pip store their cache so that they can re-use it +RUN --mount=type=cache,target=/root/.cache \ + curl -sSL https://install.python-poetry.org | python3 - + +# copy project requirement files here to ensure they will be cached. +WORKDIR $PYSETUP_PATH +COPY pyproject.toml ./ + +# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally +RUN --mount=type=cache,target=/root/.cache \ + poetry install --only main + + +################################ +# DEVELOPMENT +# Image used during development / testing - as dev container +################################ +# FROM python-base as development +# ENV FASTAPI_ENV=development +# WORKDIR $PYSETUP_PATH + +# # copy in our built poetry + venv +# COPY --from=builder-base $POETRY_HOME $POETRY_HOME +# COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH + +# # quicker install as runtime deps are already installed +# RUN --mount=type=cache,target=/root/.cache \ +# poetry install --with dev + +# # will become mountpoint of our code +# WORKDIR /src + +# EXPOSE 8000 +# # Run the application +# CMD uvicorn app:app --proxy-headers --host 0.0.0.0 --port=${PORT:-8000} + + +################################ +# PRODUCTION +# Final image used for runtime +################################ +FROM python-base as production +ENV FASTAPI_ENV=production +COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH + +COPY src /src/ +COPY static /static/ +WORKDIR /src EXPOSE 8000 - # Run the application -CMD uvicorn app:app --proxy-headers --host 0.0.0.0 --port=${PORT:-8000} \ No newline at end of file +CMD uvicorn app:app --proxy-headers --host 0.0.0.0 --port=${PORT:-8000} +#CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "app:app"] \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b305906f..1538267a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -10,9 +10,6 @@ fastapi = "^0.103.2" uvicorn = {extras = ["stable"], version = "^0.23.2"} httpx = "^0.25.0" python-dotenv = "^1.0.0" -pytest = "^7.4.2" -pre-commit = "^3.5.0" -requests-mock = "^1.11.0" dependency-injector = "^4.41.0" pymongo = "^4.5.0" pydantic-settings = "^2.0.3" @@ -21,6 +18,18 @@ lato = "0.7.0" brotli = "^1.1.0" +[tool.poetry.group.dev] + + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.2" +pre-commit = "^3.5.0" +requests-mock = "^1.11.0" + + +[tool.poetry.group.examples] +optional = true + [tool.poetry.group.examples.dependencies] ipykernel = "^6.25.2" langchain = "^0.0"