Merge branch 'dev' of https://github.com/PromptSail/prompt_sail into dev

.github/workflows/docker-publish.yml

@@ -0,0 +1,136 @@
+name: Build and Publish Docker to ghcr
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  push:
+    branches: 
+        - dev
+    # Publish semver tags as releases.
+    #tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: 
+        - build-actions
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build-test-publish-dev-backend:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    services:
+      mongodb:
+        image: mongo:latest
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: password
+        ports:
+          - 27017:27017
+
+    steps:
+
+      - name: set datetime  env
+        run: echo "NOW=$(date +'%Y.%m.%d.%H%M%S')" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 - --version 1.7.1
+
+      - name: Configure Poetry
+        run: |
+          cd backend
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+
+      - name: Install backend dependencies
+        run: cd backend && poetry install
+
+      - name: Run tests
+        run: |
+          cd backend
+          poetry run bash -c "cd .. && MONGO_URL=mongodb://root:password@localhost:27017 make test"
+          
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1
+        with:
+          cosign-release: 'v2.1.1'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ghcr.io/promptsail/prompt_sail_dev:${{ env.NOW }} #${{ steps.meta.outputs.tags }} #"ghcr.io/promptsail/dev-${{ env.NOW }}"
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

backend/Dockerfile

@@ -1,57 +1,108 @@
-FROM python:3.10.2-slim-buster AS base
+# syntax=docker/dockerfile:1
+# Keep this syntax directive! It's used to enable Docker BuildKit
 
-ENV PYTHONUNBUFFERED=1 \
-    POETRY_VERSION=1.1.13 
-
-WORKDIR /src    
-
-# Install curl and Poetry in a single step, and clean up the apt cache to keep the image small
-RUN apt-get update && apt-get install -y --no-install-recommends curl \
-    && curl -sSL https://install.python-poetry.org | python3 - --version 1.7.1 \
-    && ln -s /root/.local/bin/poetry /usr/local/bin/poetry \
-    && apt-get purge -y --auto-remove curl \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Copy the Python dependency files into the image
-COPY pyproject.toml poetry.lock ./
-
-
-# Create the virtual environment and export the dependency lists
-RUN poetry config virtualenvs.create false \
-    && poetry install --no-dev --no-interaction --no-ansi \
-    && poetry export -f requirements.txt --without-hashes --output production.txt \
-    && poetry export -f requirements.txt --dev --without-hashes --output all.txt \
-    && comm -23 all.txt production.txt > development.txt
-
-# Set up a non-root user for security purposes
-RUN useradd --create-home promptsail
-
-# Ensure the user has access to the workdir
-RUN chown -R promptsail /src
+# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865
+# but I try to keep it updated (see history)
 
-# Switch to non-root user
-USER promptsail
+################################
+# PYTHON-BASE
+# Sets up all our shared environment variables
+################################
+FROM python:3.10.2-slim-buster as python-base
 
-# Copy only the necessary files for running the application
-COPY --chown=promptsail src /src/
-COPY --chown=promptsail static /static/
-
-# Continue as new stage to minimize production image size
-FROM base as production
-
-# ARGs are only available in the build stage they were declared in, so re-declare BUILD_SHA here
-ARG BUILD_SHA
-
-# Set build SHA environment variable
-ENV BUILD_SHA=${BUILD_SHA}
-
-# Copy the production requirements and install them
-COPY --from=base --chown=promptsail src/production.txt ./
-RUN pip install --no-cache-dir -r production.txt
-
-# Expose the port the app runs on
+    # python
+ENV PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    # pip
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # poetry
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.7.1 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # paths
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+
+################################
+# BUILDER-BASE
+# Used to build deps + create our virtual environment
+################################
+FROM python-base as builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        # deps for installing poetry
+        curl \
+        # deps for building python deps
+        build-essential
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+# The --mount will mount the buildx cache directory to where 
+# Poetry and Pip store their cache so that they can re-use it
+RUN --mount=type=cache,target=/root/.cache \
+    curl -sSL https://install.python-poetry.org | python3 -
+
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+COPY pyproject.toml ./
+
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install --only main
+
+
+################################
+# DEVELOPMENT
+# Image used during development / testing - as  dev container
+################################
+# FROM python-base as development
+# ENV FASTAPI_ENV=development
+# WORKDIR $PYSETUP_PATH
+
+# # copy in our built poetry + venv
+# COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+# COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# # quicker install as runtime deps are already installed
+# RUN --mount=type=cache,target=/root/.cache \
+#     poetry install --with dev
+
+# # will become mountpoint of our code
+# WORKDIR /src
+
+# EXPOSE 8000
+# # Run the application
+# CMD uvicorn app:app --proxy-headers --host 0.0.0.0 --port=${PORT:-8000}
+
+
+################################
+# PRODUCTION
+# Final image used for runtime
+################################
+FROM python-base as production
+ENV FASTAPI_ENV=production
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+COPY  src /src/
+COPY  static /static/
+WORKDIR /src
 EXPOSE 8000
-
 # Run the application
-CMD uvicorn app:app --proxy-headers --host 0.0.0.0 --port=${PORT:-8000}
+CMD uvicorn app:app --proxy-headers --host 0.0.0.0 --port=${PORT:-8000}
+#CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "app:app"]

backend/pyproject.toml

@@ -10,9 +10,6 @@ fastapi = "^0.103.2"
 uvicorn = {extras = ["stable"], version = "^0.23.2"}
 httpx = "^0.25.0"
 python-dotenv = "^1.0.0"
-pytest = "^7.4.2"
-pre-commit = "^3.5.0"
-requests-mock = "^1.11.0"
 dependency-injector = "^4.41.0"
 pymongo = "^4.5.0"
 pydantic-settings = "^2.0.3"
@@ -21,6 +18,18 @@ lato = "0.7.0"
 brotli = "^1.1.0"
 
 
+[tool.poetry.group.dev]
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.4.2"
+pre-commit = "^3.5.0"
+requests-mock = "^1.11.0"
+
+
+[tool.poetry.group.examples]
+optional = true
+
 [tool.poetry.group.examples.dependencies]
 ipykernel = "^6.25.2"
 langchain = "^0.0"