Sofatutor specific changes

.github/workflows/pullpreview.yml

@@ -0,0 +1,34 @@
+name: preview
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [labeled, unlabeled, synchronize, reopened]
+
+env:
+  DNS: 'previews.sofatutor.us'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'push' || github.event.label.name == 'preview' || contains(github.event.pull_request.labels.*.name, 'preview')
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: sofatutor/pullpreview@sofatutor
+        with:
+          admins: mfittko,Ayushi1296
+          always_on: main
+          app_path: .
+          instance_type: small
+          dns: ${{ env.DNS }}
+          default_port: 8080
+          registries: docker://index.docker.io/v1
+        env:
+          AWS_REGION: eu-central-1
+          PULLPREVIEW_ENV_VARS: "DOMAIN=${{ env.DNS }},AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }},AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }},OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }},LITELLM_MASTER_KEY=${{ secrets.LITELLM_MASTER_KEY }},LITELLM_SALT_KEY=${{ secrets.LITELLM_SALT_KEY }}"
+          PULLPREVIEW_LAUNCH_COMMAND: COMPOSE_FILE=docker-compose.pullpreview.yml docker-compose up -d --build --force-recreate
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"

.github/workflows/sofatutor_image.yml

@@ -0,0 +1,59 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+---
+name: Create and publish a Container image
+
+on:
+  push:
+    branches:
+      - sofatutor
+    tags:
+      - "v*"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository_owner }}/${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: sofatutor/docker-login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: sofatutor/docker-metadata-action@57396166ad8aefe6098280995947635806a0e6ea
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push Docker image
+        uses: sofatutor/docker-build-push-action@c56af957549030174b10d6867f20e78cfd7debc5
+        with:
+          context: .
+          push: true
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "LiteLLM Proxy",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "litellm.proxy.proxy_cli",
+      "args": [
+        "--config",
+        "${workspaceFolder}/sofatutor_config.yaml",
+        "--debug"
+      ],
+      "env": {
+        "LITELLM_LOG": "DEBUG",
+        "LITELLM_MASTER_KEY": "test",
+        "DATABASE_URL": "postgresql://llmproxy:dbpassword9090@localhost:5433/litellm",
+        "STORE_MODEL_IN_DB": "True"
+      },
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal"
+    }
+  ]
+}

Dockerfile

@@ -1,76 +1,62 @@
-# Base image for building
 ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim
-
-# Runtime image
 ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim
-# Builder stage
-FROM $LITELLM_BUILD_IMAGE AS builder
 
-# Set the working directory to /app
-WORKDIR /app
+FROM $LITELLM_BUILD_IMAGE as builder
 
-# Install build dependencies
-RUN apt-get clean && apt-get update && \
-    apt-get install -y gcc python3-dev && \
-    rm -rf /var/lib/apt/lists/*
+WORKDIR /app
 
-RUN pip install --upgrade pip && \
-    pip install build
+RUN --mount=type=cache,target=/var/cache/apt,id=litellm_apt \
+    apt-get update && \
+    apt-get install -y gcc python3-dev
 
-# Copy the current directory contents into the container at /app
-COPY . .
+RUN --mount=type=cache,target=/root/.cache/pip,id=litellm_pip \
+    pip install --upgrade pip build
 
-# Build Admin UI
-RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
+COPY requirements.txt .
 
-# Build the package
-RUN rm -rf dist/* && python -m build
+RUN --mount=type=cache,target=/root/.cache/pip,id=litellm_pip \
+    pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 
-# There should be only one wheel file now, assume the build only creates one
-RUN ls -1 dist/*.whl | head -1
+RUN --mount=type=cache,target=/root/.cache/pip,id=litellm_pip \
+    pip install redisvl==0.0.7 --no-deps
 
-# Install the package
-RUN pip install dist/*.whl
+RUN pip uninstall jwt PyJWT -y && \
+    pip install PyJWT --no-cache-dir
 
-# install dependencies as wheels
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+COPY . .
 
-# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0 
-RUN pip install redisvl==0.0.7 --no-deps
+RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
 
-# ensure pyjwt is used, not jwt
-RUN pip uninstall jwt -y
-RUN pip uninstall PyJWT -y
-RUN pip install PyJWT --no-cache-dir
+RUN rm -rf dist/* && python -m build
 
-# Build Admin UI
-RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
+RUN --mount=type=cache,target=/root/.cache/pip,id=litellm_pip \
+    pip install dist/*.whl
 
-# Runtime stage
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
+FROM $LITELLM_RUNTIME_IMAGE as runtime
 
-# Update dependencies and clean up - handles debian security issue
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* 
+# Install runtime dependencies, e.g. curl for healthcheck
+RUN apt-get clean && apt-get update && \
+    apt-get install -y curl && \
+    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
-# Copy the current directory contents into the container at /app
-COPY . .
-RUN ls -la /app
 
-# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
-COPY --from=builder /app/dist/*.whl .
 COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app/dist/*.whl .
 
-# Install the built wheel using pip; again using a wildcard if it's the only file
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+RUN --mount=type=cache,target=/root/.cache/pip,id=litellm_pip \
+    pip install *.whl /wheels/* --no-index --find-links=/wheels/ && \
+    rm -f *.whl && \
+    rm -rf /wheels
+
+COPY . .
 
-# Generate prisma client
 RUN prisma generate
+
 RUN chmod +x entrypoint.sh
 
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["litellm"]
 
-# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
 CMD ["--port", "4000"]

docker-compose.pullpreview.yml

@@ -0,0 +1,73 @@
+services:
+  litellm:
+    build:
+      context: .
+      args:
+        target: runtime
+    image: ghcr.io/sofatutor/sofatutor/litellm:sofatutor
+    deploy:
+      replicas: 2
+      update_config:
+        parallelism: 1
+        order: stop-first
+        failure_action: rollback
+        delay: 30s
+      restart_policy:
+        condition: on-failure
+    environment:
+      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      STORE_MODEL_IN_DB: "True"
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      OPENAI_API_KEY: ${OPENAI_API_KEY}
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY}
+      LITELLM_LOG: "DEBUG"
+      REDIS_HOST: "redis"
+      REDIS_PORT: "6379"
+    command: ["--port", "4000", "--config", "sofatutor_config.yaml"]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:4000/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+    depends_on:
+      - redis
+      - db
+
+  nginx:
+    image: nginx:latest
+    ports:
+      - "8080:80"
+    volumes:
+      - ./docker/nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - litellm
+
+  db:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: llmproxy
+      POSTGRES_PASSWORD: dbpassword9090
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5433:5432"
+
+  redis:
+    image: redis:7-alpine
+    volumes:
+      - redis_data:/data
+    command: redis-server --appendonly yes
+
+volumes:
+  redis_data:
+  postgres_data:

docker/nginx.conf

@@ -0,0 +1,16 @@
+events {
+    worker_connections 1000;
+}
+
+http {
+    upstream litellm_backend {
+        server litellm:4000;
+    }
+
+    server {
+        listen 80;
+        location / {
+            proxy_pass http://litellm_backend;
+        }
+    }
+}