pre-built self-host images

lmnr-ai · Sep 25, 2024 · 94c427f · 94c427f
1 parent da1871b
commit 94c427f
Show file tree

Hide file tree

Showing 4 changed files with 199 additions and 13 deletions.
diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
@@ -0,0 +1,64 @@
+name: Build & publish lmnr images
+on:
+  release:
+    types:
+      - published
+
+env:
+  REGISTRY_GH: ghcr.io
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - dockerfile: ./app-server/Dockerfile
+            context: ./app-server
+            image: ghcr.io/lmnr-ai/app-server
+          - dockerfile: ./frontend/Dockerfile
+            context: ./frontend
+            image: ghcr.io/lmnr-ai/frontend
+          - dockerfile: ./semantic-search-service/Dockerfile
+            context: ./semantic-search-service
+            image: ghcr.io/lmnr-ai/semantic-search-service
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY_GH }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ matrix.image }}
+
+      - name: Build and push Docker images
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ matrix.image }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
diff --git a/README.md b/README.md
@@ -40,6 +40,16 @@ This will spin up the following containers:
 - postgres – the database for all the application data
 - clickhouse – columnar OLAP database for more efficient event and trace analytics
 
+#### Local development
+
+The simple set up above will pull latest Laminar images from Github Container Registry.
+
+If you want to test your local changes, you will need to build from source using [Local docker compose](./docker-compose-local-dev.yml)
+
+```sh
+docker compose -f docker-compose-local-dev.yml up --build
+```
+
 ### Instrumenting Python code
 
 First, create a project and generate a Project API Key. Then,

diff --git a/docker-compose-local-dev.yml b/docker-compose-local-dev.yml
@@ -0,0 +1,122 @@
+name: lmnr
+
+services:
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - "6333:6333"
+      - "6334:6334"
+    volumes:
+      - type: volume
+        source: qdrant-data
+        target: /data
+
+  rabbitmq:
+    image: rabbitmq
+    ports:
+      - "5672:5672"
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 7s
+      timeout: 5s
+      retries: 3
+
+  clickhouse:
+    build:
+      context: ./clickhouse
+    container_name: clickhouse
+    ports:
+      - "8123:8123"
+    volumes:
+      - type: volume
+        source: clickhouse-data
+        target: /var/lib/clickhouse/
+      - type: volume
+        source: clickhouse-logs
+        target: /var/log/clickhouse-server/
+    cap_add:
+      - SYS_NICE
+      - NET_ADMIN
+      - IPC_LOCK
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+
+  semantic-search-service:
+    build:
+      context: ./semantic-search-service
+    container_name: semantic-search-service
+    ports:
+      - "8080:8080"
+    depends_on:
+      - qdrant
+    environment:
+      - PORT=8080
+      - QDRANT_URL=http://qdrant:6334
+      - COHERE_ENDPOINT=https://api.cohere.ai/v1/embed
+      - COHERE_API_KEY=${COHERE_API_KEY}
+
+  postgres:
+    build:
+      context: ./postgres
+      args:
+        POSTGRES_USER: ${POSTGRES_USER}
+        POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+        POSTGRES_DB: ${POSTGRES_DB}
+    container_name: postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - type: volume
+        source: postgres-data
+        target: /var/lib/postgresql/data
+
+  app-server:
+    build:
+      context: ./app-server
+      args:
+        DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
+    container_name: app-server
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+    depends_on:
+      semantic-search-service:
+        condition: service_started
+      postgres:
+        condition: service_started
+      rabbitmq:
+        condition: service_healthy
+      clickhouse:
+        condition: service_started
+    environment:
+      - PORT=8000
+      - GRPC_PORT=8001
+      - DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
+      - SEMANTIC_SEARCH_URL=http://semantic-search-service:8080
+      - RABBITMQ_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/%2f
+      - FRONTEND_SHARED_SECRET=${SHARED_SECRET}
+      - CLICKHOUSE_URL=http://clickhouse:8123
+      - CLICKHOUSE_USER=${CLICKHOUSE_USER}
+
+  frontend:
+    build:
+      context: ./frontend
+    container_name: frontend
+    ports:
+      - "3000:3000"
+    env_file: ./frontend/.env.local.example
+    environment:
+      - PORT=3000
+      - BACKEND_URL=http://app-server:8000
+      - BACKEND_SHARED_SECRET=${SHARED_SECRET}
+
+volumes:
+  qdrant-data:
+  clickhouse-data:
+  clickhouse-logs:
+  postgres-data:
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -46,12 +46,8 @@ services:
         soft: 262144
         hard: 262144
 
-
-
   semantic-search-service:
-    build:
-      context: ./semantic-search-service
-    container_name: semantic-search-service
+    image: ghcr.io/lmnr-ai/semantic-search-service
     ports:
       - "8080:8080"
     depends_on:
@@ -78,11 +74,7 @@ services:
         target: /var/lib/postgresql/data
 
   app-server:
-    build:
-      context: ./app-server
-      args:
-        DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
-    container_name: app-server
+    image: ghcr.io/lmnr-ai/app-server
     ports:
       - "8000:8000"
       - "8001:8001"
@@ -106,9 +98,7 @@ services:
       - CLICKHOUSE_USER=${CLICKHOUSE_USER}
 
   frontend:
-    build:
-      context: ./frontend
-    container_name: frontend
+    image: ghcr.io/lmnr-ai/frontend
     ports:
       - "3000:3000"
     env_file: ./frontend/.env.local.example