Break VMClarity control plane into microservices (openclarity#481)

* Rename backend to apiserver This PR renames the vmclarity-backend cmd and backend packages to vmclarity-apiserver and apiserver respectively in preparation to split the orchestrator and uibackend out of the backend monolith. * Break VMClarity control plane into microservices This commit breaks VMClarity backend into several new microservices: * vmclarity-apiserver - Serves the VMClarity API and connects to the DB * vmclarity-orchestrator - All the controllers for reconciling scans * vmclarity-uibackend - A UI specific API wrapper service * vmclarity-ui - An nginx server responsible for serving the UI static files An example docker compose installer is added to deploy the VMClarity stack locally. * Update AWS installer to use the microservices * Microservices AWS working * Azure microservices working * GCP microservices working * Update README after microservices * Update docker compose with all services and configurations * Code review comments addressed * More code review comments * fix docker compose grype-server
lgecse · Aug 2, 2023 · cb5640c · cb5640c
1 parent 9e1263b
commit cb5640c
Show file tree

Hide file tree

Showing 82 changed files with 2,220 additions and 1,606 deletions.
diff --git a/.github/workflows/build-and-push-component.yaml b/.github/workflows/build-and-push-component.yaml
@@ -0,0 +1,80 @@
+name: 'Build and Push Component'
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+        description: 'Dockerfile to build and push'
+      image_name:
+        required: true
+        type: string
+        description: 'Name of the image to publish'
+      image_tag:
+        required: true
+        type: string
+        description: 'Image tag to build and push.'
+      push:
+        required: false
+        type: string
+        description: 'If set to true, push the image.'
+        default: false
+      upload:
+        required: false
+        type: string
+        description: 'If set to true, upload the image.'
+        default: false
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.ref }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set build output env var
+        if: ${{ inputs.upload == 'true' }}
+        run: |
+          echo "OUTPUTS=type=docker,dest=/tmp/${{ inputs.image_name }}.tar" >> $GITHUB_ENV
+
+      - name: Build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/openclarity/${{ inputs.image_name }}:${{ inputs.image_tag }}
+          file: ${{ inputs.dockerfile }}
+          push: ${{ inputs.push }}
+          outputs: "${{ env.OUTPUTS }}"
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          build-args: |
+            VERSION=${{ inputs.image_tag }}
+            BUILD_TIMESTAMP=${{ needs.timestamp.outputs.timestamp }}
+            COMMIT_HASH=${{ github.sha }}
+
+      - name: Upload artifact
+        if: ${{ inputs.upload == 'true' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image_name }}
+          path: /tmp/${{ inputs.image_name }}.tar
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
 
       - name: Generate API code
-        run: make api
+        run: make gen-api
 
       # This step will evaluate the repo status and exit if found changes
       # This should detect if the most up-to-date generated API code was pushed

diff --git a/.github/workflows/reusable-build-and-push.yml b/.github/workflows/reusable-build-and-push.yml
@@ -28,106 +28,52 @@ jobs:
         id: timestamp
         run: echo "::set-output name=timestamp::$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
 
-  vmclarity-backend:
+  vmclarity-apiserver:
     needs: timestamp
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.ref }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set build output env var
-        if: ${{ inputs.upload == 'true' }}
-        run: |
-          echo "OUTPUTS=type=docker,dest=/tmp/vmclarity.tar" >> $GITHUB_ENV
-
-      - name: Build
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          tags: ghcr.io/openclarity/vmclarity-backend:${{ inputs.image_tag }}
-          file: Dockerfile.backend
-          push: ${{ inputs.push }}
-          outputs: "${{ env.OUTPUTS }}"
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-          build-args: |
-            VERSION=${{ inputs.image_tag }}
-            BUILD_TIMESTAMP=${{ needs.timestamp.outputs.timestamp }}
-            COMMIT_HASH=${{ github.sha }}
-
-      - name: Upload artifact
-        if: ${{ inputs.upload == 'true' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: vmclarity
-          path: /tmp/vmclarity.tar
+    uses: ./.github/workflows/build-and-push-component.yaml
+    with:
+      dockerfile: Dockerfile.apiserver
+      image_name: vmclarity-apiserver
+      image_tag: ${{ inputs.image_tag }}
+      push: ${{ inputs.push }}
+      upload: ${{ inputs.upload }}
+
+  vmclarity-orchestrator:
+    needs: timestamp
+    uses: ./.github/workflows/build-and-push-component.yaml
+    with:
+      dockerfile: Dockerfile.orchestrator
+      image_name: vmclarity-orchestrator
+      image_tag: ${{ inputs.image_tag }}
+      push: ${{ inputs.push }}
+      upload: ${{ inputs.upload }}
+
+  vmclarity-ui-backend:
+    needs: timestamp
+    uses: ./.github/workflows/build-and-push-component.yaml
+    with:
+      dockerfile: Dockerfile.uibackend
+      image_name: vmclarity-ui-backend
+      image_tag: ${{ inputs.image_tag }}
+      push: ${{ inputs.push }}
+      upload: ${{ inputs.upload }}
+
+  vmclarity-ui:
+    needs: timestamp
+    uses: ./.github/workflows/build-and-push-component.yaml
+    with:
+      dockerfile: Dockerfile.ui
+      image_name: vmclarity-ui
+      image_tag: ${{ inputs.image_tag }}
+      push: ${{ inputs.push }}
+      upload: ${{ inputs.upload }}
 
   vmclarity-cli:
     needs: timestamp
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.ref }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set build output env var
-        if: ${{ inputs.upload == 'true' }}
-        run: |
-          echo "OUTPUTS=type=docker,dest=/tmp/vmclarity-cli.tar" >> $GITHUB_ENV
-
-      - name: Build
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          tags: ghcr.io/openclarity/vmclarity-cli:${{ inputs.image_tag }}
-          file: Dockerfile.cli
-          push: ${{ inputs.push }}
-          outputs: "${{ env.OUTPUTS }}"
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-          build-args: |
-            COMMIT_HASH=${{ github.sha }}
-
-      - name: Upload artifact
-        if: ${{ inputs.upload == 'true' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: vmclarity
-          path: /tmp/vmclarity-cli.tar
+    uses: ./.github/workflows/build-and-push-component.yaml
+    with:
+      dockerfile: Dockerfile.cli
+      image_name: vmclarity-cli
+      image_tag: ${{ inputs.image_tag }}
+      push: ${{ inputs.push }}
+      upload: ${{ inputs.upload }}
diff --git a/.golangci.yml b/.golangci.yml
@@ -76,6 +76,6 @@ issues:
     - path: _test\.go
       linters:
         - govet
-    - path: pkg/backend/database/demo.go
+    - path: pkg/apiserver/database/demo.go
       linters:
         - gomnd
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -5,7 +5,11 @@ builds:
     binary: vmclarity-cli
     env:
       - CGO_ENABLED=0
-    ldflags: "-s -w -X github.com/openclarity/vmclarity/pkg/cli.GitRevision={{ .Version }}"
+    ldflags:
+      - "-s -w"
+      - "-X github.com/openclarity/vmclarity/pkg/version.Version={{ .Version }}"
+      - "-X github.com/openclarity/vmclarity/pkg/version.CommitHash={{.Commit}}"
+      - "-X github.com/openclarity/vmclarity/pkg/version.BuildTimestamp={{.Timestamp}}"
     goos:
       - linux
       - darwin

diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
@@ -1,33 +1,30 @@
 # High Level Architecture
 
-Today, VMClarity has two halves, the VMClarity infrastructure, and the VMClarity CLI.
+Today, VMClarity has two halves, the VMClarity control plane, and the
+VMClarity CLI.
 
-The VMClarity infrastructure includes:
+The VMClarity control plane includes several microservices:
 
-- **Backend**: The core component of VMClarity. Within this service there are
-  sub-components (it is in the roadmap to break these into dedicated microservices):
+- **API Server**: The VMClarity API for managing all objects in the VMClarity
+  system. This is the only component in the system which talks to the DB.
 
-    - **API**: The VMClarity API for managing all objects in the VMClarity
-      system. This is the only component in the system which talks to the DB.
+- **Orchestrator**: Orchestrates and manages the life cycle of VMClarity
+  scan configs, scans and asset scans. Within the Orchestrator there is a
+  pluggable "provider" which connects the orchstrator to the environment to be
+  scanned and abstracts asset discovery, VM snapshotting as well as creation of
+  the scanner VMs. (**Note** The only supported provider today is AWS, other
+  hyperscalers are on the roadmap)
 
-    - **Orchestrator**: Orchestrates and manages the life cycle of VMClarity scan
-      configs, scans and asset scans. Within the Orchestrator there is a
-      pluggable "provider" which connects the orchstrator to the environment to be
-      scanned and abstracts asset discovery, VM snapshotting as well as creation of
-      the scanner VMs. (**Note** The only supported provider today is AWS, other
-      hyperscalers are on the roadmap)
+- **UI Backend**: A separate backend API which offloads some processing from
+  the browser to the infrastructure to process and filter data closer to the
+  source.
 
-    - **UI Backend**: A separate backend API which offloads some processing from
-      the browser to the infrastructure to process and filter data closer to the
-      source.
+- **UI Webserver**: A server serving the UI static files.
 
-    - **UI Server**: A server serving the UI static files.
+- **DB**: Stores the VMClarity objects from the API. Supported options are
+  SQLite and Postgres.
 
-- **DB**: Stores the VMClarity objects from the API. Today this is SQLite but
-  the database interface in VMClarity is pluggable and additional DB support
-  can be added. (Postgres is in the roadmap)
-
-- **Scanner services**: These services provide support to the VMClarity
+- **Scanner Helper services**: These services provide support to the VMClarity
   CLI to offload work that would need to be done in every scanner, for example
   downloading the latest vulnerability or malware signatures from the various DB
   sources. The components included today are:
@@ -54,4 +51,4 @@ the configured analysis on the mounted snapshot, and report the results to the
 VMClarity API. These results are then processed by the VMClarity backend into
 findings.
 
-![VMClarity Architecture Overview](img/vmclarity-arch-20230406.svg)
+![VMClarity Architecture Overview](img/vmclarity-arch-20230725.svg)
diff --git a/Dockerfile.backend → Dockerfile.apiserver b/Dockerfile.backend → Dockerfile.apiserver
@@ -1,12 +1,4 @@
 # syntax=docker/dockerfile:1.2
-FROM --platform=$BUILDPLATFORM node:20-slim AS site-build
-
-WORKDIR /app/ui-build
-
-COPY ui .
-RUN npm i
-RUN npm run build
-
 # xx is a helper for cross-compilation
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.2.1@sha256:8879a398dedf0aadaacfbd332b29ff2f84bc39ae6d4e9c0a1109db27ac5ba012 AS xx
 
@@ -33,17 +25,16 @@ ENV CGO_ENABLED=1
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     go build -ldflags="-s -w -extldflags -static \
-     -X 'github.com/openclarity/vmclarity/pkg/backend/version.Version=${VERSION}' \
-     -X 'github.com/openclarity/vmclarity/pkg/backend/version.CommitHash=${COMMIT_HASH}' \
-     -X 'github.com/openclarity/vmclarity/pkg/backend/version.BuildTimestamp=${BUILD_TIMESTAMP}'" -o bin/vmclarity-backend ./cmd/vmclarity-backend/main.go
+     -X 'github.com/openclarity/vmclarity/pkg/version.Version=${VERSION}' \
+     -X 'github.com/openclarity/vmclarity/pkg/version.CommitHash=${COMMIT_HASH}' \
+     -X 'github.com/openclarity/vmclarity/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}'" -o bin/vmclarity-apiserver ./cmd/vmclarity-apiserver/main.go
 
-RUN xx-verify bin/vmclarity-backend
+RUN xx-verify bin/vmclarity-apiserver
 
 FROM alpine:3.18
 
 WORKDIR /app
 
-COPY --from=builder ["/build/bin/vmclarity-backend", "./vmclarity-backend"]
-COPY --from=site-build ["/app/ui-build/build", "site"]
+COPY --from=builder ["/build/bin/vmclarity-apiserver", "./vmclarity-apiserver"]
 
-ENTRYPOINT ["/app/vmclarity-backend"]
+ENTRYPOINT ["/app/vmclarity-apiserver"]
diff --git a/Dockerfile.cli b/Dockerfile.cli
@@ -2,14 +2,15 @@
 ARG VMCLARITY_TOOLS_BASE=ghcr.io/openclarity/vmclarity-tools-base:v0.2.0@sha256:0e0ed706dc297366af44d736c71aefa350b54a0214290aa81b3603462e39872b
 FROM --platform=$BUILDPLATFORM golang:1.20.6-alpine AS builder
 
-
 RUN apk add --update --no-cache ca-certificates git
 RUN apk add build-base
 
 # Copy vmclarity code to /build and move to that directory
 COPY . /build
 WORKDIR /build
 
+ARG VERSION
+ARG BUILD_TIMESTAMP
 ARG COMMIT_HASH
 ARG TARGETOS
 ARG TARGETARCH
@@ -18,9 +19,11 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \
     go build \
-    -ldflags="-s -w -X 'github.com/openclarity/vmclarity/pkg/cli.GitRevision=${COMMIT_HASH}'" \
-    -o bin/vmclarity-cli \
-    cmd/vmclarity-cli/main.go
+    -ldflags="-s -w \
+        -X 'github.com/openclarity/vmclarity/pkg/version.Version=${VERSION}' \
+        -X 'github.com/openclarity/vmclarity/pkg/version.CommitHash=${COMMIT_HASH}' \
+        -X 'github.com/openclarity/vmclarity/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}'" \
+    -o bin/vmclarity-cli cmd/vmclarity-cli/main.go
 
 FROM ${VMCLARITY_TOOLS_BASE}