Feature/dockerfile (#156)

dockerfile for atum server, sslcontext
AbsaOSS · Feb 13, 2024 · a0257c6 · a0257c6
1 parent eae92e4
commit a0257c6
Show file tree

Hide file tree

Showing 22 changed files with 397 additions and 163 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-name: Build
+name: Build all modules
 
 on:
   pull_request:
@@ -23,9 +23,26 @@ on:
     types: [ opened, synchronize, reopened ]
 
 jobs:
-  test:
+  build-agent-and-model:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        scala: [2.11.12, 2.12.18, 2.13.11]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: coursier/cache-action@v5
 
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v14
+        with:
+          java-version: "[email protected]"
+
+      - name: Build and run tests
+        run: sbt "project agent" ++${{matrix.scala}} test doc "project model" ++${{matrix.scala}} test doc
+
+  build-server:
+    runs-on: ubuntu-latest
     services:
       postgres:
         image: postgres:15
@@ -39,26 +56,21 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
-
     strategy:
-      fail-fast: false
       matrix:
-        scala: [2.11.12, 2.12.18, 2.13.11]
-
-    name: Scala ${{matrix.scala}}
-
+        scala: [2.13.11]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
       - uses: coursier/cache-action@v5
 
       - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: olafurpg/setup-scala@v14
         with:
-          java-version: "adopt@1.8"
+          java-version: "adopt@1.11.0-11"
 
       - name: Prepare testing database
         run: sbt flywayMigrate
 
       - name: Build and run tests
-        run: sbt ++${{matrix.scala}} test doc
+        run: sbt "project server" ++${{matrix.scala}} test doc
diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-name: JaCoCo report
+name: JaCoCo report agent and module
 
 on:
   pull_request:
@@ -24,8 +24,6 @@ on:
 env:
   scalaLong12: 2.12.18
   scalaShort12: "2.12"
-  scalaLong13: 2.13.11
-  scalaShort13: "2.13"
   overall: 80.0
   changed: 80.0
 
@@ -34,45 +32,19 @@ jobs:
     name: Build and test
     runs-on: ubuntu-latest
 
-    services:
-      postgres:
-        image: postgres:15
-        env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: atum_db
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Setup Scala
         uses: olafurpg/setup-scala@v14
         with:
           java-version: "[email protected]"
-      - name: Prepare testing database
-        run: sbt flywayMigrate
       - name: Build and run tests
         continue-on-error: true
         id: jacocorun
-        run: sbt jacoco
-      #  server module code coverage
-      - name: Add coverage to PR
-        if: steps.jacocorun.outcome == 'success'
-        id: jacoco-server
-        uses: madrapps/[email protected]
-        with:
-          paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml
-          token: ${{ secrets.GITHUB_TOKEN }}
-          min-coverage-overall: ${{env.overall }}
-          min-coverage-changed-files: ${{ env.changed }}
-          title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }}
-          update-comment: true
+        run: |
+          sbt "project agent; jacoco"
+          sbt "project model; jacoco"
       #  agent module code coverage
       - name: Add coverage to PR
         if: steps.jacocorun.outcome == 'success'
@@ -102,8 +74,6 @@ jobs:
       - name: Get the Coverage info
         if: steps.jacocorun.outcome == 'success'
         run: |
-          echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}"
-          echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}"
           echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}"
           echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}"
           echo "Total model module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}"
@@ -114,7 +84,6 @@ jobs:
         with:
           script: |
             const coverageCheckFailed =
-              Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') ||
               Number('${{ steps.jacoco-agent.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') ||
               Number('${{ steps.jacoco-model.outputs.coverage-changed-files }}') < Number('${{ env.changed }}');
             if (coverageCheckFailed) {

diff --git a/.github/workflows/jacoco_check_server.yml b/.github/workflows/jacoco_check_server.yml
@@ -0,0 +1,117 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: JaCoCo report server
+
+on:
+  pull_request:
+    branches: [ master ]
+    types: [ opened, edited, synchronize, reopened ]
+
+env:
+  scalaLong13: 2.13.11
+  scalaShort13: "2.13"
+  overall: 80.0
+  changed: 80.0
+
+jobs:
+  test:
+    name: Build and test
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: atum_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v14
+        with:
+          java-version: "[email protected]"
+      - name: Prepare testing database
+        run: sbt flywayMigrate
+      - name: Build and run tests
+        continue-on-error: true
+        id: jacocorun
+        run: sbt "project server; jacoco"
+      #  server module code coverage
+      - name: Add coverage to PR
+        if: steps.jacocorun.outcome == 'success'
+        id: jacoco-server
+        uses: madrapps/[email protected]
+        with:
+          paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: ${{env.overall }}
+          min-coverage-changed-files: ${{ env.changed }}
+          title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }}
+          update-comment: true
+      - name: Get the Coverage info
+        if: steps.jacocorun.outcome == 'success'
+        run: |
+          echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}"
+          echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}"
+      - name: Fail PR if changed files coverage is less than ${{ env.changed }}%
+        if: steps.jacocorun.outcome == 'success'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const coverageCheckFailed =
+              Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}');
+            if (coverageCheckFailed) {
+              core.setFailed('Changed files coverage is less than ${{ env.changed }}%!');
+            }
+      - name: Edit JaCoCo comments on build failure
+        if: steps.jacocorun.outcome != 'success'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const issue_number = context.issue.number;
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const jacocoReportRegExp = /^### JaCoCo .* code coverage report .*/;
+
+            const comments = await github.rest.issues.listComments({
+              owner,
+              repo,
+              issue_number,
+            });
+
+            for (const comment of comments.data) {
+              const lines = comment.body.split('\n');
+              if (lines.length > 0 && jacocoReportRegExp.test(lines[0])) {
+                await github.rest.issues.updateComment({
+                  owner,
+                  repo,
+                  comment_id: comment.id,
+                  body: lines[0] + "\n\n### Build Failed",
+                });
+              }
+            }
+
+            core.setFailed('JaCoCo test coverage report generation failed, and related PR comments were updated.');
diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml
@@ -27,19 +27,19 @@ jobs:
         with:
           fetch-depth: 0
       - uses: olafurpg/setup-scala@v14
-      - name: Run sbt ci-release (produces war as well)
+      - name: Run sbt ci-release (produces jar as well)
         run: sbt ci-release
         env:
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.PGP_SECRET }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
 
-      - name: Find WAR file
-        id: find_war
-        run: echo "WAR_PATH=$(find . -name '*.war' | head -n 1)" >> $GITHUB_ENV
+      - name: Find JAR file
+        id: find_jar
+        run: echo "JAR_PATH=$(find . -name 'server/target/jvm-2.13/*.jar' | head -n 1)" >> $GITHUB_ENV
 
-      - name: Upload WAR file to GitHub Release
-        run: gh release upload ${{ github.event.release.tag_name }} ${{ env.WAR_PATH }} --repo ${{ github.repository }}
+      - name: Upload JAR file to GitHub Release
+        run: gh release upload ${{ github.event.release.tag_name }} ${{ env.JAR_PATH }} --repo ${{ github.repository }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -88,3 +88,6 @@ _site
 utils/resources/*.conf
 
 .bsp
+/server/certs/
+/server/selfsigned.crt
+/server/selfsigned.p12
diff --git a/build.sbt b/build.sbt
@@ -21,7 +21,7 @@ import sbt.Keys.name
 ThisBuild / organization := "za.co.absa.atum-service"
 sonatypeProfileName := "za.co.absa"
 
-ThisBuild / scalaVersion := Versions.scala212  // default version
+ThisBuild / scalaVersion := Versions.scala213  // default version
 
 ThisBuild / versionScheme := Some("early-semver")
 
@@ -33,13 +33,23 @@ lazy val printSparkScalaVersion = taskKey[Unit]("Print Spark and Scala versions
 lazy val printScalaVersion = taskKey[Unit]("Print Scala versions for atum-service is being built for.")
 
 lazy val commonSettings = Seq(
-  libraryDependencies ++= commonDependencies,
   scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"),
-  javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"),
   Test / parallelExecution := false,
   jacocoExcludes := jacocoProjectExcludes()
 )
 
+val serverMergeStrategy = assembly / assemblyMergeStrategy := {
+  case PathList("META-INF", "services", xs @ _*) => MergeStrategy.filterDistinctLines
+  case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError
+  case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError
+  case PathList("META-INF", _*) => MergeStrategy.discard
+  case PathList("META-INF", "versions", "9", xs@_*) => MergeStrategy.discard
+  case PathList("module-info.class") => MergeStrategy.discard
+  case "application.conf" => MergeStrategy.concat
+  case "reference.conf" => MergeStrategy.concat
+  case _ => MergeStrategy.first
+}
+
 enablePlugins(FlywayPlugin)
 flywayUrl := FlywayConfiguration.flywayUrl
 flywayUser := FlywayConfiguration.flywayUser
@@ -52,33 +62,33 @@ lazy val server = (projectMatrix in file("server"))
   .settings(
     commonSettings ++ Seq(
       name := "atum-server",
-      libraryDependencies ++= Dependencies.serverDependencies,
-      scalacOptions ++= Seq("-Ymacro-annotations"),
+      libraryDependencies ++= Dependencies.serverDependencies ++ testDependencies,
+      javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"),
+      scalacOptions ++= Seq("-release", "11", "-Ymacro-annotations"),
       Compile / packageBin / publishArtifact := false,
       printScalaVersion := {
         val log = streams.value.log
         log.info(s"Building ${name.value} with Scala ${scalaVersion.value}")
       },
       (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value,
       packageBin := (Compile / assembly).value,
-      artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.war",
-      webappWebInfClasses := true,
-      inheritJarManifest := true,
-      publish / skip := true,
-      jacocoReportSettings := jacocoSettings(scalaVersion.value, "atum-server")
+      artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar",
+      testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework"),
+      jacocoReportSettings := jacocoSettings(scalaVersion.value, "atum-server"),
+      serverMergeStrategy
     ): _*
   )
   .enablePlugins(AssemblyPlugin)
-  .enablePlugins(TomcatPlugin)
   .enablePlugins(AutomateHeaderPlugin)
   .jvmPlatform(scalaVersions = Seq(Versions.serviceScalaVersion))
   .dependsOn(model)
 
 lazy val agent = (projectMatrix in file("agent"))
   .settings(
     commonSettings ++ Seq(
-      name := "agent",
-      libraryDependencies ++= Dependencies.agentDependencies(
+      name := "atum-agent",
+      javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"),
+      libraryDependencies ++= jsonSerdeDependencies ++ testDependencies ++ Dependencies.agentDependencies(
         if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3,
         scalaVersion.value
       ),
@@ -97,8 +107,9 @@ lazy val agent = (projectMatrix in file("agent"))
 lazy val model = (projectMatrix in file("model"))
   .settings(
     commonSettings ++ Seq(
-      name := "model",
-      libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value),
+      name         := "atum-model",
+      javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"),
+      libraryDependencies ++= jsonSerdeDependencies ++ testDependencies ++ Dependencies.modelDependencies(scalaVersion.value),
       printScalaVersion := {
         val log = streams.value.log
         log.info(s"Building ${name.value} with Scala ${scalaVersion.value}")