From 88fa49d8d71f1ca50244c6412b44d78b9e9b6c90 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 12 Nov 2024 11:05:54 +0100 Subject: [PATCH 01/10] do not ignore test --- .../za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala index 191a987a..497bee8b 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala @@ -40,7 +40,7 @@ class AgentWithServerIntegrationTests extends DBTestSuite { .add(StructField("columnForSum", DoubleType)) // Need to add service & pg run in CI - ignore("Agent should be compatible with server") { + test("Agent should be compatible with server") { val expectedMeasurement = JsonBString( """{"mainValue": {"value": "4", "valueType": "Long"}, "supportValues": {}}""".stripMargin From 2b2d417d9adf39cc3fd3ac5a99616b93b0644255 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 12 Nov 2024 11:09:57 +0100 Subject: [PATCH 02/10] do not ignore test --- .../za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala index 497bee8b..8ca51933 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala @@ -22,10 +22,12 @@ import za.co.absa.atum.agent.model.AtumMeasure.RecordCount import za.co.absa.balta.DBTestSuite import za.co.absa.balta.classes.JsonBString import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import org.scalatest.Ignore import za.co.absa.atum.agent.dispatcher.HttpDispatcher import scala.collection.immutable.ListMap +@Ignore class AgentWithServerIntegrationTests extends DBTestSuite { private val testDataForRDD = Seq( From 2e4d4eb94359284ff3824a8e347e637f45954d9b Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 12 Nov 2024 11:41:00 +0100 Subject: [PATCH 03/10] test with postgresql --- project/Dependencies.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e9783300..a1eb7c17 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -208,6 +208,7 @@ object Dependencies { lazy val nameOf = "com.github.dwickern" %% "scala-nameof" % Versions.scalaNameof % Provided // it's provided, as it's a macro needed only at runtime lazy val balta = "za.co.absa" %% "balta" % Versions.balta % Test + lazy val postgresql = "org.postgresql" % "postgresql" % Versions.postgresql % Test Seq( sparkCore, @@ -220,7 +221,7 @@ object Dependencies { logback, nameOf ) ++ - testDependencies :+ balta + testDependencies :+ balta :+ postgresql } def modelDependencies(scalaVersion: Version): Seq[ModuleID] = { From 1c8e4c40096b7b3fd17e126a5d8bf5bc42f97932 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 12 Nov 2024 12:34:09 +0100 Subject: [PATCH 04/10] test without postgresql --- project/Dependencies.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a1eb7c17..c81bf997 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -208,7 +208,7 @@ object Dependencies { lazy val nameOf = "com.github.dwickern" %% "scala-nameof" % Versions.scalaNameof % Provided // it's provided, as it's a macro needed only at runtime lazy val balta = "za.co.absa" %% "balta" % Versions.balta % Test - lazy val postgresql = "org.postgresql" % "postgresql" % Versions.postgresql % Test +// lazy val postgresql = "org.postgresql" % "postgresql" % Versions.postgresql % Test Seq( sparkCore, @@ -221,7 +221,7 @@ object Dependencies { logback, nameOf ) ++ - testDependencies :+ balta :+ postgresql + testDependencies :+ balta //:+ postgresql } def modelDependencies(scalaVersion: Version): Seq[ModuleID] = { From 7bb7e14934a4ef8f664b463bc27a252d82a0b849 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 12 Nov 2024 12:34:52 +0100 Subject: [PATCH 05/10] test without postgresql --- .../za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala index 8ca51933..82487cd6 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala @@ -27,7 +27,6 @@ import za.co.absa.atum.agent.dispatcher.HttpDispatcher import scala.collection.immutable.ListMap -@Ignore class AgentWithServerIntegrationTests extends DBTestSuite { private val testDataForRDD = Seq( From d6292e042768b4c9bb5a0eaf2651c6872d256e21 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 13 Nov 2024 09:13:34 +0100 Subject: [PATCH 06/10] testAgentServer sbt cmd --- .sbtrc | 3 +++ ...WithServerIntegrationTests.scala => AgentServerTests.scala} | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) rename agent/src/test/scala/za/co/absa/atum/agent/{AgentWithServerIntegrationTests.scala => AgentServerTests.scala} (98%) diff --git a/.sbtrc b/.sbtrc index 12237cb8..22cabe35 100644 --- a/.sbtrc +++ b/.sbtrc @@ -30,3 +30,6 @@ alias testDB=; project database; testOnly * # Run all tests alias testAll=; testOnly * + +# Run Agent-Server integration tests +alias testAgentServer=; testOnly *AgentServerTests diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala similarity index 98% rename from agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala rename to agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala index 82487cd6..1a0e920c 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentWithServerIntegrationTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala @@ -27,7 +27,7 @@ import za.co.absa.atum.agent.dispatcher.HttpDispatcher import scala.collection.immutable.ListMap -class AgentWithServerIntegrationTests extends DBTestSuite { +class AgentServerTests extends DBTestSuite { private val testDataForRDD = Seq( Row("A", 8.0), From ff8920a7e3a551408e6effddf1be82f0189bc38c Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 13 Nov 2024 09:15:03 +0100 Subject: [PATCH 07/10] clean up --- .../src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala index 1a0e920c..f8597903 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala @@ -22,7 +22,6 @@ import za.co.absa.atum.agent.model.AtumMeasure.RecordCount import za.co.absa.balta.DBTestSuite import za.co.absa.balta.classes.JsonBString import com.typesafe.config.{ConfigFactory, ConfigValueFactory} -import org.scalatest.Ignore import za.co.absa.atum.agent.dispatcher.HttpDispatcher import scala.collection.immutable.ListMap From 9d6c3e11e4b805dee96a43fb3998e5f34136e503 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 13 Nov 2024 09:16:00 +0100 Subject: [PATCH 08/10] clean up --- project/Dependencies.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c81bf997..e9783300 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -208,7 +208,6 @@ object Dependencies { lazy val nameOf = "com.github.dwickern" %% "scala-nameof" % Versions.scalaNameof % Provided // it's provided, as it's a macro needed only at runtime lazy val balta = "za.co.absa" %% "balta" % Versions.balta % Test -// lazy val postgresql = "org.postgresql" % "postgresql" % Versions.postgresql % Test Seq( sparkCore, @@ -221,7 +220,7 @@ object Dependencies { logback, nameOf ) ++ - testDependencies :+ balta //:+ postgresql + testDependencies :+ balta } def modelDependencies(scalaVersion: Version): Seq[ModuleID] = { From d435946af1ceb2033d3f5fdb2559eeda44bacea4 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 13 Nov 2024 10:55:28 +0100 Subject: [PATCH 09/10] added file names pattern for agent-server tests --- .github/workflows/test_filenames_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index d3e24ee2..cec8856e 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -33,7 +33,7 @@ jobs: id: scan-test-files uses: AbsaOSS/filename-inspector@v0.1.0 with: - name-patterns: '*UnitTests.*,*IntegrationTests.*' + name-patterns: '*UnitTests.*,*IntegrationTests.*,*AgentServerTests.*' paths: '**/src/test/scala/**' report-format: 'console' excludes: | From 0d92b9f525ce3b1168aea520e17476d15e1c878f Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 14 Nov 2024 12:16:59 +0100 Subject: [PATCH 10/10] renamings --- .github/workflows/test_filenames_check.yml | 2 +- .sbtrc | 4 ++-- ...tServerTests.scala => AgentServerCompatibilityTests.scala} | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename agent/src/test/scala/za/co/absa/atum/agent/{AgentServerTests.scala => AgentServerCompatibilityTests.scala} (98%) diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index cec8856e..ed0a061a 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -33,7 +33,7 @@ jobs: id: scan-test-files uses: AbsaOSS/filename-inspector@v0.1.0 with: - name-patterns: '*UnitTests.*,*IntegrationTests.*,*AgentServerTests.*' + name-patterns: '*UnitTests.*,*IntegrationTests.*,*CompatibilityTests.*' paths: '**/src/test/scala/**' report-format: 'console' excludes: | diff --git a/.sbtrc b/.sbtrc index 22cabe35..c89544c9 100644 --- a/.sbtrc +++ b/.sbtrc @@ -31,5 +31,5 @@ alias testDB=; project database; testOnly * # Run all tests alias testAll=; testOnly * -# Run Agent-Server integration tests -alias testAgentServer=; testOnly *AgentServerTests +# Run agent-server compatibility tests +alias testCompatibility=; testOnly *CompatibilityTests diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala similarity index 98% rename from agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala rename to agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala index f8597903..992aabe1 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentServerTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.agent.dispatcher.HttpDispatcher import scala.collection.immutable.ListMap -class AgentServerTests extends DBTestSuite { +class AgentServerCompatibilityTests extends DBTestSuite { private val testDataForRDD = Seq( Row("A", 8.0),