From 6a9d5d35e4d82c88a181975a9897890e514ce7a3 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 16 Oct 2023 17:39:45 +0200 Subject: [PATCH] #87: DTOs for measures and measure results, MeasurementBuilder, Agent refactored --- .../za/co/absa/atum/agent/AtumAgent.scala | 15 ++++---- .../za/co/absa/atum/agent/AtumContext.scala | 11 +++--- .../agent/dispatcher/ConsoleDispatcher.scala | 12 +++---- .../atum/agent/dispatcher/Dispatcher.scala | 5 ++- .../agent/dispatcher/HttpDispatcher.scala | 11 ++---- .../atum/agent/model/MeasurementBuilder.scala | 36 +++++++++++++++++++ .../atum/agent/model/MeasuresMapper.scala | 18 +++++----- ...MeasureTest.scala => MeasureDTOTest.scala} | 2 +- .../za/co/absa/atum/model/MeasureResult.scala | 21 ----------- .../za/co/absa/atum/model/Partitioning.scala | 21 ----------- .../AdditionalDataDTO.scala} | 4 +-- .../absa/atum/model/dto/AtumContextDTO.scala | 8 ++--- .../absa/atum/model/dto/CheckpointDTO.scala | 6 ++-- .../{Measure.scala => dto/MeasureDTO.scala} | 4 +-- .../atum/model/dto/MeasureResultDTO.scala | 26 ++++++++++++++ .../MeasurementDTO.scala} | 8 ++--- .../PartitionDTO.scala} | 4 +-- .../absa/atum/model/dto/PartitioningDTO.scala | 6 ++-- 18 files changed, 113 insertions(+), 105 deletions(-) create mode 100644 agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala rename agent/src/test/scala/za/co/absa/atum/agent/model/{MeasureTest.scala => MeasureDTOTest.scala} (97%) delete mode 100644 model/src/main/scala/za/co/absa/atum/model/MeasureResult.scala delete mode 100644 model/src/main/scala/za/co/absa/atum/model/Partitioning.scala rename model/src/main/scala/za/co/absa/atum/model/{AdditionalData.scala => dto/AdditionalDataDTO.scala} (91%) rename model/src/main/scala/za/co/absa/atum/model/{Measure.scala => dto/MeasureDTO.scala} (92%) create mode 100644 model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala rename model/src/main/scala/za/co/absa/atum/model/{Measurement.scala => dto/MeasurementDTO.scala} (79%) rename model/src/main/scala/za/co/absa/atum/model/{Partition.scala => dto/PartitionDTO.scala} (91%) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index 7e576afcf..4c29517e8 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.agent import com.typesafe.config.{Config, ConfigFactory} import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.dispatcher.{ConsoleDispatcher, HttpDispatcher} -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, PartitioningDTO} /** * Place holder for the agent that communicate with the API. @@ -47,17 +47,20 @@ class AtumAgent private[agent] () { * @return */ def getOrCreateAtumContext(atumPartitions: AtumPartitions): AtumContext = { - val atumContextDTO = dispatcher.getOrCreateAtumContext(AtumPartitions.toPartitioning(atumPartitions), None) + val partitioningDTO = PartitioningDTO(AtumPartitions.toSeqPartitionDTO(atumPartitions), None) + val atumContextDTO = dispatcher.getOrCreateAtumContext(partitioningDTO) lazy val atumContext = AtumContext.fromDTO(atumContextDTO, this) getExistingOrNewContext(atumPartitions, atumContext) } def getOrCreateAtumSubContext(subPartitions: AtumPartitions)(implicit parentAtumContext: AtumContext): AtumContext = { val newPartitions: AtumPartitions = parentAtumContext.atumPartitions ++ subPartitions - val atumContextDTO = dispatcher.getOrCreateAtumContext( - AtumPartitions.toPartitioning(newPartitions), - Some(AtumPartitions.toPartitioning(parentAtumContext.atumPartitions)) - ) + + val newPartitionsDTO = AtumPartitions.toSeqPartitionDTO(newPartitions) + val maybeParentPartitionsDTO = Some(AtumPartitions.toSeqPartitionDTO(parentAtumContext.atumPartitions)) + val partitioningDTO = PartitioningDTO(newPartitionsDTO, maybeParentPartitionsDTO) + + val atumContextDTO = dispatcher.getOrCreateAtumContext(partitioningDTO) lazy val atumContext = AtumContext.fromDTO(atumContextDTO, this) getExistingOrNewContext(newPartitions, atumContext) } diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index 5a4f9b0be..3f58d8676 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -19,8 +19,7 @@ package za.co.absa.atum.agent import org.apache.spark.sql.DataFrame import za.co.absa.atum.agent.model.{Measure, MeasuresMapper} import AtumContext.AtumPartitions -import za.co.absa.atum.model.{Partition, Partitioning} -import za.co.absa.atum.model.dto.AtumContextDTO +import za.co.absa.atum.model.dto.{AtumContextDTO, PartitionDTO} import scala.collection.immutable.ListMap @@ -90,12 +89,12 @@ object AtumContext { ListMap(elems:_*) } - private[agent] def toPartitioning(atumPartitions: AtumPartitions): Partitioning = { - Partitioning(atumPartitions.toSeq.map{ case (key, value) => Partition(key, value) }) + private[agent] def toSeqPartitionDTO(atumPartitions: AtumPartitions): Seq[PartitionDTO] = { + atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq } - private[agent] def fromPartitioning(partitioning: Partitioning): AtumPartitions = { - AtumPartitions(partitioning.partitioning.map(partition => partition.key -> partition.value)) + private[agent] def fromPartitioning(partitioning: Seq[PartitionDTO]): AtumPartitions = { + AtumPartitions(partitioning.map(partition => partition.key -> partition.value)) } } diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/ConsoleDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/ConsoleDispatcher.scala index db5760cf0..ef51e109f 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/ConsoleDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/ConsoleDispatcher.scala @@ -17,8 +17,7 @@ package za.co.absa.atum.agent.dispatcher import org.apache.spark.internal.Logging -import za.co.absa.atum.model.Partitioning -import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO} +import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO, PartitioningDTO} /** * dispatcher useful for development, testing and debugging @@ -27,12 +26,9 @@ class ConsoleDispatcher extends Dispatcher with Logging { logInfo("using console dispatcher") - override def getOrCreateAtumContext( - partitioning: Partitioning, - parentPartitioning: Option[Partitioning] - ): AtumContextDTO = { - println(s"Fetching AtumContext using ConsoleDispatcher with $partitioning and $parentPartitioning") - AtumContextDTO(partitioning = partitioning) + override def getOrCreateAtumContext(partitioning: PartitioningDTO): AtumContextDTO = { + println(s"Fetching AtumContext using ConsoleDispatcher with partitioning $partitioning") + AtumContextDTO(partitioning = partitioning.partitioning) } override def saveCheckpoint(checkpoint: CheckpointDTO): Unit = { println(s"Saving checkpoint to server. $checkpoint") diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/Dispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/Dispatcher.scala index 42a402226..ffd3f97bb 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/Dispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/Dispatcher.scala @@ -16,11 +16,10 @@ package za.co.absa.atum.agent.dispatcher -import za.co.absa.atum.model.Partitioning -import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO} +import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO, PartitioningDTO} trait Dispatcher { - def getOrCreateAtumContext(partitioning: Partitioning, parentPartitioning: Option[Partitioning]): AtumContextDTO + def getOrCreateAtumContext(partitioning: PartitioningDTO): AtumContextDTO def saveCheckpoint(checkpoint: CheckpointDTO): Unit } diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index 604a0d01e..444260d60 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -20,7 +20,6 @@ import com.typesafe.config.Config import org.apache.spark.internal.Logging import sttp.client3._ import sttp.model.Uri -import za.co.absa.atum.model.Partitioning import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO, PartitioningDTO} class HttpDispatcher(config: Config) extends Dispatcher with Logging { @@ -31,17 +30,13 @@ class HttpDispatcher(config: Config) extends Dispatcher with Logging { logInfo("using http dispatcher") logInfo(s"serverUri $serverUri") - override def getOrCreateAtumContext( - partitioning: Partitioning, - parentPartitioning: Option[Partitioning] - ): AtumContextDTO = { - val partitioningDTO = PartitioningDTO(partitioning, parentPartitioning) + override def getOrCreateAtumContext(partitioning: PartitioningDTO): AtumContextDTO = { basicRequest - .body(s"$partitioningDTO") + .body(s"$partitioning") .post(serverUri) .send(backend) - AtumContextDTO(partitioning = partitioning) // todo: implement request + AtumContextDTO(partitioning = partitioning.partitioning) // todo: implement request } override def saveCheckpoint(checkpoint: CheckpointDTO): Unit = { diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala new file mode 100644 index 000000000..f14dd1ae0 --- /dev/null +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala @@ -0,0 +1,36 @@ +package za.co.absa.atum.agent.model + +import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO} +import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} + +object MeasurementBuilder { + + def buildLongMeasurement(functionName: String, controlCols: Seq[String], resultValue: Long): MeasurementDTO = { + MeasurementDTO( + MeasureDTO(functionName, controlCols), + MeasureResultDTO(TypedValue(resultValue.toString, ResultValueType.Long)) + ) + } + + def buildDoubleMeasureResult(functionName: String, controlCols: Seq[String], resultValue: Double): MeasurementDTO = { + MeasurementDTO( + MeasureDTO(functionName, controlCols), + MeasureResultDTO(TypedValue(resultValue.toString, ResultValueType.Double)) + ) + } + + def buildBigDecimalMeasureResult(functionName: String, controlCols: Seq[String], resultValue: BigDecimal): MeasurementDTO = { + MeasurementDTO( + MeasureDTO(functionName, controlCols), + MeasureResultDTO(TypedValue(resultValue.toString, ResultValueType.BigDecimal)) + ) + } + + def buildStringMeasureResult(functionName: String, controlCols: Seq[String], resultValue: String): MeasurementDTO = { + MeasurementDTO( + MeasureDTO(functionName, controlCols), + MeasureResultDTO(TypedValue(resultValue, ResultValueType.String)) + ) + } + +} diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasuresMapper.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasuresMapper.scala index 915252373..ad15b7326 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasuresMapper.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasuresMapper.scala @@ -17,24 +17,26 @@ package za.co.absa.atum.agent.model import za.co.absa.atum.agent.model.Measure._ +import za.co.absa.atum.model.dto case class UnsupportedMeasureException(msg: String) extends Exception(msg) object MeasuresMapper { - def mapToMeasures(measures: Set[za.co.absa.atum.model.Measure]): Set[za.co.absa.atum.agent.model.Measure] = { + def mapToMeasures(measures: Set[dto.MeasureDTO]): Set[za.co.absa.atum.agent.model.Measure] = { measures.map(createMeasure) } - private def createMeasure(measure: za.co.absa.atum.model.Measure): za.co.absa.atum.agent.model.Measure = { + private def createMeasure(measure: dto.MeasureDTO): za.co.absa.atum.agent.model.Measure = { val controlColumn = measure.controlColumns.head measure.functionName match { - case "RecordCount" => RecordCount(controlColumn) - case "DistinctRecordCount" => DistinctRecordCount(controlColumn) - case "SumOfValuesOfColumn" => SumOfValuesOfColumn(controlColumn) - case "AbsSumOfValuesOfColumn" => AbsSumOfValuesOfColumn(controlColumn) - case "SumOfHashesOfColumn" => SumOfHashesOfColumn(controlColumn) - case unsupportedMeasure => throw UnsupportedMeasureException(s"Measure not supported: $unsupportedMeasure") + case RecordCount.getClass.getSimpleName => RecordCount(controlColumn) + case DistinctRecordCount.getClass.getSimpleName => DistinctRecordCount(controlColumn) + case SumOfValuesOfColumn.getClass.getSimpleName => SumOfValuesOfColumn(controlColumn) + case AbsSumOfValuesOfColumn.getClass.getSimpleName => AbsSumOfValuesOfColumn(controlColumn) + case SumOfHashesOfColumn.getClass.getSimpleName => SumOfHashesOfColumn(controlColumn) + case unsupportedMeasure => + throw UnsupportedMeasureException(s"Measure not supported: $unsupportedMeasure") } } diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureTest.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureDTOTest.scala similarity index 97% rename from agent/src/test/scala/za/co/absa/atum/agent/model/MeasureTest.scala rename to agent/src/test/scala/za/co/absa/atum/agent/model/MeasureDTOTest.scala index be7a75bfd..e68d8ec5a 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureTest.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureDTOTest.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.agent.AtumContext.{AtumPartitions, DatasetWrapper} import za.co.absa.atum.agent.model.Measure._ import za.co.absa.spark.commons.test.SparkTestBase -class MeasureTest extends AnyFlatSpec with Matchers with SparkTestBase { self => +class MeasureDTOTest extends AnyFlatSpec with Matchers with SparkTestBase { self => "Measure" should "be based on the dataframe" in { diff --git a/model/src/main/scala/za/co/absa/atum/model/MeasureResult.scala b/model/src/main/scala/za/co/absa/atum/model/MeasureResult.scala deleted file mode 100644 index 1812b3250..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/MeasureResult.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model - -trait MeasureResultBase[T] extends Any { def result: T } - -case class MeasureResult[T](result: T) extends AnyVal with MeasureResultBase[T] diff --git a/model/src/main/scala/za/co/absa/atum/model/Partitioning.scala b/model/src/main/scala/za/co/absa/atum/model/Partitioning.scala deleted file mode 100644 index 45de00664..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/Partitioning.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model - -case class Partitioning( - partitioning: Seq[Partition] -) diff --git a/model/src/main/scala/za/co/absa/atum/model/AdditionalData.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala similarity index 91% rename from model/src/main/scala/za/co/absa/atum/model/AdditionalData.scala rename to model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala index 60bf6d08e..cc3b983fe 100644 --- a/model/src/main/scala/za/co/absa/atum/model/AdditionalData.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala @@ -14,8 +14,8 @@ * limitations under the License. */ -package za.co.absa.atum.model +package za.co.absa.atum.model.dto -case class AdditionalData( +case class AdditionalDataDTO( additionalData: Map[String, Option[String]] ) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala index 13c7bdf7c..1956bb862 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala @@ -16,10 +16,8 @@ package za.co.absa.atum.model.dto -import za.co.absa.atum.model._ - case class AtumContextDTO( - partitioning: Partitioning, - measures: Set[Measure] = Set.empty, - additionalData: AdditionalData = AdditionalData(additionalData = Map.empty) + partitioning: Seq[PartitionDTO], + measures: Set[MeasureDTO] = Set.empty, + additionalData: AdditionalDataDTO = AdditionalDataDTO(additionalData = Map.empty) ) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala index 4d903735f..7f0325491 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala @@ -16,8 +16,6 @@ package za.co.absa.atum.model.dto -import za.co.absa.atum.model._ - import java.time.ZonedDateTime import java.util.UUID @@ -26,8 +24,8 @@ case class CheckpointDTO( name: String, author: String, measuredByAtumAgent: Boolean = false, - partitioning: Partitioning, + partitioning: Seq[PartitionDTO], processStartTime: ZonedDateTime, processEndTime: Option[ZonedDateTime], - measurements: Seq[Measurement] + measurements: Seq[MeasurementDTO] ) diff --git a/model/src/main/scala/za/co/absa/atum/model/Measure.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala similarity index 92% rename from model/src/main/scala/za/co/absa/atum/model/Measure.scala rename to model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala index 79d1abb1c..d82730685 100644 --- a/model/src/main/scala/za/co/absa/atum/model/Measure.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala @@ -14,9 +14,9 @@ * limitations under the License. */ -package za.co.absa.atum.model +package za.co.absa.atum.model.dto -case class Measure( +case class MeasureDTO( functionName: String, controlColumns: Seq[String] ) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala new file mode 100644 index 000000000..e00f2117f --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -0,0 +1,26 @@ +package za.co.absa.atum.model.dto + +import com.fasterxml.jackson.core.`type`.TypeReference +import com.fasterxml.jackson.module.scala.JsonScalaEnumeration +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue + +case class MeasureResultDTO( + mainValue: TypedValue, + supportValues: Map[String, TypedValue] = Map.empty + ) + + +object MeasureResultDTO { + case class TypedValue( + value: String, + @JsonScalaEnumeration(classOf[ResultValueType]) valueType: ResultValueType.ResultValueType + ) + + class ResultValueType extends TypeReference[ResultValueType.type] + + object ResultValueType extends Enumeration { + type ResultValueType = Value + val String, Long, BigDecimal, Double = Value + } + +} diff --git a/model/src/main/scala/za/co/absa/atum/model/Measurement.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala similarity index 79% rename from model/src/main/scala/za/co/absa/atum/model/Measurement.scala rename to model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala index 8bc1982a8..fcb9f528b 100644 --- a/model/src/main/scala/za/co/absa/atum/model/Measurement.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala @@ -14,9 +14,9 @@ * limitations under the License. */ -package za.co.absa.atum.model +package za.co.absa.atum.model.dto -case class Measurement( - measure: Measure, - result: MeasureResult[String] +case class MeasurementDTO( + measure: MeasureDTO, + result: MeasureResultDTO ) diff --git a/model/src/main/scala/za/co/absa/atum/model/Partition.scala b/model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala similarity index 91% rename from model/src/main/scala/za/co/absa/atum/model/Partition.scala rename to model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala index e21d5d66b..4b19b20df 100644 --- a/model/src/main/scala/za/co/absa/atum/model/Partition.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala @@ -14,9 +14,9 @@ * limitations under the License. */ -package za.co.absa.atum.model +package za.co.absa.atum.model.dto -case class Partition( +case class PartitionDTO( key: String, value: String ) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningDTO.scala index b1d4ae787..6e55f2df3 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningDTO.scala @@ -16,9 +16,7 @@ package za.co.absa.atum.model.dto -import za.co.absa.atum.model.Partitioning - case class PartitioningDTO ( - partitioning: Partitioning, - parentPartitioning: Option[Partitioning] + partitioning: Seq[PartitionDTO], + parentPartitioning: Option[Seq[PartitionDTO]] )