Merge pull request #14 from ps-dev/ADAPT-1802-dataClassification

[ADAPT-1802] Consider the obsolete(old) values of data classification as the schema evolution requires them
ps-dev · Jan 11, 2024 · 63e3ea5 · 63e3ea5
2 parents da2a93d + 5e388eb
commit 63e3ea5
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 30 deletions.
diff --git a/ingestors/kafka/src/main/resources/HydraMetadataTopic.avsc b/ingestors/kafka/src/main/resources/HydraMetadataTopic.avsc
@@ -38,7 +38,7 @@
       "type": {
         "name": "dataClassificationEnum",
         "type": "enum",
-        "symbols": ["Public", "InternalUse", "Confidential", "Restricted"]
+        "symbols": ["Public", "InternalUseOnly", "ConfidentialPII", "RestrictedFinancial", "RestrictedEmployeeData", "InternalUse", "Confidential", "Restricted"]
       }
     },
     {

diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/model/DataClassification.scala b/ingestors/kafka/src/main/scala/hydra/kafka/model/DataClassification.scala
@@ -5,6 +5,17 @@ import enumeratum.{Enum, EnumEntry}
 import scala.collection.immutable
 
 sealed trait DataClassification extends EnumEntry
+sealed trait ObsoleteDataClassification extends DataClassification
+
+object ObsoleteDataClassification extends Enum[ObsoleteDataClassification]  {
+
+  case object InternalUseOnly extends ObsoleteDataClassification
+  case object ConfidentialPII extends ObsoleteDataClassification
+  case object RestrictedFinancial extends ObsoleteDataClassification
+  case object RestrictedEmployeeData extends ObsoleteDataClassification
+
+  override val values: immutable.IndexedSeq[ObsoleteDataClassification] = findValues
+}
 
 object DataClassification extends Enum[DataClassification]  {
 
@@ -13,7 +24,7 @@ object DataClassification extends Enum[DataClassification]  {
   case object Confidential extends DataClassification
   case object Restricted extends DataClassification
 
-  override val values: immutable.IndexedSeq[DataClassification] = findValues
+  override val values: immutable.IndexedSeq[DataClassification] = findValues ++ ObsoleteDataClassification.values
 }
 
 sealed trait SubDataClassification extends EnumEntry

diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/model/TopicMetadata.scala b/ingestors/kafka/src/main/scala/hydra/kafka/model/TopicMetadata.scala
@@ -232,22 +232,34 @@ object TopicMetadataV2ValueOptionalTagList {
     Codec.deriveEnum[DataClassification](
       symbols = List(
         "Public",
+        "InternalUseOnly",
+        "ConfidentialPII",
+        "RestrictedFinancial",
+        "RestrictedEmployeeData",
         "InternalUse",
         "Confidential",
         "Restricted"
       ),
       encode = {
-        case Public       => "Public"
-        case InternalUse  => "InternalUse"
-        case Confidential => "Confidential"
-        case Restricted   => "Restricted"
+        case Public                                            => "Public"
+        case ObsoleteDataClassification.InternalUseOnly        => "InternalUseOnly"
+        case ObsoleteDataClassification.ConfidentialPII        => "ConfidentialPII"
+        case ObsoleteDataClassification.RestrictedFinancial    => "RestrictedFinancial"
+        case ObsoleteDataClassification.RestrictedEmployeeData => "RestrictedEmployeeData"
+        case InternalUse                                       => "InternalUse"
+        case Confidential                                      => "Confidential"
+        case Restricted                                        => "Restricted"
       },
       decode = {
-        case "Public"       => Right(Public)
-        case "InternalUse"  => Right(InternalUse)
-        case "Confidential" => Right(Confidential)
-        case "Restricted"   => Right(Restricted)
-        case other          => Left(AvroError(s"$other is not a DataClassification. Valid value is one of: ${DataClassification.values}"))
+        case "Public"                 => Right(Public)
+        case "InternalUseOnly"        => Right(ObsoleteDataClassification.InternalUseOnly)
+        case "ConfidentialPII"        => Right(ObsoleteDataClassification.ConfidentialPII)
+        case "RestrictedFinancial"    => Right(ObsoleteDataClassification.RestrictedFinancial)
+        case "RestrictedEmployeeData" => Right(ObsoleteDataClassification.RestrictedEmployeeData)
+        case "InternalUse"            => Right(InternalUse)
+        case "Confidential"           => Right(Confidential)
+        case "Restricted"             => Right(Restricted)
+        case other                    => Left(AvroError(s"$other is not a DataClassification. Valid value is one of: ${DataClassification.values}"))
       }
     )
 

diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/BootstrapEndpointSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/BootstrapEndpointSpec.scala
@@ -16,7 +16,7 @@ import hydra.core.http.security.entity.AwsConfig
 import hydra.core.http.security.{AccessControlService, AwsSecurityService}
 import hydra.core.protocol.{Ingest, IngestorCompleted, IngestorError}
 import hydra.kafka.marshallers.HydraKafkaJsonSupport
-import hydra.kafka.model.{DataClassification, SubDataClassification, TopicMetadata}
+import hydra.kafka.model.{DataClassification, ObsoleteDataClassification, SubDataClassification, TopicMetadata}
 import hydra.kafka.producer.AvroRecord
 import hydra.kafka.services.StreamsManagerActor
 import hydra.kafka.util.KafkaUtils
@@ -487,15 +487,7 @@ class BootstrapEndpointSpec
     }
 
     DataClassification.values foreach { dc =>
-      s"$dc: accept valid DataClassification value" in {
-        Post("/streams", dataClassificationRequest(dc.entryName)) ~> bootstrapRoute ~> check {
-          status shouldBe StatusCodes.OK
-        }
-      }
-    }
-
-    SubDataClassification.values foreach { dc =>
-      s"$dc: accept deprecated valid DataClassification value" in {
+      s"$dc: accept valid as well as obsolete(enforced by backward schema evolution) DataClassification value" in {
         Post("/streams", dataClassificationRequest(dc.entryName)) ~> bootstrapRoute ~> check {
           status shouldBe StatusCodes.OK
         }

diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
@@ -381,15 +381,7 @@ class TopicMetadataEndpointSpec
     }
 
     DataClassification.values foreach { dc =>
-      s"$dc: accept valid DataClassification value" in {
-        Put("/v2/metadata/dvs.test.subject", HttpEntity(ContentTypes.`application/json`, dataClassificationRequest(dc.entryName))) ~> route ~> check {
-          response.status shouldBe StatusCodes.OK
-        }
-      }
-    }
-
-    SubDataClassification.values foreach { dc =>
-      s"$dc: accept deprecated valid DataClassification value" in {
+      s"$dc: accept valid as well as obsolete(enforced by backward schema evolution) DataClassification value" in {
         Put("/v2/metadata/dvs.test.subject", HttpEntity(ContentTypes.`application/json`, dataClassificationRequest(dc.entryName))) ~> route ~> check {
           response.status shouldBe StatusCodes.OK
         }