From f1798b2cdbadcb3b41da8ea94ba1f5e318e809a2 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 7 Mar 2022 15:35:23 +0100 Subject: [PATCH 01/67] #1693 API v3: VersionedModel v3 init - VersionedModelControllerV3 base contours, empty DatasetControllerV3 to test --- .../versionedModel/VersionedSummary.scala | 2 + .../enceladus/rest_api/SpringFoxConfig.scala | 10 +- .../controllers/v3/DatasetControllerV3.scala | 34 ++++ .../v3/VersionedModelControllerV3.scala | 147 ++++++++++++++++++ .../VersionedMongoRepository.scala | 15 +- .../services/VersionedModelService.scala | 4 + .../rest_api/utils/implicits/package.scala | 2 +- 7 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index 87a0d365d..bc28bfbb0 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -16,3 +16,5 @@ package za.co.absa.enceladus.model.versionedModel case class VersionedSummary(_id: String, latestVersion: Int) + +case class VersionsList(_id: String, versions: Seq[Int]) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala index 6121ad047..b5980e65c 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala @@ -39,11 +39,17 @@ class SpringFoxConfig extends ProjectMetadata { } private def filteredPaths: Predicate[String] = - or[String](regex("/api/dataset.*"), regex("/api/schema.*"), + or[String]( + // api v2 + regex("/api/dataset.*"), regex("/api/schema.*"), regex("/api/mappingTable.*"), regex("/api/properties.*"), regex("/api/monitoring.*"),regex("/api/runs.*"), regex("/api/user.*"), regex("/api/spark.*"), - regex("/api/configuration.*") + regex("/api/configuration.*"), + + // api v3 + regex("/api-v3/datasets.*") + ) private def apiInfo = diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala new file mode 100644 index 000000000..ac207ca02 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.rest_api.services.DatasetService + +@RestController +@RequestMapping(path = Array("/api-v3/datasets")) +class DatasetControllerV3 @Autowired()(datasetService: DatasetService) + extends VersionedModelControllerV3(datasetService) { + + // TODO + // /{datasetName}/{version}/rules + // /{datasetName}/{version}/rules/{index} + // /{datasetName}/{version}/rules + +} + + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala new file mode 100644 index 000000000..7193c95ca --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -0,0 +1,147 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import com.mongodb.client.result.UpdateResult +import org.springframework.http.HttpStatus +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.model.menas.audit._ +import za.co.absa.enceladus.model.versionedModel._ +import za.co.absa.enceladus.model.{ExportableObject, UsedIn} +import za.co.absa.enceladus.rest_api.controllers.BaseController +import za.co.absa.enceladus.rest_api.exceptions.NotFoundException +import za.co.absa.enceladus.rest_api.services.VersionedModelService + +import java.util.Optional +import java.util.concurrent.CompletableFuture + +abstract class VersionedModelControllerV3[C <: VersionedModel with Product + with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { + + import za.co.absa.enceladus.rest_api.utils.implicits._ + + import scala.concurrent.ExecutionContext.Implicits.global + + // todo maybe offset/limit? + @GetMapping(Array("")) + @ResponseStatus(HttpStatus.OK) + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption) + } + + @GetMapping(Array("/{name}")) + @ResponseStatus(HttpStatus.OK) + def getVersionsList(@PathVariable name: String): CompletableFuture[Seq[Int]] = { + versionedModelService.getAllVersionsValues(name) + } + + @GetMapping(Array("/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + def getVersionDetail(@PathVariable name: String, + @PathVariable version: Int): CompletableFuture[C] = { + versionedModelService.getVersion(name, version).map { + case Some(entity) => entity + case None => throw notFound() + } + } + + @GetMapping(Array("/{name}/latest")) + @ResponseStatus(HttpStatus.OK) + def getLatestDetail(@PathVariable name: String): CompletableFuture[C] = { + versionedModelService.getLatestVersion(name).map { + case Some(entity) => entity + case None => throw NotFoundException() + } + } + + @GetMapping(Array("/{name}/audit-trail")) + @ResponseStatus(HttpStatus.OK) + def getAuditTrail(@PathVariable name: String): CompletableFuture[AuditTrail] = { + versionedModelService.getAuditTrail(name) + } + + @GetMapping(Array("/{name}/{version}/used-in")) + @ResponseStatus(HttpStatus.OK) + def usedIn(@PathVariable name: String, + @PathVariable version: Int): CompletableFuture[UsedIn] = { + versionedModelService.getUsedIn(name, Some(version)) + } + + @GetMapping(Array("/{name}/{version}/export")) + @ResponseStatus(HttpStatus.OK) + def exportSingleEntity(@PathVariable name: String, @PathVariable version: Int): CompletableFuture[String] = { + versionedModelService.exportSingleItem(name, version) + } + + @GetMapping(Array("/{name}/export")) + @ResponseStatus(HttpStatus.OK) + def exportLatestEntity(@PathVariable name: String): CompletableFuture[String] = { + versionedModelService.exportLatestItem(name) + } + + @PostMapping(Array("/{name}/import")) + @ResponseStatus(HttpStatus.CREATED) + def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = { + // todo check that the name pathVar and object conform + versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { + case Some(entity) => entity // todo redo to have header Location present + case None => throw notFound() + } + } + + @PostMapping(Array("")) + @ResponseStatus(HttpStatus.CREATED) + def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C): CompletableFuture[C] = { + versionedModelService.isDisabled(item.name).flatMap { isDisabled => + if (isDisabled) { + versionedModelService.recreate(principal.getUsername, item) + } else { + versionedModelService.create(item, principal.getUsername) + } + }.map { + case Some(entity) => entity // todo redo to have header Location present + case None => throw notFound() + } + } + + @PutMapping(Array("")) + @ResponseStatus(HttpStatus.OK) + def edit(@AuthenticationPrincipal user: UserDetails, + @RequestBody item: C): CompletableFuture[C] = { + versionedModelService.update(user.getUsername, item).map { + case Some(entity) => entity + case None => throw notFound() + } + } + + @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + def disable(@PathVariable name: String, + @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { + val v = if (version.isPresent) { + // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException + Some(version.get.toInt) + } else { + None + } + versionedModelService.disableVersion(name, v) + } + +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index d59d7d6ad..32fad03d3 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -16,7 +16,6 @@ package za.co.absa.enceladus.rest_api.repositories import java.time.ZonedDateTime - import org.mongodb.scala._ import org.mongodb.scala.bson._ import org.mongodb.scala.bson.collection.immutable.Document @@ -28,7 +27,7 @@ import org.mongodb.scala.model.Updates._ import org.mongodb.scala.model._ import org.mongodb.scala.result.UpdateResult import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionsList} import scala.concurrent.Future import scala.reflect.ClassTag @@ -93,6 +92,18 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).headOption().map(_.map(_.latestVersion)) } + def getAllVersionsValues(name: String): Future[Seq[Int]] = { + val pipeline = Seq( + filter(getNameFilter(name)), + Aggregates.sort(Sorts.ascending("version")), + Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array + ) + collection.aggregate[VersionsList](pipeline).headOption().map { + case None => Seq.empty + case Some(verList) => verList.versions + } + } + def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { val filter = if (inclDisabled) getNameFilter(name) else getNameFilterEnabled(name) collection diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index b11fae9d0..04928607f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -56,6 +56,10 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getAllVersions(name) } + def getAllVersionsValues(name: String): Future[Seq[Int]] = { + versionedMongoRepository.getAllVersionsValues(name) + } + def getLatestVersion(name: String): Future[Option[C]] = { versionedMongoRepository.getLatestVersionValue(name).flatMap({ case Some(version) => getVersion(name, version) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala index 897bc0d17..c6aae6588 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala @@ -60,7 +60,7 @@ package object implicits { classOf[Run], classOf[Schema], classOf[SchemaField], classOf[SplineReference], classOf[RunSummary], classOf[RunDatasetNameGroupedSummary], classOf[RunDatasetVersionGroupedSummary], classOf[RuntimeConfig], classOf[OozieSchedule], classOf[OozieScheduleInstance], classOf[ScheduleTiming], classOf[DataFormat], - classOf[UserInfo], classOf[VersionedSummary], classOf[MenasAttachment], classOf[MenasReference], + classOf[UserInfo], classOf[VersionedSummary], classOf[VersionsList], classOf[MenasAttachment], classOf[MenasReference], classOf[PropertyDefinition], classOf[PropertyType], classOf[Essentiality], classOf[LandingPageInformation], classOf[TodaysRunsStatistics], classOf[DataFrameFilter] From 0f62851deaddb711b231ec23f1cd4326f78baeae Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 7 Mar 2022 17:06:12 +0100 Subject: [PATCH 02/67] #1693 API v3: some basic VersionedModelControllerV3 integration test - login allowed at /api/login & /api-v3/login - v2/v3 BaseRestApiTest distinquished --- .../rest_api/WebSecurityConfig.scala | 8 + .../v3/VersionedModelControllerV3.scala | 9 +- .../VersionedMongoRepository.scala | 7 +- .../services/VersionedModelService.scala | 4 +- .../AuthenticationIntegrationSuite.scala | 2 +- .../controllers/BaseRestApiTest.scala | 8 +- .../DatasetApiIntegrationSuite.scala | 2 +- ...ropertyDefinitionApiIntegrationSuite.scala | 2 +- .../controllers/RunApiIntegrationSuite.scala | 2 +- .../SchemaApiFeaturesIntegrationSuite.scala | 2 +- .../DatasetControllerV3IntegrationSuite.scala | 188 ++++++++++++++++++ 11 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala index 43ab92128..d7d8e1936 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala @@ -68,11 +68,19 @@ class WebSecurityConfig @Autowired()(beanFactory: BeanFactory, .anyRequest() .authenticated() .and() + // v2 login .formLogin() .loginProcessingUrl("/api/login") .successHandler(authenticationSuccessHandler) .failureHandler(authenticationFailureHandler) .permitAll() + .and() + // v3 login + .formLogin() + .loginProcessingUrl("/api-v3/login") + .successHandler(authenticationSuccessHandler) + .failureHandler(authenticationFailureHandler) + .permitAll() .and() .addFilterBefore(kerberosFilter, classOf[UsernamePasswordAuthenticationFilter]) .addFilterAfter(jwtAuthFilter, classOf[SpnegoAuthenticationProcessingFilter]) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 7193c95ca..78dd7002e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -46,8 +46,11 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionsList(@PathVariable name: String): CompletableFuture[Seq[Int]] = { - versionedModelService.getAllVersionsValues(name) + def getVersionsList(@PathVariable name: String): CompletableFuture[VersionsList] = { + versionedModelService.getAllVersionsValues(name) map { + case Some(entity) => entity + case None => throw notFound() + } } @GetMapping(Array("/{name}/{version}")) @@ -126,7 +129,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product def edit(@AuthenticationPrincipal user: UserDetails, @RequestBody item: C): CompletableFuture[C] = { versionedModelService.update(user.getUsername, item).map { - case Some(entity) => entity + case Some(entity) => entity // todo change not to return conent case None => throw notFound() } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 32fad03d3..f86972690 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -92,16 +92,13 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).headOption().map(_.map(_.latestVersion)) } - def getAllVersionsValues(name: String): Future[Seq[Int]] = { + def getAllVersionsValues(name: String): Future[Option[VersionsList]] = { val pipeline = Seq( filter(getNameFilter(name)), Aggregates.sort(Sorts.ascending("version")), Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array ) - collection.aggregate[VersionsList](pipeline).headOption().map { - case None => Seq.empty - case Some(verList) => verList.versions - } + collection.aggregate[VersionsList](pipeline).headOption().map(_.map(vlist => VersionsList("versions", vlist.versions))) } def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index 04928607f..d99072416 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -21,7 +21,7 @@ import org.springframework.security.core.context.SecurityContextHolder import org.springframework.security.core.userdetails.UserDetails import za.co.absa.enceladus.model.{ModelVersion, Schema, UsedIn, Validation} import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionsList} import za.co.absa.enceladus.rest_api.exceptions._ import za.co.absa.enceladus.rest_api.repositories.VersionedMongoRepository import za.co.absa.enceladus.model.menas.audit._ @@ -56,7 +56,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getAllVersions(name) } - def getAllVersionsValues(name: String): Future[Seq[Int]] = { + def getAllVersionsValues(name: String): Future[Option[VersionsList]] = { versionedMongoRepository.getAllVersionsValues(name) } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala index 209497733..43fd206b1 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala @@ -29,7 +29,7 @@ import scala.concurrent.{Await, Future} @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class AuthenticationIntegrationSuite extends BaseRestApiTest { +class AuthenticationIntegrationSuite extends BaseRestApiTestV2 { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 87fbcb3dc..46ef50596 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -32,7 +32,10 @@ import za.co.absa.enceladus.rest_api.integration.repositories.BaseRepositoryTest import scala.concurrent.Future import scala.reflect.ClassTag -abstract class BaseRestApiTest extends BaseRepositoryTest { +abstract class BaseRestApiTestV2 extends BaseRestApiTest("/api") +abstract class BaseRestApiTestV3 extends BaseRestApiTest("/api-v3") + +abstract class BaseRestApiTest(apiPath: String) extends BaseRepositoryTest { import scala.concurrent.ExecutionContext.Implicits.global @@ -50,7 +53,8 @@ abstract class BaseRestApiTest extends BaseRepositoryTest { @Value("${menas.auth.inmemory.admin.password}") val adminPasswd: String = "" - private lazy val baseUrl = s"http://localhost:$port/api" + // expecting apiPath to be /api for v2 and /api-v3 for v3 + private lazy val baseUrl = s"http://localhost:$port$apiPath" private lazy val authHeaders = getAuthHeaders(user, passwd) private lazy val authHeadersAdmin = getAuthHeaders(adminUser, adminPasswd) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala index 82dd8717d..837d7d1c2 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala @@ -35,7 +35,7 @@ import za.co.absa.enceladus.rest_api.integration.fixtures._ @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetApiIntegrationSuite extends BaseRestApiTest with BeforeAndAfterAll { +class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll { @Autowired private val datasetFixture: DatasetFixtureService = null diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala index 7c194a8d0..c8b9ad592 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala @@ -31,7 +31,7 @@ import za.co.absa.enceladus.model.test.factories.PropertyDefinitionFactory @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTest with BeforeAndAfterAll with Matchers { +class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll with Matchers { @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala index 51e383391..f17e17905 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala @@ -30,7 +30,7 @@ import za.co.absa.enceladus.model.{Run, SplineReference, Validation} @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class RunApiIntegrationSuite extends BaseRestApiTest { +class RunApiIntegrationSuite extends BaseRestApiTestV2 { import za.co.absa.enceladus.rest_api.integration.RunImplicits.RunExtensions import za.co.absa.enceladus.model.Validation._ diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala index 15a510f34..6251e5f71 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala @@ -47,7 +47,7 @@ import scala.collection.immutable.HashMap @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTest with BeforeAndAfterAll { +class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll { private val port = 8877 // same port as in test/resources/application.conf in the `menas.schemaRegistry.baseUrl` key private val wireMockServer = new WireMockServer(WireMockConfiguration.wireMockConfig().port(port)) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala new file mode 100644 index 000000000..07e8ab465 --- /dev/null +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -0,0 +1,188 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.integration.controllers.v3 + +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule +import za.co.absa.enceladus.model.dataFrameFilter._ +import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.properties.essentiality.Essentiality +import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ +import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, PropertyType, StringPropertyType} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} +import za.co.absa.enceladus.model.versionedModel.VersionsList +import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.rest_api.integration.fixtures._ +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTest, BaseRestApiTestV3, toExpected} + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll { + + @Autowired + private val datasetFixture: DatasetFixtureService = null + + @Autowired + private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + + private val apiUrl = "/datasets" + + // fixtures are cleared after each test + override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture) + + s"POST $apiUrl" can { + "return 201" when { + "a Dataset is created" should { + "return the created Dataset (with empty properties stripped)" in { + val dataset = DatasetFactory.getDummyDataset("dummyDs", + properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) + + val response = sendPost[Dataset, Dataset](s"$apiUrl", bodyOpt = Some(dataset)) + assertCreated(response) + + val actual = response.getBody + val expected = toExpected(dataset, actual).copy(properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))) // keyC stripped + assert(actual == expected) + } + } + } + } + + s"PUT $apiUrl" can { + "return 200" when { + "a Schema with the given name and version is the latest that exists" should { + "return the updated Schema (with empty properties stripped)" in { + val datasetA1 = DatasetFactory.getDummyDataset("datasetA", + description = Some("init version"), properties = Some(Map("keyA" -> "valA"))) + datasetFixture.add(datasetA1) + + val exampleMappingCr = MappingConformanceRule(0, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable", + mappingTableVersion = 9, //scalastyle:ignore magic.number + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + description = Some("updated"), + properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> "")), + conformance = List(exampleMappingCr) + ) + + val response = sendPut[Dataset, Dataset](s"$apiUrl", bodyOpt = Some(datasetA2)) + assertOk(response) + + // todo should be ok/no content and then actually no content -> run get again + val actual = response.getBody + val expectedDs = DatasetFactory.getDummyDataset( + name = "datasetA", + version = 2, + description = Some("updated"), + parent = Some(DatasetFactory.toParent(datasetA1)), + properties = Some(Map("keyA" -> "valA", "keyB" -> "valB")), + conformance = List(exampleMappingCr) + ) + val expected = toExpected(expectedDs, actual) + assert(actual == expected) + } + } + } + } + + s"GET $apiUrl/{name}" should { + "return 200" when { + "a Dataset with the given name exists - so it gives versions" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", + version = 2, + parent = Some(DatasetFactory.toParent(datasetV1))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[VersionsList](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == VersionsList("versions", Seq(1, 2))) + } + } + + "return 404" when { + "a Dataset with the given name does not exist" in { + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName") + assertNotFound(response) + } + } + } + + s"GET $apiUrl/{name}/{version}/export" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/2/export") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct Dataset version" should { + "return the exported Dataset representation" in { + val dataset = DatasetFactory.getDummyDataset(name = "dataset", version = 2, + properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) + datasetFixture.add(dataset) + val response = sendGet[String](s"$apiUrl/dataset/2/export") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"metadata":{"exportVersion":1},"item":{ + |"name":"dataset", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + + } + + // todo properties test for datasets or in general for any VersionedModel + +} From c01cfda48290e338979cfed51d0d541d0b17dcec Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 8 Mar 2022 09:53:52 +0100 Subject: [PATCH 03/67] #1693 API v3: VersionedModelControllerV3 integration test cont'd - test cases added --- .../DatasetControllerV3IntegrationSuite.scala | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 07e8ab465..d034c2dd7 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -56,15 +56,33 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) - val response = sendPost[Dataset, Dataset](s"$apiUrl", bodyOpt = Some(dataset)) + val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) val actual = response.getBody val expected = toExpected(dataset, actual).copy(properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))) // keyC stripped assert(actual == expected) + + // todo change this with location header } } } + "the dataset is disabled (i.e. all version are disabled)" should { + "create a new version of Dataset" in { + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) + datasetFixture.add(dataset1, dataset2) + + val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create + val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset3)) + + // todo change this with location header + val actual = response.getBody + val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) + assert(actual == expected) + } + } + // todo what to do if "the last dataset version is disabled" } s"PUT $apiUrl" can { From 4fd08e295533b0a642120ce2c920af4acc572823 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 21 Mar 2022 15:29:23 +0100 Subject: [PATCH 04/67] #1693 API v3: VersionedModelControllerV3 - post to yield location header and no content -> IT updated --- .../v3/VersionedModelControllerV3.scala | 22 ++++-- .../controllers/BaseRestApiTest.scala | 2 + .../DatasetControllerV3IntegrationSuite.scala | 78 ++++++++++++------- 3 files changed, 67 insertions(+), 35 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 78dd7002e..fee8584a9 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -16,10 +16,11 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import com.mongodb.client.result.UpdateResult -import org.springframework.http.HttpStatus +import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ +import org.springframework.web.servlet.support.ServletUriComponentsBuilder import za.co.absa.enceladus.model.menas.audit._ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn} @@ -27,8 +28,10 @@ import za.co.absa.enceladus.rest_api.controllers.BaseController import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService +import java.net.URI import java.util.Optional import java.util.concurrent.CompletableFuture +import javax.servlet.http.HttpServletRequest abstract class VersionedModelControllerV3[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -111,7 +114,9 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PostMapping(Array("")) @ResponseStatus(HttpStatus.CREATED) - def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C): CompletableFuture[C] = { + def create(@AuthenticationPrincipal principal: UserDetails, + @RequestBody item: C, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { versionedModelService.isDisabled(item.name).flatMap { isDisabled => if (isDisabled) { versionedModelService.recreate(principal.getUsername, item) @@ -119,8 +124,15 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product versionedModelService.create(item, principal.getUsername) } }.map { - case Some(entity) => entity // todo redo to have header Location present - case None => throw notFound() + case Some(entity) => + val location: URI = ServletUriComponentsBuilder + .fromRequest(request) + .path("/{name}/{version}") + .buildAndExpand(entity.name, entity.version.toString) + .toUri() // will create location e.g. /api/dataset/MyExampleDataset/1 + + ResponseEntity.created(location).build() + case None => throw notFound() } } @@ -129,7 +141,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product def edit(@AuthenticationPrincipal user: UserDetails, @RequestBody item: C): CompletableFuture[C] = { versionedModelService.update(user.getUsername, item).map { - case Some(entity) => entity // todo change not to return conent + case Some(entity) => entity // todo change not to return content case None => throw notFound() } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 46ef50596..9f0e619d7 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -253,4 +253,6 @@ abstract class BaseRestApiTest(apiPath: String) extends BaseRepositoryTest { assert(responseEntity.getStatusCode == HttpStatus.CREATED) } + def stripBaseUrl(fullUrl: String): String = fullUrl.stripPrefix(baseUrl) + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index d034c2dd7..a45fde450 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.integration.controllers.v3 import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -33,10 +34,12 @@ import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTest, BaseRestApiTestV3, toExpected} +import scala.collection.JavaConverters._ + @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll { +class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll with Matchers { @Autowired private val datasetFixture: DatasetFixtureService = null @@ -49,6 +52,33 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA // fixtures are cleared after each test override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture) + + s"GET $apiUrl/{name}" should { + "return 200" when { + "a Dataset with the given name exists - so it gives versions" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", + version = 2, + parent = Some(DatasetFactory.toParent(datasetV1))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[VersionsList](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == VersionsList("versions", Seq(1, 2))) + } + } + + "return 404" when { + "a Dataset with the given name does not exist" in { + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName") + assertNotFound(response) + } + } + } + s"POST $apiUrl" can { "return 201" when { "a Dataset is created" should { @@ -58,12 +88,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) + val locationHeaders = response.getHeaders.get("location").asScala + locationHeaders should have size 1 + val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. - val actual = response.getBody + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody val expected = toExpected(dataset, actual).copy(properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))) // keyC stripped - assert(actual == expected) - // todo change this with location header + assert(actual == expected) } } } @@ -75,10 +110,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset3)) + assertCreated(response) + val locationHeaders = response.getHeaders.get("location").asScala + locationHeaders should have size 1 + val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. + + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) - // todo change this with location header - val actual = response.getBody + val actual = response2.getBody val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) + assert(actual == expected) } } @@ -140,31 +182,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"GET $apiUrl/{name}" should { - "return 200" when { - "a Dataset with the given name exists - so it gives versions" in { - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", - version = 2, - parent = Some(DatasetFactory.toParent(datasetV1))) - datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[VersionsList](s"$apiUrl/datasetA") - assertOk(response) - assert(response.getBody == VersionsList("versions", Seq(1, 2))) - } - } - - "return 404" when { - "a Dataset with the given name does not exist" in { - val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(dataset) - - val response = sendGet[String](s"$apiUrl/anotherDatasetName") - assertNotFound(response) - } - } - } s"GET $apiUrl/{name}/{version}/export" should { "return 404" when { From 5c8e0051f22708b4a7f7b235a7666214e70f60ec Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 23 Mar 2022 09:42:30 +0100 Subject: [PATCH 05/67] #1693 API v3: VersionedModelControllerV3 - put to yield no content -> IT updated --- .../controllers/RestExceptionHandler.scala | 5 ++ .../v3/VersionedModelControllerV3.scala | 30 ++++++---- .../DatasetControllerV3IntegrationSuite.scala | 55 ++++++++++++------- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala index 5fa14a03b..5ef13e650 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala @@ -44,6 +44,11 @@ class RestExceptionHandler { private val logger = LoggerFactory.getLogger(this.getClass) + @ExceptionHandler(value = Array(classOf[IllegalArgumentException])) + def handleIllegalArgumentException(exception: IllegalArgumentException): ResponseEntity[Any] = { + ResponseEntity.badRequest().body(exception.getMessage) + } + @ExceptionHandler(value = Array(classOf[AsyncRequestTimeoutException])) def handleAsyncRequestTimeoutException(exception: AsyncRequestTimeoutException): ResponseEntity[Any] = { val message = Option(exception.getMessage).getOrElse("Request timeout expired.") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index fee8584a9..a07872178 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -32,6 +32,7 @@ import java.net.URI import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest +import scala.concurrent.Future abstract class VersionedModelControllerV3[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -71,7 +72,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product def getLatestDetail(@PathVariable name: String): CompletableFuture[C] = { versionedModelService.getLatestVersion(name).map { case Some(entity) => entity - case None => throw NotFoundException() + case None => throw NotFoundException() } } @@ -84,7 +85,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("/{name}/{version}/used-in")) @ResponseStatus(HttpStatus.OK) def usedIn(@PathVariable name: String, - @PathVariable version: Int): CompletableFuture[UsedIn] = { + @PathVariable version: Int): CompletableFuture[UsedIn] = { versionedModelService.getUsedIn(name, Some(version)) } @@ -108,7 +109,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product // todo check that the name pathVar and object conform versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { case Some(entity) => entity // todo redo to have header Location present - case None => throw notFound() + case None => throw notFound() } } @@ -136,20 +137,29 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } - @PutMapping(Array("")) - @ResponseStatus(HttpStatus.OK) + @PutMapping(Array("/{name}/{version}")) + @ResponseStatus(HttpStatus.NO_CONTENT) def edit(@AuthenticationPrincipal user: UserDetails, - @RequestBody item: C): CompletableFuture[C] = { - versionedModelService.update(user.getUsername, item).map { - case Some(entity) => entity // todo change not to return content - case None => throw notFound() + @PathVariable name: String, + @PathVariable version: Int, + @RequestBody item: C): CompletableFuture[ResponseEntity[Nothing]] = { + + if (name != item.name) { + Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${item.name}'")) + } else if (version != item.version) { + Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) + } else { + versionedModelService.update(user.getUsername, item).map { + case Some(entity) => ResponseEntity.noContent().build() + case None => throw notFound() + } } } @DeleteMapping(Array("/{name}", "/{name}/{version}")) @ResponseStatus(HttpStatus.OK) def disable(@PathVariable name: String, - @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { + @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { val v = if (version.isPresent) { // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException Some(version.get.toInt) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index a45fde450..eee12729e 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -20,6 +20,7 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule @@ -109,7 +110,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1, dataset2) val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create - val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset3)) + val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) assertCreated(response) val locationHeaders = response.getHeaders.get("location").asScala locationHeaders should have size 1 @@ -129,11 +130,14 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"PUT $apiUrl" can { "return 200" when { - "a Schema with the given name and version is the latest that exists" should { - "return the updated Schema (with empty properties stripped)" in { + "a Dataset with the given name and version is the latest that exists" should { + "update the dataset (with empty properties stripped)" in { val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version"), properties = Some(Map("keyA" -> "valA"))) - datasetFixture.add(datasetA1) + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + description = Some("second version"), properties = Some(Map("keyA" -> "valA")), version = 2) + + datasetFixture.add(datasetA1, datasetA2) val exampleMappingCr = MappingConformanceRule(0, controlCheckpoint = true, @@ -156,33 +160,44 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA overrideMappingTableOwnFilter = Some(true) ) - val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + val datasetA3 = DatasetFactory.getDummyDataset("datasetA", description = Some("updated"), properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> "")), - conformance = List(exampleMappingCr) + conformance = List(exampleMappingCr), + version = 2 // update references the last version ) - val response = sendPut[Dataset, Dataset](s"$apiUrl", bodyOpt = Some(datasetA2)) - assertOk(response) + val response = sendPut[Dataset, String](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) + response.getStatusCode shouldBe HttpStatus.NO_CONTENT - // todo should be ok/no content and then actually no content -> run get again - val actual = response.getBody - val expectedDs = DatasetFactory.getDummyDataset( - name = "datasetA", - version = 2, - description = Some("updated"), - parent = Some(DatasetFactory.toParent(datasetA1)), - properties = Some(Map("keyA" -> "valA", "keyB" -> "valB")), - conformance = List(exampleMappingCr) - ) - val expected = toExpected(expectedDs, actual) + val response2 = sendGet[Dataset](s"$apiUrl/datasetA/3") // next version + assertOk(response2) + val actual = response2.getBody + val expected = toExpected(datasetA3.copy(version = 3, parent = Some(DatasetFactory.toParent(datasetA2)), properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))), actual) // blank property stripped assert(actual == expected) } } } - } + "return 405" when { + "a Dataset with the given name and version" should { + "fail when version/name in the URL and payload is mismatched" in { + val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) + datasetFixture.add(datasetA1) + + val response = sendPut[Dataset, String](s"$apiUrl/datasetA/7", + bodyOpt = Some(DatasetFactory.getDummyDataset("datasetA", version = 5))) + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody should include("version mismatch: 7 != 5") + val response2 = sendPut[Dataset, String](s"$apiUrl/datasetABC/4", + bodyOpt = Some(DatasetFactory.getDummyDataset("datasetXYZ", version = 4))) + response2.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response2.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") + } + } + } + } s"GET $apiUrl/{name}/{version}/export" should { "return 404" when { From 458bfed98714aa14167492ec858c8d44a3bdc75f Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 23 Mar 2022 10:36:44 +0100 Subject: [PATCH 06/67] #1693 API v3: VersionedModelControllerV3 - IT for /ds/export (latest version, includes a sample CR) --- .../DatasetControllerV3IntegrationSuite.scala | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index eee12729e..9ae55cd96 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -23,7 +23,7 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule +import za.co.absa.enceladus.model.conformanceRule.{LiteralConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality @@ -179,6 +179,45 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + s"GET $apiUrl/{name}/export" should { + "return 404" when { + "when the name does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/export") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct Dataset" should { + "return the exported Dataset representation for the latest version" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), + properties = Some(Map("key1" -> "val1", "key2" -> "val2")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) + ) + datasetFixture.add(dataset1, dataset2) + val response = sendGet[String](s"$apiUrl/dataset/export") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"metadata":{"exportVersion":1},"item":{ + |"name":"dataset", + |"description":"Hi, I am the latest version", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + + } + "return 405" when { "a Dataset with the given name and version" should { "fail when version/name in the URL and payload is mismatched" in { @@ -210,9 +249,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "there is a correct Dataset version" should { "return the exported Dataset representation" in { - val dataset = DatasetFactory.getDummyDataset(name = "dataset", version = 2, + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) - datasetFixture.add(dataset) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset", version = 3, description = Some("showing non-latest export")) + datasetFixture.add(dataset2, dataset3) val response = sendGet[String](s"$apiUrl/dataset/2/export") assertOk(response) From a7a6cf952b259012a6dee54e0fcf4556f139af7b Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 23 Mar 2022 15:20:53 +0100 Subject: [PATCH 07/67] #1693 API v3: VersionedModelControllerV3 - location header for put/post and post-import + IT to prove correct behavior --- .../v3/VersionedModelControllerV3.scala | 43 +++-- .../DatasetControllerV3IntegrationSuite.scala | 150 +++++++++++++----- 2 files changed, 134 insertions(+), 59 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index a07872178..694d9d581 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -105,11 +105,17 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @ResponseStatus(HttpStatus.CREATED) def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, - @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = { - // todo check that the name pathVar and object conform - versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { - case Some(entity) => entity // todo redo to have header Location present - case None => throw notFound() + @RequestBody importObject: ExportableObject[C], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + if (name != importObject.item.name) { + Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${importObject.item.name}'")) + } else { + versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { + case Some(entity) => + // stripping two last segments, instead of /api-v3/dastasets/dsName/import + /dsName/dsVersion we want /api-v3/dastasets + /dsName/dsVersion + createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 2) + case None => throw notFound() + } } } @@ -125,14 +131,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product versionedModelService.create(item, principal.getUsername) } }.map { - case Some(entity) => - val location: URI = ServletUriComponentsBuilder - .fromRequest(request) - .path("/{name}/{version}") - .buildAndExpand(entity.name, entity.version.toString) - .toUri() // will create location e.g. /api/dataset/MyExampleDataset/1 - - ResponseEntity.created(location).build() + case Some(entity) => createdWithNameVersionLocation(entity.name, entity.version, request) case None => throw notFound() } } @@ -142,7 +141,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product def edit(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: Int, - @RequestBody item: C): CompletableFuture[ResponseEntity[Nothing]] = { + @RequestBody item: C, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { if (name != item.name) { Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${item.name}'")) @@ -150,7 +150,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) } else { versionedModelService.update(user.getUsername, item).map { - case Some(entity) => ResponseEntity.noContent().build() + case Some(entity) => createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 2) case None => throw notFound() } } @@ -169,4 +169,17 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product versionedModelService.disableVersion(name, v) } + def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, + stripLastSegments: Int = 0): ResponseEntity[Nothing] = { + val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString + + val location: URI = ServletUriComponentsBuilder.fromRequest(request) + .path(s"$strippingPrefix/{name}/{version}") + .buildAndExpand(name, version.toString) + .normalize() // will normalize `/one/two/../three` into `/one/tree` + .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 + + ResponseEntity.created(location).build() + } + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 9ae55cd96..7d98204a6 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -89,10 +89,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) - val locationHeaders = response.getHeaders.get("location").asScala - locationHeaders should have size 1 - val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. + val locationHeader = response.getHeaders.get("location").asScala.headOption + locationHeader shouldBe defined + locationHeader.get should endWith("/api-v3/datasets/dummyDs/1") + val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) @@ -168,54 +169,21 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ) val response = sendPut[Dataset, String](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) - response.getStatusCode shouldBe HttpStatus.NO_CONTENT + assertCreated(response) + val locationHeader = response.getHeaders.get("location").asScala.headOption + locationHeader shouldBe defined + locationHeader.get should endWith("/api-v3/datasets/datasetA/3") - val response2 = sendGet[Dataset](s"$apiUrl/datasetA/3") // next version + val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) + val actual = response2.getBody val expected = toExpected(datasetA3.copy(version = 3, parent = Some(DatasetFactory.toParent(datasetA2)), properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))), actual) // blank property stripped - assert(actual == expected) - } - } - } - - s"GET $apiUrl/{name}/export" should { - "return 404" when { - "when the name does not exist" in { - val response = sendGet[String](s"$apiUrl/notFoundDataset/export") - assertNotFound(response) - } - } - "return 200" when { - "there is a correct Dataset" should { - "return the exported Dataset representation for the latest version" in { - val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") - val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), - properties = Some(Map("key1" -> "val1", "key2" -> "val2")), - conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) - ) - datasetFixture.add(dataset1, dataset2) - val response = sendGet[String](s"$apiUrl/dataset/export") - - assertOk(response) - - val body = response.getBody - assert(body == - """{"metadata":{"exportVersion":1},"item":{ - |"name":"dataset", - |"description":"Hi, I am the latest version", - |"hdfsPath":"/dummy/path", - |"hdfsPublishPath":"/dummy/publish/path", - |"schemaName":"dummySchema", - |"schemaVersion":1, - |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], - |"properties":{"key2":"val2","key1":"val1"} - |}}""".stripMargin.replaceAll("[\\r\\n]", "")) - } + assert(actual == expected) } } - } "return 405" when { @@ -238,6 +206,100 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + s"GET $apiUrl/{name}/export" should { + "return 404" when { + "when the name does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/export") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct Dataset" should { + "return the exported Dataset representation for the latest version" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), + properties = Some(Map("key1" -> "val1", "key2" -> "val2")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) + ) + datasetFixture.add(dataset1, dataset2) + val response = sendGet[String](s"$apiUrl/dataset/export") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"metadata":{"exportVersion":1},"item":{ + |"name":"dataset", + |"description":"Hi, I am the latest version", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + + } + + s"POST $apiUrl/{name}/import" should { + val importableDs = + """{"metadata":{"exportVersion":1},"item":{ + |"name":"datasetXYZ", + |"description":"Hi, I am the latest version", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "") + + "return 405" when { + "a Dataset with the given name" should { + "fail when name in the URL and payload is mismatched" in { + val response = sendPost[String, String](s"$apiUrl/datasetABC/import", + bodyOpt = Some(importableDs)) + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") + } + } + } + +// "return 201" when { // todo create+update +// "there is a correct Dataset" should { +// "return the exported Dataset representation for the latest version" in { +// val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") +// val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), +// properties = Some(Map("key1" -> "val1", "key2" -> "val2")), +// conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) +// ) +// datasetFixture.add(dataset1, dataset2) +// val response = sendGet[String](s"$apiUrl/dataset/export") +// +// assertOk(response) +// +// val body = response.getBody +// assert(body == +// """{"metadata":{"exportVersion":1},"item":{ +// |"name":"dataset", +// |"description":"Hi, I am the latest version", +// |"hdfsPath":"/dummy/path", +// |"hdfsPublishPath":"/dummy/publish/path", +// |"schemaName":"dummySchema", +// |"schemaVersion":1, +// |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], +// |"properties":{"key2":"val2","key1":"val1"} +// |}}""".stripMargin.replaceAll("[\\r\\n]", "")) +// } +// } +// } + + } + s"GET $apiUrl/{name}/{version}/export" should { "return 404" when { "when the name+version does not exist" in { From af489b13b212128a42e20f8924a6e59579f04345 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 24 Mar 2022 11:22:30 +0100 Subject: [PATCH 08/67] #1693 API v3: VersionedModelControllerV3 - location header for post-import + IT to prove correct behavior - fix with common location processing with segment stripping (+normalization) --- .../v3/VersionedModelControllerV3.scala | 4 +- .../DatasetControllerV3IntegrationSuite.scala | 91 ++++++++++++------- 2 files changed, 63 insertions(+), 32 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 694d9d581..e471c0ac9 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -169,7 +169,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product versionedModelService.disableVersion(name, v) } - def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, + protected def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, stripLastSegments: Int = 0): ResponseEntity[Nothing] = { val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString @@ -179,6 +179,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product .normalize() // will normalize `/one/two/../three` into `/one/tree` .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 + // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918 + ResponseEntity.created(location).build() } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 7d98204a6..b217b1e91 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -29,7 +29,7 @@ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, PropertyType, StringPropertyType} -import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.VersionsList import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -45,13 +45,16 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA @Autowired private val datasetFixture: DatasetFixtureService = null + @Autowired + private val schemaFixture: SchemaFixtureService = null + @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null private val apiUrl = "/datasets" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture) + override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture) s"GET $apiUrl/{name}" should { @@ -249,7 +252,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val importableDs = """{"metadata":{"exportVersion":1},"item":{ |"name":"datasetXYZ", - |"description":"Hi, I am the latest version", + |"description":"Hi, I am the import", |"hdfsPath":"/dummy/path", |"hdfsPublishPath":"/dummy/publish/path", |"schemaName":"dummySchema", @@ -269,34 +272,60 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } -// "return 201" when { // todo create+update -// "there is a correct Dataset" should { -// "return the exported Dataset representation for the latest version" in { -// val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") -// val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), -// properties = Some(Map("key1" -> "val1", "key2" -> "val2")), -// conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) -// ) -// datasetFixture.add(dataset1, dataset2) -// val response = sendGet[String](s"$apiUrl/dataset/export") -// -// assertOk(response) -// -// val body = response.getBody -// assert(body == -// """{"metadata":{"exportVersion":1},"item":{ -// |"name":"dataset", -// |"description":"Hi, I am the latest version", -// |"hdfsPath":"/dummy/path", -// |"hdfsPublishPath":"/dummy/publish/path", -// |"schemaName":"dummySchema", -// |"schemaVersion":1, -// |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], -// |"properties":{"key2":"val2","key1":"val1"} -// |}}""".stripMargin.replaceAll("[\\r\\n]", "")) -// } -// } -// } + "return 201" when { + "there is a existing Dataset" should { + "a +1 version of dataset is added" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("init version")) + datasetFixture.add(dataset1) + + val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + assertCreated(response) + val locationHeader = response.getHeaders.get("location").asScala.headOption + locationHeader shouldBe defined + locationHeader.get should endWith("/api-v3/datasets/datasetXYZ/2") + + val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = DatasetFactory.getDummyDataset(name = "datasetXYZ", version = 2, description = Some("Hi, I am the import"), + properties = Some(Map("key1" -> "val1", "key2" -> "val2")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), + parent = Some(DatasetFactory.toParent(dataset1)) + ) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + + "there is no such Dataset, yet" should { + "a the version of dataset created" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + + val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + assertCreated(response) + val locationHeader = response.getHeaders.get("location").asScala.headOption + locationHeader shouldBe defined + locationHeader.get should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version + + val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("Hi, I am the import"), + properties = Some(Map("key1" -> "val1", "key2" -> "val2")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) + ) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + } } From 21f800876df19712c8015c02f66c5d2c5de62b77 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 24 Mar 2022 12:33:06 +0100 Subject: [PATCH 09/67] #1693 API v3: VersionedModelControllerV3 -IT for GET datasets/{name}, GET datasets/{name}/{version} --- .../DatasetControllerV3IntegrationSuite.scala | 115 ++++++++++++++---- 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index b217b1e91..a01da6437 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -57,32 +57,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture) - s"GET $apiUrl/{name}" should { - "return 200" when { - "a Dataset with the given name exists - so it gives versions" in { - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", - version = 2, - parent = Some(DatasetFactory.toParent(datasetV1))) - datasetFixture.add(datasetV1, datasetV2) - - val response = sendGet[VersionsList](s"$apiUrl/datasetA") - assertOk(response) - assert(response.getBody == VersionsList("versions", Seq(1, 2))) - } - } - - "return 404" when { - "a Dataset with the given name does not exist" in { - val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(dataset) - - val response = sendGet[String](s"$apiUrl/anotherDatasetName") - assertNotFound(response) - } - } - } - s"POST $apiUrl" can { "return 201" when { "a Dataset is created" should { @@ -132,7 +106,94 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA // todo what to do if "the last dataset version is disabled" } - s"PUT $apiUrl" can { + s"GET $apiUrl/{name}" should { + "return 200" when { + "a Dataset with the given name exists - so it gives versions" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", + version = 2, + parent = Some(DatasetFactory.toParent(datasetV1))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[VersionsList](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == VersionsList("versions", Seq(1, 2))) + } + } + + "return 404" when { + "a Dataset with the given name does not exist" in { + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName") + assertNotFound(response) + } + } + } + + s"GET $apiUrl/{name}/latest" should { + "return 200" when { + "a Dataset with the given name exists - gives latest version entity" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", + version = 2, parent = Some(DatasetFactory.toParent(datasetV1))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[Dataset](s"$apiUrl/datasetA/latest") + assertOk(response) + + val actual = response.getBody + val expected = toExpected(datasetV2, actual) + + assert(actual == expected) + } + } + + "return 404" when { + "a Dataset with the given name does not exist" in { + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName/latest") + assertNotFound(response) + } + } + } + + s"GET $apiUrl/{name}/{version}" should { + "return 200" when { + "a Dataset with the given name and version exists - gives specified version of entity" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, description = Some("second")) + val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3, description = Some("third")) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[Dataset](s"$apiUrl/datasetA/2") + assertOk(response) + + val actual = response.getBody + val expected = toExpected(datasetV2, actual) + + assert(actual == expected) + } + } + + "return 404" when { + "a Dataset with the given name/version does not exist" in { + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName/1") + assertNotFound(response) + + val response2 = sendGet[String](s"$apiUrl/datasetA/7") + assertNotFound(response2) + } + } + } + + s"PUT $apiUrl/{name}/{version}" can { "return 200" when { "a Dataset with the given name and version is the latest that exists" should { "update the dataset (with empty properties stripped)" in { From 6f63117efd946a9971bb5e0f23501ef7c95bfef5 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 24 Mar 2022 13:33:03 +0100 Subject: [PATCH 10/67] #1693 API v3: VersionedModelControllerV3 - IT for GET datasets/{name}/audit-trail added --- .../DatasetControllerV3IntegrationSuite.scala | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index a01da6437..0ca647a52 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -270,6 +270,49 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + s"GET $apiUrl/{name}/audit-trail" should { + "return 404" when { + "when the name does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/audit-trail") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct Dataset" should { + "return an audit trail for the dataset" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA") + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), + parent = Some(DatasetFactory.toParent(dataset1)) + ) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, + properties = Some(Map("key1" -> "val1")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), // untouched + parent = Some(DatasetFactory.toParent(dataset2)) + ) + + datasetFixture.add(dataset1, dataset2) + val response = sendGet[String](s"$apiUrl/datasetA/audit-trail") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"entries":[{ + |"menasRef":{"collection":null,"name":"datasetA","version":2}, + |"updatedBy":"dummyUser","updated":"2017-12-04T16:19:17Z", + |"changes":[{"field":"conformance","oldValue":null,"newValue":"LiteralConformanceRule(0,outputCol1,false,litValue1)","message":"Conformance rule added."}] + |},{ + |"menasRef":{"collection":null,"name":"datasetA","version":1}, + |"updatedBy":"dummyUser","updated":"2017-12-04T16:19:17Z", + |"changes":[{"field":"","oldValue":null,"newValue":null,"message":"Dataset datasetA created."}] + |}]}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + } + s"GET $apiUrl/{name}/export" should { "return 404" when { "when the name does not exist" in { From 76e87ce0bc5851bd52b360163bed97d5dd6191a6 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 24 Mar 2022 14:16:13 +0100 Subject: [PATCH 11/67] #1693 API v3: VersionedModelControllerV3 - GET /datasets/{name}/{version} now works for # or 'latest' (IT = regression test) --- .../v3/VersionedModelControllerV3.scala | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index e471c0ac9..51b5b5efb 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -25,6 +25,7 @@ import za.co.absa.enceladus.model.menas.audit._ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn} import za.co.absa.enceladus.rest_api.controllers.BaseController +import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService @@ -33,6 +34,7 @@ import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest import scala.concurrent.Future +import scala.util.{Failure, Success, Try} abstract class VersionedModelControllerV3[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -60,19 +62,27 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("/{name}/{version}")) @ResponseStatus(HttpStatus.OK) def getVersionDetail(@PathVariable name: String, - @PathVariable version: Int): CompletableFuture[C] = { - versionedModelService.getVersion(name, version).map { + @PathVariable version: String): CompletableFuture[C] = { + forVersionExpression(version) { actualVersion: Int => + versionedModelService.getVersion(name, actualVersion) + }(versionedModelService.getLatestVersion(name)).map { case Some(entity) => entity case None => throw notFound() } } - @GetMapping(Array("/{name}/latest")) - @ResponseStatus(HttpStatus.OK) - def getLatestDetail(@PathVariable name: String): CompletableFuture[C] = { - versionedModelService.getLatestVersion(name).map { - case Some(entity) => entity - case None => throw NotFoundException() + protected def forVersionExpression(versionStr: String) + (doForNumberedVersion: Int => Future[Option[C]]) + (doForLatest: Future[Option[C]]): Future[Option[C]] = { + if (versionStr.toLowerCase == LatestVersionKey) { + doForLatest + } else { + Try(versionStr.toInt) match { + case Success(actualVersion) => doForNumberedVersion(actualVersion) + case Failure(exception) => + Future.failed(new IllegalArgumentException(s"Cannot convert '$versionStr' to a valid version expression. " + + s"Either use 'latest' or an actual version number. Underlying problem: ${exception.getMessage}")) + } } } @@ -170,14 +180,14 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } protected def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, - stripLastSegments: Int = 0): ResponseEntity[Nothing] = { + stripLastSegments: Int = 0): ResponseEntity[Nothing] = { val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString val location: URI = ServletUriComponentsBuilder.fromRequest(request) - .path(s"$strippingPrefix/{name}/{version}") - .buildAndExpand(name, version.toString) - .normalize() // will normalize `/one/two/../three` into `/one/tree` - .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 + .path(s"$strippingPrefix/{name}/{version}") + .buildAndExpand(name, version.toString) + .normalize() // will normalize `/one/two/../three` into `/one/tree` + .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918 @@ -185,3 +195,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } + +object VersionedModelControllerV3 { + val LatestVersionKey = "latest" +} From d62cd9091c9d7261a7642fa8d4bc63795a0369bb Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 24 Mar 2022 15:38:42 +0100 Subject: [PATCH 12/67] #1693 API v3: VersionedModelControllerV3 - common implementation of /{name}/{version} and /{name}/latest - improved --- .../v3/VersionedModelControllerV3.scala | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 51b5b5efb..73a133ead 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -63,28 +63,13 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @ResponseStatus(HttpStatus.OK) def getVersionDetail(@PathVariable name: String, @PathVariable version: String): CompletableFuture[C] = { - forVersionExpression(version) { actualVersion: Int => - versionedModelService.getVersion(name, actualVersion) - }(versionedModelService.getLatestVersion(name)).map { + forVersionExpression(name, version)(versionedModelService.getVersion).map { case Some(entity) => entity case None => throw notFound() } } - protected def forVersionExpression(versionStr: String) - (doForNumberedVersion: Int => Future[Option[C]]) - (doForLatest: Future[Option[C]]): Future[Option[C]] = { - if (versionStr.toLowerCase == LatestVersionKey) { - doForLatest - } else { - Try(versionStr.toInt) match { - case Success(actualVersion) => doForNumberedVersion(actualVersion) - case Failure(exception) => - Future.failed(new IllegalArgumentException(s"Cannot convert '$versionStr' to a valid version expression. " + - s"Either use 'latest' or an actual version number. Underlying problem: ${exception.getMessage}")) - } - } - } + @GetMapping(Array("/{name}/audit-trail")) @ResponseStatus(HttpStatus.OK) @@ -96,19 +81,19 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @ResponseStatus(HttpStatus.OK) def usedIn(@PathVariable name: String, @PathVariable version: Int): CompletableFuture[UsedIn] = { - versionedModelService.getUsedIn(name, Some(version)) + versionedModelService.getUsedIn(name, Some(version)) // todo use forVersionExpression, too } @GetMapping(Array("/{name}/{version}/export")) @ResponseStatus(HttpStatus.OK) - def exportSingleEntity(@PathVariable name: String, @PathVariable version: Int): CompletableFuture[String] = { - versionedModelService.exportSingleItem(name, version) + def exportSingleEntity(@PathVariable name: String, @PathVariable version: String): CompletableFuture[String] = { + forVersionExpression(name, version)(versionedModelService.exportSingleItem) } @GetMapping(Array("/{name}/export")) @ResponseStatus(HttpStatus.OK) def exportLatestEntity(@PathVariable name: String): CompletableFuture[String] = { - versionedModelService.exportLatestItem(name) + versionedModelService.exportLatestItem(name) // todo: remove in favor of the above? (that supports /{name}/latest/export) } @PostMapping(Array("/{name}/import")) @@ -179,6 +164,31 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product versionedModelService.disableVersion(name, v) } + /** + * For entity's name and version expression (either a number or 'latest'), the forVersionFn is called. + * + * @param name + * @param versionStr + * @param forVersionFn + * @return + */ + protected def forVersionExpression[T](name: String, versionStr: String) + (forVersionFn: (String, Int) => Future[T]): Future[T] = { + versionStr.toLowerCase match { + case LatestVersionKey => + versionedModelService.getLatestVersionValue(name).flatMap { + case None => Future.failed(notFound()) + case Some(actualLatestVersion) => forVersionFn(name, actualLatestVersion) + } + case nonLatestVersionString => Try(nonLatestVersionString.toInt) match { + case Success(actualVersion) => forVersionFn(name, actualVersion) + case Failure(exception) => + Future.failed(new IllegalArgumentException(s"Cannot convert '$versionStr' to a valid version expression. " + + s"Either use 'latest' or an actual version number. Underlying problem: ${exception.getMessage}")) + } + } + } + protected def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, stripLastSegments: Int = 0): ResponseEntity[Nothing] = { val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString From fc4c359bde2c702466291ac1bbbaead3c8a12f8d Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 24 Mar 2022 16:46:27 +0100 Subject: [PATCH 13/67] #1693 API v3: VersionedModelControllerV3 - /{name}/{version}/used-in - supports latest for as version-expression, impl for datasets improved by actual existence checking + IT test cases for non-existing/non-latest queries --- .../v3/VersionedModelControllerV3.scala | 5 +-- .../rest_api/services/DatasetService.scala | 11 ++++- .../DatasetControllerV3IntegrationSuite.scala | 45 ++++++++++++++++++- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 73a133ead..5c930e842 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -70,7 +70,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } - @GetMapping(Array("/{name}/audit-trail")) @ResponseStatus(HttpStatus.OK) def getAuditTrail(@PathVariable name: String): CompletableFuture[AuditTrail] = { @@ -80,8 +79,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("/{name}/{version}/used-in")) @ResponseStatus(HttpStatus.OK) def usedIn(@PathVariable name: String, - @PathVariable version: Int): CompletableFuture[UsedIn] = { - versionedModelService.getUsedIn(name, Some(version)) // todo use forVersionExpression, too + @PathVariable version: String): CompletableFuture[UsedIn] = { + forVersionExpression(name, version) { case (name, versionInt) => versionedModelService.getUsedIn(name, Some(versionInt)) } } @GetMapping(Array("/{name}/{version}/export")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 368ffe6bb..309b53809 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -28,6 +28,7 @@ import za.co.absa.enceladus.model.properties.essentiality.Mandatory import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel import DatasetService._ +import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import scala.concurrent.Future @@ -93,7 +94,15 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { - Future.successful(UsedIn()) + val existingEntityF = version match { + case Some(version) => getVersion(name, version) + case None => getLatestVersion(name) + } + + existingEntityF.flatMap { + case Some(_) => Future.successful(UsedIn()) // empty usedIn for existing datasets + case None => Future.failed(NotFoundException(s"Dataset '$name' in version ${version.getOrElse("any")}' nof found")) + } } override def create(newDataset: Dataset, username: String): Future[Option[Dataset]] = { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 0ca647a52..be79264ff 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -31,7 +31,7 @@ import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, PropertyType, StringPropertyType} import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.VersionsList -import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTest, BaseRestApiTestV3, toExpected} @@ -466,7 +466,50 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } } + } + + s"GET $apiUrl/{name}/{version}/export" should { + "return 404" when { + "when the dataset of latest version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/latest/used-in") + assertNotFound(response) + } + } + + "return 404" when { + "when the dataset of name/version does not exist" in { + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetA) + + val response = sendGet[String](s"$apiUrl/notFoundDataset/1/used-in") + assertNotFound(response) + + val response2 = sendGet[String](s"$apiUrl/datasetA/7/used-in") + assertNotFound(response2) + } + } + "return 200" when { + "any exiting latest dataset" in { + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetA) + val response = sendGet[UsedIn](s"$apiUrl/datasetA/latest/used-in") + assertOk(response) + + response.getBody shouldBe UsedIn(None, None) + } + } + + "return 200" when { + "for existing name+version for dataset" in { + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2) + datasetFixture.add(dataset2) + val response = sendGet[UsedIn](s"$apiUrl/dataset/2/used-in") + + assertOk(response) + response.getBody shouldBe UsedIn(None, None) + } + } } // todo properties test for datasets or in general for any VersionedModel From e6c8d8ce4c05cfdba25923b17710028f55240233 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 29 Mar 2022 15:17:41 +0200 Subject: [PATCH 14/67] #1693 API v3: VersionedModelControllerV3 - GET/PUT /{name}/{version}/properties - supports latest for as version-expression; get impl is unvalidated, put impl checks validity - login is now common, under /api/login for both v2 and v3 (did not work previously) --- .../rest_api/WebSecurityConfig.scala | 8 -- .../controllers/DatasetController.scala | 2 +- .../VersionedModelController.scala | 5 +- .../controllers/v3/DatasetControllerV3.scala | 39 +++++++ .../v3/VersionedModelControllerV3.scala | 8 +- .../rest_api/services/DatasetService.scala | 36 +++++- .../controllers/BaseRestApiTest.scala | 9 +- .../DatasetControllerV3IntegrationSuite.scala | 108 +++++++++++++++++- 8 files changed, 189 insertions(+), 26 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala index d7d8e1936..43ab92128 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/WebSecurityConfig.scala @@ -68,19 +68,11 @@ class WebSecurityConfig @Autowired()(beanFactory: BeanFactory, .anyRequest() .authenticated() .and() - // v2 login .formLogin() .loginProcessingUrl("/api/login") .successHandler(authenticationSuccessHandler) .failureHandler(authenticationFailureHandler) .permitAll() - .and() - // v3 login - .formLogin() - .loginProcessingUrl("/api-v3/login") - .successHandler(authenticationSuccessHandler) - .failureHandler(authenticationFailureHandler) - .permitAll() .and() .addFilterBefore(kerberosFilter, classOf[UsernamePasswordAuthenticationFilter]) .addFilterAfter(jwtAuthFilter, classOf[SpnegoAuthenticationProcessingFilter]) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index 0f33bf7d1..b56396087 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -113,7 +113,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def replaceProperties(@AuthenticationPrincipal principal: UserDetails, @PathVariable datasetName: String, @RequestBody newProperties: Optional[Map[String, String]]): CompletableFuture[ResponseEntity[Option[Dataset]]] = { - datasetService.replaceProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map { + datasetService.updatePropertiesV2(principal.getUsername, datasetName, newProperties.toScalaOption).map { case None => throw notFound() case Some(dataset) => val location: URI = new URI(s"/api/dataset/${dataset.name}/${dataset.version}") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index dbe997478..9f6bf5756 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers import java.util.Optional import java.util.concurrent.CompletableFuture - import com.mongodb.client.result.UpdateResult import org.springframework.http.HttpStatus import org.springframework.security.core.annotation.AuthenticationPrincipal @@ -29,6 +28,8 @@ import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService import za.co.absa.enceladus.model.menas.audit._ +import scala.concurrent.Future + abstract class VersionedModelController[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -69,7 +70,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @GetMapping(Array("/detail/{name}/latestVersion")) @ResponseStatus(HttpStatus.OK) - def getLatestVersionNumber(@PathVariable name: String): CompletableFuture[Int] = { + def getLatestVersionNumber(@PathVariable name: String): Future[Int] = { versionedModelService.getLatestVersionNumber(name) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index ac207ca02..9a0b86290 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -16,14 +16,53 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.rest_api.services.DatasetService +import za.co.absa.enceladus.rest_api.utils.implicits._ + +import java.net.URI +import java.util.concurrent.CompletableFuture +import javax.servlet.http.HttpServletRequest +import scala.concurrent.ExecutionContext.Implicits.global @RestController @RequestMapping(path = Array("/api-v3/datasets")) class DatasetControllerV3 @Autowired()(datasetService: DatasetService) extends VersionedModelControllerV3(datasetService) { + @GetMapping(Array("/{name}/{version}/properties")) + @ResponseStatus(HttpStatus.OK) + def getAllPropertiesForVersion(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Map[String, String]] = { + forVersionExpression(name, version)(datasetService.getVersion).map { + case Some(entity) => entity.propertiesAsMap + case None => throw notFound() + } + } + + @PutMapping(Array("/{name}/{version}/properties")) + @ResponseStatus(HttpStatus.OK) + def updateProperties(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody newProperties: java.util.Map[String, String], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + forVersionExpression(name, version) { case (dsName, dsVersion) => + datasetService.updateProperties(principal.getUsername, dsName, dsVersion, newProperties.toScalaMap).map { + + case Some(entity) => + // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties + createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties") + case None => throw notFound() + } + } + } + + // todo putIntoInfoFile switch needed? + // TODO // /{datasetName}/{version}/rules // /{datasetName}/{version}/rules/{index} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 5c930e842..0c19739f8 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -24,12 +24,12 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder import za.co.absa.enceladus.model.menas.audit._ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn} -import za.co.absa.enceladus.rest_api.controllers.BaseController +import za.co.absa.enceladus.rest_api.controllers.{BaseController} import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey -import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService import java.net.URI +import java.util import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest @@ -189,11 +189,11 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } protected def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, - stripLastSegments: Int = 0): ResponseEntity[Nothing] = { + stripLastSegments: Int = 0, suffix: String = ""): ResponseEntity[Nothing] = { val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString val location: URI = ServletUriComponentsBuilder.fromRequest(request) - .path(s"$strippingPrefix/{name}/{version}") + .path(s"$strippingPrefix/{name}/{version}$suffix") .buildAndExpand(name, version.toString) .normalize() // will normalize `/one/two/../three` into `/one/tree` .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 309b53809..ef26b9396 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.properties.essentiality.Mandatory import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel import DatasetService._ -import za.co.absa.enceladus.rest_api.exceptions.NotFoundException +import za.co.absa.enceladus.rest_api.exceptions.{NotFoundException, ValidationException} import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import scala.concurrent.Future @@ -123,8 +123,24 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } - def replaceProperties(username: String, datasetName: String, - updatedProperties: Option[Map[String, String]]): Future[Option[Dataset]] = { + def updateProperties(username: String, datasetName: String, datasetVersion: Int, + updatedProperties: Map[String, String]): Future[Option[Dataset]] = { + for { + s <- validateProperties(updatedProperties).flatMap { + case validation if !validation.isValid => Future.failed(ValidationException(validation)) // warnings are ok for update + case _ => Future.successful(()) // todo perhaps communicate warnings as result? + } + + // updateFuture includes latest-check and version increase + update <- updateFuture(username, datasetName, datasetVersion) { latest => + Future.successful(latest.copy(properties = Some(removeBlankProperties(updatedProperties)))) + } + } yield update + } + + // kept for API v2 usage only + def updatePropertiesV2(username: String, datasetName: String, + updatedProperties: Option[Map[String, String]]): Future[Option[Dataset]] = { for { latestVersion <- getLatestVersionNumber(datasetName) update <- update(username, datasetName, latestVersion) { latest => @@ -205,7 +221,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } - def validateProperties(properties: Map[String, String], forRun: Boolean): Future[Validation] = { + def validateProperties(properties: Map[String, String], forRun: Boolean = false): Future[Validation] = { datasetPropertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => val propDefsMap = Map(propDefs.map { propDef => (propDef.name, propDef) }: _*) // map(key, propDef) @@ -437,10 +453,20 @@ object DatasetService { */ def removeBlankProperties(properties: Option[Map[String, String]]): Option[Map[String, String]] = { properties.map { - _.filter { case (_, propValue) => propValue.nonEmpty } + removeBlankProperties } } + /** + * Removes properties having empty-string value. Effectively mapping such properties' values from Some("") to None. + * This is Backend-implementation related to DatasetService.replaceBlankProperties(dataset) on Frontend + * @param properties original properties + * @return properties without empty-string value entries + */ + def removeBlankProperties(properties: Map[String, String]): Map[String, String] = { + properties.filter { case (_, propValue) => propValue.nonEmpty } + } + private[services] def replacePrefixIfFound(fieldName: String, replacement: String, lookFor: String): Option[String] = { fieldName match { case `lookFor` => Some(replacement) // exact match diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 9f0e619d7..ab2b8aa1e 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -32,10 +32,10 @@ import za.co.absa.enceladus.rest_api.integration.repositories.BaseRepositoryTest import scala.concurrent.Future import scala.reflect.ClassTag -abstract class BaseRestApiTestV2 extends BaseRestApiTest("/api") -abstract class BaseRestApiTestV3 extends BaseRestApiTest("/api-v3") +abstract class BaseRestApiTestV2 extends BaseRestApiTest("/api/login", "/api") +abstract class BaseRestApiTestV3 extends BaseRestApiTest("/api/login", "/api-v3") -abstract class BaseRestApiTest(apiPath: String) extends BaseRepositoryTest { +abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseRepositoryTest { import scala.concurrent.ExecutionContext.Implicits.global @@ -55,6 +55,7 @@ abstract class BaseRestApiTest(apiPath: String) extends BaseRepositoryTest { // expecting apiPath to be /api for v2 and /api-v3 for v3 private lazy val baseUrl = s"http://localhost:$port$apiPath" + private lazy val loginBaseUrl = s"http://localhost:$port$loginPath" private lazy val authHeaders = getAuthHeaders(user, passwd) private lazy val authHeadersAdmin = getAuthHeaders(adminUser, adminPasswd) @@ -75,7 +76,7 @@ abstract class BaseRestApiTest(apiPath: String) extends BaseRepositoryTest { } def getAuthHeaders(username: String, password: String): HttpHeaders = { - val loginUrl = s"$baseUrl/login?username=$username&password=$password&submit=Login" + val loginUrl = s"$loginBaseUrl?username=$username&password=$password&submit=Login" val response = restTemplate.postForEntity(loginUrl, HttpEntity.EMPTY, classOf[String]) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index be79264ff..5b4e84193 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -468,7 +468,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"GET $apiUrl/{name}/{version}/export" should { + s"GET $apiUrl/{name}/{version}/used-in" should { "return 404" when { "when the dataset of latest version does not exist" in { val response = sendGet[String](s"$apiUrl/notFoundDataset/latest/used-in") @@ -512,6 +512,110 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo properties test for datasets or in general for any VersionedModel + s"GET $apiUrl/{name}/{version}/properties" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/properties") + assertNotFound(response) + } + } + + "return 200" when { + "there is a specific Dataset version" should { + Seq( + ("empty1", Some(Map.empty[String, String])), + ("empty2", None), + ("non-empty", Some(Map("key1" -> "val1", "key2" -> "val2"))) + ).foreach { case (propertiesCaseName, propertiesData) => + s"return dataset properties ($propertiesCaseName)" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, properties = propertiesData) + val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3, properties = Some(Map("other" -> "prop"))) + datasetFixture.add(datasetV1, datasetV2, datasetV3) + + val response = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") + assertOk(response) + + val expectedProperties = propertiesData.getOrElse(Map.empty[String, String]) + val body = response.getBody + assert(body == expectedProperties) + } + } + } + } + + "return 200" when { + "there is a latest Dataset version" should { + s"return dataset properties" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[Map[String, String]](s"$apiUrl/datasetA/latest/properties") + assertOk(response) + + val body = response.getBody + assert(body == Map("key1" -> "val1", "key2" -> "val2")) + } + } + } + } + + + s"PUT $apiUrl/{name}/{version}/properties" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendPut[Map[String, String], String](s"$apiUrl/notFoundDataset/456/properties", bodyOpt = Some(Map.empty)) + assertNotFound(response) + } + } + + "return 400" when { + "when version is not the latest (only last version can be updated)" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(datasetV1, datasetV2, datasetV3) + + val response = sendPut[Map[String, String], String](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) + assertBadRequest(response) + } + } + + // todo add update-validation failing case + // todo check validity and refuse if not valid (open: both not having proper propDef, and also not valid for a propdef?) + + + "201 Created with location" when { + Seq( + ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) + ("empty properties map", "{}", Some(Map.empty)) + ).foreach { case (testCaseName, payload, expectedPropertiesSet) => + s"properties are replaced with a new version ($testCaseName)" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") + ) + + val response1 = sendPut[String, String](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) + assertCreated(response1) + val headers1 = response1.getHeaders + assert(headers1.getFirst("Location").endsWith("/api-v3/datasets/datasetA/2/properties")) + + + val response2 = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") + assertOk(response2) + val responseBody = response2.getBody + responseBody shouldBe expectedPropertiesSet.getOrElse(Map.empty) + } + } + } + } + + // todo CR tests } From 3566df5237cf80100b3bff7828026edabd2ac529 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 30 Mar 2022 10:19:12 +0200 Subject: [PATCH 15/67] #1693 API v3: VersionedModelControllerV3 - GET/PUT /{name}/{version}/properties - supports latest for as version-expression; get impl is unvalidated, put impl checks validity - extended for different validation cases - login is now common, under /api/login for both v2 and v3 (did not work previously) --- .../DatasetControllerV3IntegrationSuite.scala | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 5b4e84193..ac59476d5 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -32,6 +32,7 @@ import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, Pro import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.VersionsList import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.ValidationException import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTest, BaseRestApiTestV3, toExpected} @@ -577,14 +578,52 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) datasetFixture.add(datasetV1, datasetV2, datasetV3) - val response = sendPut[Map[String, String], String](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("version" -> + List("Version 2 of datasetA is not the latest version, therefore cannot be edited") + )) + } + + "when properties are not backed by propDefs" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("keyA")) + + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("undefinedProperty1" -> "someValue"))) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } - } - // todo add update-validation failing case - // todo check validity and refuse if not valid (open: both not having proper propDef, and also not valid for a propdef?) + "when properties are not valid (based on propDefs)" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("mandatoryA", essentiality = Essentiality.Mandatory), + PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) + ) + + val response1 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("AorB" -> "a"))) // this is ok, but mandatoryA is missing + + assertBadRequest(response1) + response1.getBody shouldBe Validation(Map("mandatoryA" -> List("Dataset property 'mandatoryA' is mandatory, but does not exist!"))) + + val response2 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("mandatoryA" -> "valueA", "AorB" -> "c"))) // mandatoryA is ok, but AorB has invalid value + + assertBadRequest(response2) + response2.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) + } + } + // todo: maybe pass through validation warnings on update? "201 Created with location" when { Seq( From 219360a20b4f8fd3d754170e05202997bc803b59 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 30 Mar 2022 11:07:27 +0200 Subject: [PATCH 16/67] #1693 Fix: (v2) VersionedModelController.getLatestVersionNumber(): Future -> CompletableFuture (mistake reverted) --- .../rest_api/controllers/VersionedModelController.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index 9f6bf5756..978d51b72 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -70,7 +70,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @GetMapping(Array("/detail/{name}/latestVersion")) @ResponseStatus(HttpStatus.OK) - def getLatestVersionNumber(@PathVariable name: String): Future[Int] = { + def getLatestVersionNumber(@PathVariable name: String): CompletableFuture[Int] = { versionedModelService.getLatestVersionNumber(name) } From 112ac1e20bb48283781120630400994aa8a44a09 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 30 Mar 2022 20:59:49 +0200 Subject: [PATCH 17/67] #1693 VersionedModelControllerV3: `/{name}/{version}/validation` impl added. IT mostly adjusted, but there are todos - DatasetServiceV3 introduced to carry difference in behavior to DatasetService. original entity validation has been divided into create-validation and regular-entity validation. - buildfix for VersionedModelServiceTest --- .../controllers/v3/DatasetControllerV3.scala | 5 +- .../v3/VersionedModelControllerV3.scala | 11 +- .../rest_api/services/DatasetService.scala | 1 + .../services/VersionedModelService.scala | 54 +++++-- .../services/v3/DatasetServiceV3.scala | 49 ++++++ .../DatasetControllerV3IntegrationSuite.scala | 144 +++++++++++++++--- .../services/VersionedModelServiceTest.scala | 34 +++-- 7 files changed, 247 insertions(+), 51 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index 9a0b86290..f51e6634e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -20,17 +20,16 @@ import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ -import za.co.absa.enceladus.rest_api.services.DatasetService +import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3 import za.co.absa.enceladus.rest_api.utils.implicits._ -import java.net.URI import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest import scala.concurrent.ExecutionContext.Implicits.global @RestController @RequestMapping(path = Array("/api-v3/datasets")) -class DatasetControllerV3 @Autowired()(datasetService: DatasetService) +class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) extends VersionedModelControllerV3(datasetService) { @GetMapping(Array("/{name}/{version}/properties")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 0c19739f8..f71e46532 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -23,8 +23,8 @@ import org.springframework.web.bind.annotation._ import org.springframework.web.servlet.support.ServletUriComponentsBuilder import za.co.absa.enceladus.model.menas.audit._ import za.co.absa.enceladus.model.versionedModel._ -import za.co.absa.enceladus.model.{ExportableObject, UsedIn} -import za.co.absa.enceladus.rest_api.controllers.{BaseController} +import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.controllers.BaseController import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey import za.co.absa.enceladus.rest_api.services.VersionedModelService @@ -113,6 +113,13 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } + @GetMapping(Array("/{name}/{version}/validation")) + @ResponseStatus(HttpStatus.OK) + def validation(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Validation] = { + forVersionExpression(name, version)(versionedModelService.validate) + } + @PostMapping(Array("")) @ResponseStatus(HttpStatus.CREATED) def create(@AuthenticationPrincipal principal: UserDetails, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index ef26b9396..11bc8f077 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -285,6 +285,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } + // CR-related methods: private def validateConformanceRules(conformanceRules: List[ConformanceRule], maybeSchema: Future[Option[Schema]]): Future[Validation] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index d99072416..05b02d8f2 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -28,6 +28,7 @@ import za.co.absa.enceladus.model.menas.audit._ import scala.concurrent.Future import com.mongodb.MongoWriteException +import VersionedModelService._ abstract class VersionedModelService[C <: VersionedModel with Product with Auditable[C]] (versionedMongoRepository: VersionedMongoRepository[C]) extends ModelService(versionedMongoRepository) { @@ -195,7 +196,10 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit private[rest_api] def create(item: C, username: String): Future[Option[C]] = { for { - validation <- validate(item) + validation <- for { + generalValidation <- validate(item) + creationValidation <- validateForCreation(item) + } yield generalValidation.merge(creationValidation) _ <- if (validation.isValid) { versionedMongoRepository.create(item, username) .recover { @@ -270,30 +274,58 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.isDisabled(name) } + /** + * Retrieves model@version and calls + * [[za.co.absa.enceladus.rest_api.services.VersionedModelService#validate(java.lang.Object)]] + * + * In order to extend this behavior, override the mentioned method instead. (that's why this is `final`) + * @param name + * @param version + * @return + */ + final def validate(name: String, version: Int): Future[Validation] = { + getVersion(name, version).flatMap({ + case Some(entity) => validate(entity) + case _ => Future.failed(NotFoundException(s"Entity by name=$name, version=$version is not found!")) + }) + } + + /** + * Provides common validation (currently entity name validation). Override to extend for further specific validations. + * @param item + * @return + */ def validate(item: C): Future[Validation] = { validateName(item.name) } - protected[services] def validateName(name: String): Future[Validation] = { - val validation = Validation() + def validateForCreation(item: C): Future[Validation] = { + isUniqueName(item.name).map { isUnique => + if (isUnique) { + Validation.empty + } else { + Validation.empty.withError("name", s"entity with name already exists: '${item.name}'") + } + } + } + protected[services] def validateName(name: String): Future[Validation] = { if (hasWhitespace(name)) { - Future.successful(validation.withError("name", s"name contains whitespace: '$name'")) + Future.successful(Validation.empty.withError("name", s"name contains whitespace: '$name'")) } else { - isUniqueName(name).map { isUnique => - if (isUnique) { - validation - } else { - validation.withError("name", s"entity with name already exists: '$name'") - } - } + Future.successful(Validation.empty) } } +} + +object VersionedModelService { private[services] def hasWhitespace(name: String): Boolean = Option(name).exists(definedName => !definedName.matches("""\w+""")) + private[services] def hasValidNameChars(name: String): Boolean = Option(name).exists(definedName => definedName.matches("""[a-zA-Z0-9._-]+""")) + private[services] def hasValidApiVersion(version: Option[String]): Boolean = version.contains(ModelVersion.toString) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala new file mode 100644 index 000000000..1f797affb --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} +import za.co.absa.enceladus.rest_api.services.{DatasetService, PropertyDefinitionService} + +import scala.concurrent.Future + + +// this DatasetService is a V3 difference wrapper - once V2 is removed, implementations can/should be merged +@Service +class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoRepository, + oozieRepository: OozieRepository, + datasetPropertyDefinitionService: PropertyDefinitionService) + extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) { + + import scala.concurrent.ExecutionContext.Implicits.global + + // general entity validation is extendable for V3 - here with properties validation + override def validate(item: Dataset): Future[Validation] = { + // todo check schema presence same way as for import + + for { + originalValidation <- super.validate(item) + dsSpecificValidation <- validateProperties(item.propertiesAsMap) + } yield originalValidation.merge(dsSpecificValidation) + } + + +} + + diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index ac59476d5..87e8e79f0 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -25,16 +25,13 @@ import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.conformanceRule.{LiteralConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.dataFrameFilter._ -import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality -import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ -import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, PropertyType, StringPropertyType} +import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.VersionsList import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} -import za.co.absa.enceladus.rest_api.exceptions.ValidationException +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ -import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTest, BaseRestApiTestV3, toExpected} import scala.collection.JavaConverters._ @@ -59,19 +56,26 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"POST $apiUrl" can { + // todo 400 if validation of entity does not pass + "return 201" when { "a Dataset is created" should { "return the created Dataset (with empty properties stripped)" in { val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") + ) + val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) - val locationHeader = response.getHeaders.get("location").asScala.headOption - locationHeader shouldBe defined - locationHeader.get should endWith("/api-v3/datasets/dummyDs/1") + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/dummyDs/1") - val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) @@ -235,11 +239,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val response = sendPut[Dataset, String](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) assertCreated(response) - val locationHeader = response.getHeaders.get("location").asScala.headOption - locationHeader shouldBe defined - locationHeader.get should endWith("/api-v3/datasets/datasetA/3") + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetA/3") - val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) @@ -366,7 +369,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA |"properties":{"key2":"val2","key1":"val1"} |}}""".stripMargin.replaceAll("[\\r\\n]", "") - "return 405" when { + "return 400" when { "a Dataset with the given name" should { "fail when name in the URL and payload is mismatched" in { val response = sendPost[String, String](s"$apiUrl/datasetABC/import", @@ -377,6 +380,8 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + // todo 400 when validation of imported entity does not pass + "return 201" when { "there is a existing Dataset" should { "a +1 version of dataset is added" in { @@ -384,13 +389,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset1 = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("init version")) datasetFixture.add(dataset1) + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) + val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) - val locationHeader = response.getHeaders.get("location").asScala.headOption - locationHeader shouldBe defined - locationHeader.get should endWith("/api-v3/datasets/datasetXYZ/2") + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetXYZ/2") - val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) @@ -409,14 +418,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "there is no such Dataset, yet" should { "a the version of dataset created" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) - val locationHeader = response.getHeaders.get("location").asScala.headOption - locationHeader shouldBe defined - locationHeader.get should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version - val relativeLocation = stripBaseUrl(locationHeader.get) // because locationHeader contains domain, port, etc. + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) @@ -590,7 +602,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when properties are not backed by propDefs" in { val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetV1) - propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("keyA")) + // propdefs are empty val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("undefinedProperty1" -> "someValue"))) @@ -623,6 +635,92 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + // todo check behavior on undefined property being added + + "201 Created with location" when { + Seq( + ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) + ("empty properties map", "{}", Some(Map.empty)) + ).foreach { case (testCaseName, payload, expectedPropertiesSet) => + s"properties are replaced with a new version ($testCaseName)" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") + ) + + val response1 = sendPut[String, String](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) + assertCreated(response1) + val headers1 = response1.getHeaders + assert(headers1.getFirst("Location").endsWith("/api-v3/datasets/datasetA/2/properties")) + + + val response2 = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") + assertOk(response2) + val responseBody = response2.getBody + responseBody shouldBe expectedPropertiesSet.getOrElse(Map.empty) + } + } + } + } + + // similar to put-properties validation + s"GET $apiUrl/{name}/{version}/validation" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/validation") + assertNotFound(response) + } + } + + // todo name validation - common for versioned entities + + "return 200" when { + "when properties are not backed by propDefs" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("undefinedProperty1" -> "someValue"))) + datasetFixture.add(datasetV1) + // propdefs are empty + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + + assertOk(response) + response.getBody shouldBe + Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) + } + + "when properties are not valid (based on propDefs) - mandatoriness check" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = None) // prop 'mandatoryA' not present + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("mandatoryA", essentiality = Essentiality.Mandatory) + ) + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + assertOk(response) + response.getBody shouldBe Validation(Map("mandatoryA" -> List("Dataset property 'mandatoryA' is mandatory, but does not exist!"))) + } + + "when properties are not valid (based on propDefs) - property conformance" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("AorB" -> "c"))) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) + ) + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + assertOk(response) + response.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) + } + + // todo check behavior on undefined property being added + + } + // todo: maybe pass through validation warnings on update? "201 Created with location" when { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala index c555e6d0b..34032f344 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala @@ -29,23 +29,13 @@ abstract class VersionedModelServiceTest[C <: VersionedModel with Product with A private val validName = "validName" - test("Validate dataset with valid, unique name") { - Mockito.when(modelRepository.isUniqueName(validName)).thenReturn(Future.successful(true)) - + test("Validate a valid model name") { val result = Await.result(service.validateName(validName), shortTimeout) assert(result.isValid) assert(result == Validation()) } - test("Validate dataset with valid, taken name") { - Mockito.when(modelRepository.isUniqueName(validName)).thenReturn(Future.successful(false)) - - val result = Await.result(service.validateName(validName), shortTimeout) - assert(!result.isValid) - assert(result == Validation(Map("name" -> List(s"entity with name already exists: '$validName'")))) - } - - test("Validate dataset with invalid name") { + test("Validate an invalid model name") { assertHasWhitespace(" InvalidName") assertHasWhitespace("InvalidName\t") assertHasWhitespace("Invalid\nName") @@ -59,4 +49,24 @@ abstract class VersionedModelServiceTest[C <: VersionedModel with Product with A assert(result == Validation(Map("name" -> List(s"name contains whitespace: '$name'")))) } + protected val validModels: Seq[C] = Seq.empty // expecting to override to check models + + test("Validate valid models") { + val results = validModels.map{model => (model, Await.result(service.validate(model), shortTimeout))} + + results.foreach { case (model, validation) => + assert(validation.isValid, s"Expected $model to be valid, but $validation found") + } + } + + protected val invalidModels: Seq[C] = Seq.empty // expecting to override to check models + + test("Validate invalid models") { + val results = invalidModels.map{model => (model, Await.result(service.validate(model), shortTimeout))} + + results.foreach { case (model, validation) => + assert(!validation.isValid, s"Expected $model to be invalid, but $validation found") + } + } + } From 101b37a1cc18944b49fd3d9c61eec90f44832691 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 31 Mar 2022 09:46:35 +0200 Subject: [PATCH 18/67] #1693 VersionedModelControllerV3 - IT extended for `POST /{name}/{version}` in proper validations --- .../DatasetControllerV3IntegrationSuite.scala | 86 ++++++++++--------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 87e8e79f0..cec0f7278 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -55,57 +55,66 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture) - s"POST $apiUrl" can { - // todo 400 if validation of entity does not pass - + s"POST $apiUrl" should { "return 201" when { - "a Dataset is created" should { - "return the created Dataset (with empty properties stripped)" in { - val dataset = DatasetFactory.getDummyDataset("dummyDs", - properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) + "a Dataset is created" in { + val dataset = DatasetFactory.getDummyDataset("dummyDs", + properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) - propertyDefinitionFixture.add( - PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), - PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), - PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") - ) + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") + ) + + val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/dummyDs/1") + + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(dataset, actual).copy(properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))) // keyC stripped + + assert(actual == expected) + } + "create a new version of Dataset" when { + "the dataset is disabled (i.e. all version are disabled)" in { + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) + datasetFixture.add(dataset1, dataset2) - val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) + val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create + val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) assertCreated(response) - val locationHeader = response.getHeaders.getFirst("location") - locationHeader should endWith("/api-v3/datasets/dummyDs/1") + val locationHeaders = response.getHeaders.get("location").asScala + locationHeaders should have size 1 + val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. - val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) assertOk(response2) val actual = response2.getBody - val expected = toExpected(dataset, actual).copy(properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))) // keyC stripped + val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) assert(actual == expected) } } } - "the dataset is disabled (i.e. all version are disabled)" should { - "create a new version of Dataset" in { - val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) - val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) - datasetFixture.add(dataset1, dataset2) - - val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create - val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) - assertCreated(response) - val locationHeaders = response.getHeaders.get("location").asScala - locationHeaders should have size 1 - val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. - val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) - assertOk(response2) + "return 400" when { + "datasets properties are not backed by propDefs (undefined properties)" in { + val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("undefinedProperty1" -> "value1"))) + // propdefs are empty - val actual = response2.getBody - val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) - assert(actual == expected) + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } } // todo what to do if "the last dataset version is disabled" @@ -599,7 +608,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA )) } - "when properties are not backed by propDefs" in { + "when properties are not backed by propDefs (undefined properties)" in { val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetV1) // propdefs are empty @@ -635,8 +644,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo check behavior on undefined property being added - "201 Created with location" when { Seq( ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) @@ -679,7 +686,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA // todo name validation - common for versioned entities "return 200" when { - "when properties are not backed by propDefs" in { + "when properties are not backed by propDefs (undefined properties)" in { val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("undefinedProperty1" -> "someValue"))) datasetFixture.add(datasetV1) // propdefs are empty @@ -716,9 +723,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA assertOk(response) response.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) } - - // todo check behavior on undefined property being added - } // todo: maybe pass through validation warnings on update? From b93495823dd11aa1cae5c1ffa4fa844db630d5be Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 1 Apr 2022 17:12:08 +0200 Subject: [PATCH 19/67] #1693 VersionedModelControllerV3 - conformance rule mgmt GET+POST datasets/dsName/version/rules, GET datasets/dsName/version/rules/# + IT --- .../controllers/v3/DatasetControllerV3.scala | 46 +++++- .../services/v3/DatasetServiceV3.scala | 12 ++ .../DatasetControllerV3IntegrationSuite.scala | 152 +++++++++++++++--- 3 files changed, 181 insertions(+), 29 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index f51e6634e..5890e9845 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -20,6 +20,8 @@ import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.model.Dataset +import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3 import za.co.absa.enceladus.rest_api.utils.implicits._ @@ -62,10 +64,46 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) // todo putIntoInfoFile switch needed? - // TODO - // /{datasetName}/{version}/rules - // /{datasetName}/{version}/rules/{index} - // /{datasetName}/{version}/rules + @GetMapping(Array("/{name}/{version}/rules")) + @ResponseStatus(HttpStatus.OK) + def getConformanceRules(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Seq[ConformanceRule]] = { + forVersionExpression(name, version)(datasetService.getVersion).map { + case Some(entity) => entity.conformance + case None => throw notFound() + } + } + + @PostMapping(Array("/{name}/{version}/rules")) + @ResponseStatus(HttpStatus.CREATED) + def addConformanceRule(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody rule: ConformanceRule, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + forVersionExpression(name, version)(datasetService.getVersion).flatMap { + case Some(entity) => datasetService.addConformanceRule(user.getUsername, name, entity.version, rule).map { + case Some(updatedDs) => + val addedRuleOrder = updatedDs.conformance.last.order + createdWithNameVersionLocation(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules + suffix = s"/rules/$addedRuleOrder") + case _ => throw notFound() + } + case None => throw notFound() + } + } + + @GetMapping(Array("/{name}/{version}/rules/{order}")) + @ResponseStatus(HttpStatus.OK) + def getConformanceRuleByOrder(@PathVariable name: String, + @PathVariable version: String, + @PathVariable order: Int): CompletableFuture[ConformanceRule] = { + for { + optDs <- forVersionExpression(name, version)(datasetService.getVersion) + ds = optDs.getOrElse(throw notFound()) + rule = ds.conformance.find(_.order == order).getOrElse(throw notFound()) + } yield rule + } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index 1f797affb..ca460dcc1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.services.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} import za.co.absa.enceladus.rest_api.services.{DatasetService, PropertyDefinitionService} @@ -43,6 +44,17 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito } yield originalValidation.merge(dsSpecificValidation) } + override def addConformanceRule(username: String, datasetName: String, + datasetVersion: Int, rule: ConformanceRule): Future[Option[Dataset]] = { + update(username, datasetName, datasetVersion) { dataset => + val existingRuleOrders = dataset.conformance.map(_.order).toSet + if (!existingRuleOrders.contains(rule.order)) { + dataset.copy(conformance = dataset.conformance :+ rule) // adding the rule + } else { + throw new IllegalArgumentException(s"Rule with order ${rule.order} cannot be added, another rule with this order already exists.") + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index cec0f7278..3f0ea6440 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -23,7 +23,7 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.conformanceRule.{LiteralConformanceRule, MappingConformanceRule} +import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, LiteralConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType @@ -672,6 +672,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } } + // todo: maybe pass through validation warnings on update? } // similar to put-properties validation @@ -724,39 +725,140 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA response.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) } } + } - // todo: maybe pass through validation warnings on update? + private val exampleMcrRule0 = MappingConformanceRule(0, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable", + mappingTableVersion = 9, //scalastyle:ignore magic.number + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + + private val exampleLitRule1 = LiteralConformanceRule(order = 1, controlCheckpoint = true, outputColumn = "something", value = "1.01") + private val dsWithRules1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( + exampleMcrRule0, exampleLitRule1 + )) + + s"GET $apiUrl/{name}/{version}/rules" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/rules") + assertNotFound(response) + } + } - "201 Created with location" when { - Seq( - ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) - ("empty properties map", "{}", Some(Map.empty)) - ).foreach { case (testCaseName, payload, expectedPropertiesSet) => - s"properties are replaced with a new version ($testCaseName)" in { - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(datasetV1) + "return 200" when { + "when there are no conformance rules" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) - propertyDefinitionFixture.add( - PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), - PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), - PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") - ) + val response = sendGet[Array[ConformanceRule]](s"$apiUrl/datasetA/1/rules") - val response1 = sendPut[String, String](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) - assertCreated(response1) - val headers1 = response1.getHeaders - assert(headers1.getFirst("Location").endsWith("/api-v3/datasets/datasetA/2/properties")) + assertOk(response) + response.getBody shouldBe Seq() + } + "when there are some conformance rules" in { + datasetFixture.add(dsWithRules1) - val response2 = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") - assertOk(response2) - val responseBody = response2.getBody - responseBody shouldBe expectedPropertiesSet.getOrElse(Map.empty) - } + val response = sendGet[Array[ConformanceRule]](s"$apiUrl/datasetA/1/rules") + assertOk(response) + response.getBody shouldBe dsWithRules1.conformance.toArray } } } - // todo CR tests + s"POST $apiUrl/{name}/{version}/rules" should { + "return 404" when { + "when the name+version does not exist" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", + bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + assertNotFound(response) + } + } + + "return 400" when { + "when the there is a conflicting conf rule #" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( + LiteralConformanceRule(order = 0,"column1", true, "ABC")) + ) + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", + bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + assertBadRequest(response) + + response.getBody should include("Rule with order 0 cannot be added, another rule with this order already exists.") + } + } + + "return 201" when { + "when conf rule is added" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( + LiteralConformanceRule(order = 0,"column1", true, "ABC")) + ) + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) + assertCreated(response) + + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetA/2/rules/1") // increased version in the url and added rule #1 + + val response2 = sendGet[Dataset](s"$apiUrl/datasetA/2") + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = datasetV1.copy(version = 2, parent = Some(DatasetFactory.toParent(datasetV1)), + conformance = List(datasetV1.conformance.head, exampleLitRule1)) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + } + + s"GET $apiUrl/{name}/{version}/rules/{index}" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/rules/1") + assertNotFound(response) + } + + "when the rule with # does not exist" in { + datasetFixture.add(dsWithRules1) + + val response = sendGet[String](s"$apiUrl/datasetA/1/rules/345") + assertNotFound(response) + } + } + + "return 200" when { + "when there is a conformance rule with the order#" in { + datasetFixture.add(dsWithRules1) + + val response = sendGet[ConformanceRule](s"$apiUrl/datasetA/1/rules/1") + assertOk(response) + response.getBody shouldBe LiteralConformanceRule(order = 1, controlCheckpoint = true, outputColumn = "something", value = "1.01") + } + } + } } From 51ad0137ea5995c81c7e609e26788f7a933babbc Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 6 Apr 2022 12:59:27 +0200 Subject: [PATCH 20/67] #1693 PR review update - typos etc --- .../enceladus/model/versionedModel/VersionedSummary.scala | 2 +- .../controllers/v3/VersionedModelControllerV3.scala | 2 +- .../rest_api/repositories/VersionedMongoRepository.scala | 6 +++--- .../absa/enceladus/rest_api/services/DatasetService.scala | 6 +++--- .../enceladus/rest_api/services/VersionedModelService.scala | 4 ++-- .../absa/enceladus/rest_api/utils/implicits/package.scala | 2 +- .../v3/DatasetControllerV3IntegrationSuite.scala | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index bc28bfbb0..e9e931d59 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -17,4 +17,4 @@ package za.co.absa.enceladus.model.versionedModel case class VersionedSummary(_id: String, latestVersion: Int) -case class VersionsList(_id: String, versions: Seq[Int]) +case class VersionList(_id: String, versions: Seq[Int]) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index f71e46532..ff7e2f409 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -52,7 +52,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionsList(@PathVariable name: String): CompletableFuture[VersionsList] = { + def getVersionsList(@PathVariable name: String): CompletableFuture[VersionList] = { versionedModelService.getAllVersionsValues(name) map { case Some(entity) => entity case None => throw notFound() diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index f86972690..51274b7ec 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -27,7 +27,7 @@ import org.mongodb.scala.model.Updates._ import org.mongodb.scala.model._ import org.mongodb.scala.result.UpdateResult import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionsList} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionList} import scala.concurrent.Future import scala.reflect.ClassTag @@ -92,13 +92,13 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).headOption().map(_.map(_.latestVersion)) } - def getAllVersionsValues(name: String): Future[Option[VersionsList]] = { + def getAllVersionsValues(name: String): Future[Option[VersionList]] = { val pipeline = Seq( filter(getNameFilter(name)), Aggregates.sort(Sorts.ascending("version")), Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array ) - collection.aggregate[VersionsList](pipeline).headOption().map(_.map(vlist => VersionsList("versions", vlist.versions))) + collection.aggregate[VersionList](pipeline).headOption().map(_.map(vlist => VersionList("versions", vlist.versions))) } def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 11bc8f077..f31daab9e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -94,14 +94,14 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { - val existingEntityF = version match { + val existingEntity = version match { case Some(version) => getVersion(name, version) case None => getLatestVersion(name) } - existingEntityF.flatMap { + existingEntity.flatMap { case Some(_) => Future.successful(UsedIn()) // empty usedIn for existing datasets - case None => Future.failed(NotFoundException(s"Dataset '$name' in version ${version.getOrElse("any")}' nof found")) + case None => Future.failed(NotFoundException(s"Dataset '$name' in version ${version.getOrElse("any")}' not found")) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index 05b02d8f2..4d9f7459f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -21,7 +21,7 @@ import org.springframework.security.core.context.SecurityContextHolder import org.springframework.security.core.userdetails.UserDetails import za.co.absa.enceladus.model.{ModelVersion, Schema, UsedIn, Validation} import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionsList} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionList} import za.co.absa.enceladus.rest_api.exceptions._ import za.co.absa.enceladus.rest_api.repositories.VersionedMongoRepository import za.co.absa.enceladus.model.menas.audit._ @@ -57,7 +57,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getAllVersions(name) } - def getAllVersionsValues(name: String): Future[Option[VersionsList]] = { + def getAllVersionsValues(name: String): Future[Option[VersionList]] = { versionedMongoRepository.getAllVersionsValues(name) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala index c6aae6588..00b517746 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala @@ -60,7 +60,7 @@ package object implicits { classOf[Run], classOf[Schema], classOf[SchemaField], classOf[SplineReference], classOf[RunSummary], classOf[RunDatasetNameGroupedSummary], classOf[RunDatasetVersionGroupedSummary], classOf[RuntimeConfig], classOf[OozieSchedule], classOf[OozieScheduleInstance], classOf[ScheduleTiming], classOf[DataFormat], - classOf[UserInfo], classOf[VersionedSummary], classOf[VersionsList], classOf[MenasAttachment], classOf[MenasReference], + classOf[UserInfo], classOf[VersionedSummary], classOf[VersionList], classOf[MenasAttachment], classOf[MenasReference], classOf[PropertyDefinition], classOf[PropertyType], classOf[Essentiality], classOf[LandingPageInformation], classOf[TodaysRunsStatistics], classOf[DataFrameFilter] diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 3f0ea6440..6f8c7fffc 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.model.versionedModel.VersionsList +import za.co.absa.enceladus.model.versionedModel.VersionList import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -129,9 +129,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA parent = Some(DatasetFactory.toParent(datasetV1))) datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[VersionsList](s"$apiUrl/datasetA") + val response = sendGet[VersionList](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == VersionsList("versions", Seq(1, 2))) + assert(response.getBody == VersionList("versions", Seq(1, 2))) } } From ba5edce32d9e69b428bcec508ba3f22895631cc2 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 6 Apr 2022 17:02:23 +0200 Subject: [PATCH 21/67] #1693 Validation-warnings as payload to caller in V3 - v2 ignores this added information - needs --- .../controllers/DatasetController.scala | 2 +- .../controllers/MappingTableController.scala | 4 +- .../controllers/SchemaController.scala | 2 +- .../VersionedModelController.scala | 6 +-- .../controllers/v3/DatasetControllerV3.scala | 17 ++++--- .../v3/VersionedModelControllerV3.scala | 23 +++++---- .../rest_api/services/DatasetService.scala | 17 ++++--- .../services/MappingTableService.scala | 10 ++-- .../services/PropertyDefinitionService.scala | 8 +-- .../rest_api/services/SchemaService.scala | 12 ++--- .../services/VersionedModelService.scala | 51 ++++++++++++------- .../services/v3/DatasetServiceV3.scala | 2 +- .../DatasetControllerV3IntegrationSuite.scala | 29 ++++++++--- 13 files changed, 108 insertions(+), 75 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index b56396087..dc11c0933 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -64,7 +64,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) latestVersion <- datasetService.getLatestVersionValue(datasetName) res <- latestVersion match { case Some(version) => datasetService.addConformanceRule(user.getUsername, datasetName, version, rule).map { - case Some(ds) => ds + case Some((ds, validation)) => ds // v2 disregarding validation case _ => throw notFound() } case _ => throw notFound() diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala index 57eb11a9b..3709ead7d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala @@ -40,7 +40,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ @RequestBody upd: MenasObject[Array[DefaultValue]]): CompletableFuture[MappingTable] = { mappingTableService.updateDefaults(user.getUsername, upd.id.name, upd.id.version, upd.value.toList).map { - case Some(entity) => entity + case Some(entity) => entity._1 // v2 disregarding validation case None => throw notFound() } } @@ -51,7 +51,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ @RequestBody newDefault: MenasObject[DefaultValue]): CompletableFuture[MappingTable] = { mappingTableService.addDefault(user.getUsername, newDefault.id.name, newDefault.id.version, newDefault.value).map { - case Some(entity) => entity + case Some(entity) => entity._1 // v2 disregarding validation case None => throw notFound() } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala index a29070477..9438a3d01 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala @@ -148,7 +148,7 @@ class SchemaController @Autowired()( // the parsing of sparkStruct can fail, therefore we try to save it first before saving the attachment update <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct) _ <- attachmentService.uploadAttachment(menasAttachment) - } yield update + } yield update.map(_._1) // v2 disregarding the validation } catch { case e: SchemaParsingException => throw e.copy(schemaType = schemaType) //adding schema type } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index 978d51b72..366c72ea5 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -115,7 +115,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @ResponseStatus(HttpStatus.CREATED) def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = { - versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { + versionedModelService.importSingleItemV2(importObject.item, principal.getUsername, importObject.metadata).map { case Some(entity) => entity case None => throw notFound() } @@ -131,7 +131,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au versionedModelService.create(item, principal.getUsername) } }.map { - case Some(entity) => entity + case Some((entity, validation)) => entity // v2 does not support validation-warnings on create case None => throw notFound() } } @@ -141,7 +141,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au def edit(@AuthenticationPrincipal user: UserDetails, @RequestBody item: C): CompletableFuture[C] = { versionedModelService.update(user.getUsername, item).map { - case Some(entity) => entity + case Some((entity, validation)) => entity // v2 disregarding validation on edit case None => throw notFound() } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index 5890e9845..b56cf937a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -20,7 +20,7 @@ import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ -import za.co.absa.enceladus.model.Dataset +import za.co.absa.enceladus.model.Validation import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3 import za.co.absa.enceladus.rest_api.utils.implicits._ @@ -50,13 +50,14 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) @PathVariable name: String, @PathVariable version: String, @RequestBody newProperties: java.util.Map[String, String], - request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { forVersionExpression(name, version) { case (dsName, dsVersion) => datasetService.updateProperties(principal.getUsername, dsName, dsVersion, newProperties.toScalaMap).map { - case Some(entity) => + case Some((entity, validation)) => // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties - createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties") + createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties") + .body(validation) // todo include in tests case None => throw notFound() } } @@ -80,13 +81,13 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) @PathVariable name: String, @PathVariable version: String, @RequestBody rule: ConformanceRule, - request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { forVersionExpression(name, version)(datasetService.getVersion).flatMap { case Some(entity) => datasetService.addConformanceRule(user.getUsername, name, entity.version, rule).map { - case Some(updatedDs) => + case Some((updatedDs, validation)) => val addedRuleOrder = updatedDs.conformance.last.order - createdWithNameVersionLocation(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules - suffix = s"/rules/$addedRuleOrder") + createdWithNameVersionLocationBuilder(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules + suffix = s"/rules/$addedRuleOrder").body(validation) case _ => throw notFound() } case None => throw notFound() diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index ff7e2f409..b3ee9399d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -100,14 +100,14 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @RequestBody importObject: ExportableObject[C], - request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { if (name != importObject.item.name) { Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${importObject.item.name}'")) } else { - versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { - case Some(entity) => + versionedModelService.importSingleItemV3(importObject.item, principal.getUsername, importObject.metadata).map { + case Some((entity, validation)) => // stripping two last segments, instead of /api-v3/dastasets/dsName/import + /dsName/dsVersion we want /api-v3/dastasets + /dsName/dsVersion - createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 2) + createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 2).body(validation) case None => throw notFound() } } @@ -124,7 +124,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @ResponseStatus(HttpStatus.CREATED) def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C, - request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { versionedModelService.isDisabled(item.name).flatMap { isDisabled => if (isDisabled) { versionedModelService.recreate(principal.getUsername, item) @@ -132,7 +132,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product versionedModelService.create(item, principal.getUsername) } }.map { - case Some(entity) => createdWithNameVersionLocation(entity.name, entity.version, request) + case Some((entity, validation)) => createdWithNameVersionLocationBuilder(entity.name, entity.version, request).body(validation) case None => throw notFound() } } @@ -143,7 +143,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PathVariable name: String, @PathVariable version: Int, @RequestBody item: C, - request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = { + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { if (name != item.name) { Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${item.name}'")) @@ -151,7 +151,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) } else { versionedModelService.update(user.getUsername, item).map { - case Some(entity) => createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 2) + case Some((updatedEntity, validation)) => + createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) case None => throw notFound() } } @@ -195,8 +196,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } - protected def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest, - stripLastSegments: Int = 0, suffix: String = ""): ResponseEntity[Nothing] = { + protected def createdWithNameVersionLocationBuilder(name: String, version: Int, request: HttpServletRequest, + stripLastSegments: Int = 0, suffix: String = ""): ResponseEntity.BodyBuilder = { val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString val location: URI = ServletUriComponentsBuilder.fromRequest(request) @@ -207,7 +208,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918 - ResponseEntity.created(location).build() + ResponseEntity.created(location) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index f31daab9e..1b930e111 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -44,7 +44,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository import scala.concurrent.ExecutionContext.Implicits.global - override def update(username: String, dataset: Dataset): Future[Option[Dataset]] = { + override def update(username: String, dataset: Dataset): Future[Option[(Dataset, Validation)]] = { super.updateFuture(username, dataset.name, dataset.version) { latest => updateSchedule(dataset, latest).map({ withSchedule => withSchedule @@ -105,7 +105,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } - override def create(newDataset: Dataset, username: String): Future[Option[Dataset]] = { + override def create(newDataset: Dataset, username: String): Future[Option[(Dataset, Validation)]] = { val dataset = Dataset(name = newDataset.name, description = newDataset.description, hdfsPath = newDataset.hdfsPath, @@ -117,18 +117,19 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository super.create(dataset, username) } - def addConformanceRule(username: String, datasetName: String, datasetVersion: Int, rule: ConformanceRule): Future[Option[Dataset]] = { + def addConformanceRule(username: String, datasetName: String, datasetVersion: Int, + rule: ConformanceRule): Future[Option[(Dataset, Validation)]] = { update(username, datasetName, datasetVersion) { dataset => dataset.copy(conformance = dataset.conformance :+ rule) } } def updateProperties(username: String, datasetName: String, datasetVersion: Int, - updatedProperties: Map[String, String]): Future[Option[Dataset]] = { + updatedProperties: Map[String, String]): Future[Option[(Dataset, Validation)]] = { for { - s <- validateProperties(updatedProperties).flatMap { + successfulValidation <- validateProperties(updatedProperties).flatMap { case validation if !validation.isValid => Future.failed(ValidationException(validation)) // warnings are ok for update - case _ => Future.successful(()) // todo perhaps communicate warnings as result? + case validation => Future.successful(validation) // empty or with warnings } // updateFuture includes latest-check and version increase @@ -146,7 +147,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository update <- update(username, datasetName, latestVersion) { latest => latest.copy(properties = removeBlankProperties(updatedProperties)) } - } yield update + } yield update.map(_._1) // v2 does not expect validation on update } private def validateExistingProperty(key: String, value: String, @@ -244,7 +245,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository def getLatestVersions(missingProperty: Option[String]): Future[Seq[Dataset]] = datasetMongoRepository.getLatestVersions(missingProperty) - override def importItem(item: Dataset, username: String): Future[Option[Dataset]] = { + override def importItem(item: Dataset, username: String): Future[Option[(Dataset, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala index f5c085b1a..9bcecfe86 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala @@ -38,7 +38,7 @@ class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTabl used.map(refs => UsedIn(Some(refs), None)) } - override def create(mt: MappingTable, username: String): Future[Option[MappingTable]] = { + override def create(mt: MappingTable, username: String): Future[Option[(MappingTable, Validation)]] = { val mappingTable = MappingTable(name = mt.name, description = mt.description, schemaName = mt.schemaName, @@ -48,19 +48,19 @@ class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTabl super.create(mappingTable, username) } - def updateDefaults(username: String, mtName: String, mtVersion: Int, defaultValues: List[DefaultValue]): Future[Option[MappingTable]] = { + def updateDefaults(username: String, mtName: String, mtVersion: Int, defaultValues: List[DefaultValue]): Future[Option[(MappingTable, Validation)]] = { super.update(username, mtName, mtVersion) { latest => latest.setDefaultMappingValue(defaultValues) } } - def addDefault(username: String, mtName: String, mtVersion: Int, defaultValue: DefaultValue): Future[Option[MappingTable]] = { + def addDefault(username: String, mtName: String, mtVersion: Int, defaultValue: DefaultValue): Future[Option[(MappingTable, Validation)]] = { super.update(username, mtName, mtVersion) { latest => latest.setDefaultMappingValue(latest.defaultMappingValue :+ defaultValue) } } - override def update(username: String, mt: MappingTable): Future[Option[MappingTable]] = { + override def update(username: String, mt: MappingTable): Future[Option[(MappingTable, Validation)]] = { super.update(username, mt.name, mt.version) { latest => latest .setHDFSPath(mt.hdfsPath) @@ -71,7 +71,7 @@ class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTabl } } - override def importItem(item: MappingTable, username: String): Future[Option[MappingTable]] = { + override def importItem(item: MappingTable, username: String): Future[Option[(MappingTable, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala index ffbac0e04..7b4e5a039 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala @@ -18,7 +18,7 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import za.co.absa.enceladus.rest_api.repositories.PropertyDefinitionMongoRepository -import za.co.absa.enceladus.model.UsedIn +import za.co.absa.enceladus.model.{UsedIn, Validation} import za.co.absa.enceladus.model.properties.PropertyDefinition import scala.concurrent.Future @@ -31,7 +31,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = Future.successful(UsedIn()) - override def update(username: String, propertyDef: PropertyDefinition): Future[Option[PropertyDefinition]] = { + override def update(username: String, propertyDef: PropertyDefinition): Future[Option[(PropertyDefinition, Validation)]] = { super.update(username, propertyDef.name, propertyDef.version) { latest => latest .setPropertyType(propertyDef.propertyType) @@ -45,7 +45,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert propertyDefMongoRepository.distinctCount() } - override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[PropertyDefinition]] = { + override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[(PropertyDefinition, Validation)]] = { val propertyDefBase = PropertyDefinition( name = newPropertyDef.name, description = newPropertyDef.description, @@ -63,7 +63,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert super.create(propertyDefinition, username) } - override private[services] def importItem(item: PropertyDefinition, username: String): Future[Option[PropertyDefinition]] = { + override private[services] def importItem(item: PropertyDefinition, username: String): Future[Option[(PropertyDefinition, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index 4eb64a691..684f8907b 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -17,7 +17,7 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.model.{Schema, UsedIn} +import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} import scala.concurrent.Future @@ -39,13 +39,13 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, } yield UsedIn(Some(usedInD), Some(usedInM)) } - def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[Option[Schema]] = { + def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[Option[(Schema, Validation)]] = { super.update(username, schemaName, schemaVersion)({ oldSchema => oldSchema.copy(fields = sparkMenasConvertor.convertSparkToMenasFields(fields.fields).toList) }) } - override def recreate(username: String, schema: Schema): Future[Option[Schema]] = { + override def recreate(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { for { latestVersion <- getLatestVersionNumber(schema.name) update <- super.update(username, schema.name, latestVersion) { latest => @@ -56,19 +56,19 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, } yield update } - override def update(username: String, schema: Schema): Future[Option[Schema]] = { + override def update(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { super.update(username, schema.name, schema.version) { latest => latest.setDescription(schema.description).asInstanceOf[Schema] } } - override def create(newSchema: Schema, username: String): Future[Option[Schema]] = { + override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { val schema = Schema(name = newSchema.name, description = newSchema.description) super.create(schema, username) } - override def importItem(item: Schema, username: String): Future[Option[Schema]] = { + override def importItem(item: Schema, username: String): Future[Option[(Schema, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index 4d9f7459f..bb13f677f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -30,8 +30,9 @@ import scala.concurrent.Future import com.mongodb.MongoWriteException import VersionedModelService._ +// scalastyle:off number.of.methods abstract class VersionedModelService[C <: VersionedModel with Product with Auditable[C]] - (versionedMongoRepository: VersionedMongoRepository[C]) extends ModelService(versionedMongoRepository) { +(versionedMongoRepository: VersionedMongoRepository[C]) extends ModelService(versionedMongoRepository) { import scala.concurrent.ExecutionContext.Implicits.global @@ -93,7 +94,8 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit }) } - def importSingleItem(item: C, username: String, metadata: Map[String, String]): Future[Option[C]] = { + // v2 has external validate validation applied only to imports (not create/edits) via validateSingleImport + def importSingleItemV2(item: C, username: String, metadata: Map[String, String]): Future[Option[C]] = { for { validation <- validateSingleImport(item, metadata) result <- { @@ -103,7 +105,12 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit throw ValidationException(validation) } } - } yield result + } yield result.map(_._1) // v disregards internal common update-based validation + } + + // v3 has internal validation on importItem (because it is based on update + def importSingleItemV3(item: C, username: String, metadata: Map[String, String]): Future[Option[(C, Validation)]] = { + importItem(item, username) } private[services] def validateSingleImport(item: C, metadata: Map[String, String]): Future[Validation] = { @@ -127,7 +134,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit ) } - private[services] def importItem(item: C, username: String): Future[Option[C]] + private[services] def importItem(item: C, username: String): Future[Option[(C, Validation)]] private[services] def validateSchema(schemaName: String, schemaVersion: Int, @@ -173,11 +180,11 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit val allParents = getParents(name) allParents.flatMap({ parents => - val msgs = if(parents.size < 2) Seq() else { + val msgs = if (parents.size < 2) Seq() else { val pairs = parents.sliding(2) pairs.map(p => p.head.getAuditMessages(p(1))).toSeq } - if(parents.isEmpty) { + if (parents.isEmpty) { this.getLatestVersion(name).map({ case Some(entity) => AuditTrail(msgs.reverse :+ entity.createdMessage) case None => throw NotFoundException() @@ -194,7 +201,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit MenasReference(Some(versionedMongoRepository.collectionBaseName), item.name, item.version) } - private[rest_api] def create(item: C, username: String): Future[Option[C]] = { + private[rest_api] def create(item: C, username: String): Future[Option[(C, Validation)]] = { for { validation <- for { generalValidation <- validate(item) @@ -210,39 +217,45 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit throw ValidationException(validation) } detail <- getLatestVersion(item.name) - } yield detail + } yield detail.map(d => (d, validation)) // valid validation may contain warnings } - def recreate(username: String, item: C): Future[Option[C]] = { + def recreate(username: String, item: C): Future[Option[(C, Validation)]] = { for { latestVersion <- getLatestVersionNumber(item.name) update <- update(username, item.setVersion(latestVersion).asInstanceOf[C]) } yield update } - def update(username: String, item: C): Future[Option[C]] + def update(username: String, item: C): Future[Option[(C, Validation)]] - private[services] def updateFuture(username: String, itemName: String, itemVersion: Int)(transform: C => Future[C]): Future[Option[C]] = { + private[services] def updateFuture(username: String, itemName: String, itemVersion: Int)(transform: C => Future[C]): Future[Option[(C, Validation)]] = { for { versionToUpdate <- getLatestVersion(itemName) - transformed <- if (versionToUpdate.isEmpty) { + (transformed, transformedValidation) <- if (versionToUpdate.isEmpty) { Future.failed(NotFoundException(s"Version $itemVersion of $itemName not found")) } else if (versionToUpdate.get.version != itemVersion) { Future.failed(ValidationException(Validation().withError("version", s"Version $itemVersion of $itemName is not the latest version, therefore cannot be edited"))) - } - else { - transform(versionToUpdate.get) + } else { + for { + updatedEntity <- transform(versionToUpdate.get) + validation <- validate(updatedEntity) + } yield if (validation.isValid) { + (updatedEntity, validation) // successful outcome, validation may still hold warnings + } else { + throw ValidationException(validation) + } } update <- versionedMongoRepository.update(username, transformed) .recover { case e: MongoWriteException => throw ValidationException(Validation().withError("version", s"entity '$itemName' with this version already exists: ${itemVersion + 1}")) } - } yield Some(update) + } yield Some((update, transformedValidation)) } - private[services] def update(username: String, itemName: String, itemVersion: Int)(transform: C => C): Future[Option[C]] = { - this.updateFuture(username, itemName, itemVersion){ item: C => + private[services] def update(username: String, itemName: String, itemVersion: Int)(transform: C => C): Future[Option[(C, Validation)]] = { + this.updateFuture(username, itemName, itemVersion) { item: C => Future { transform(item) } @@ -279,6 +292,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit * [[za.co.absa.enceladus.rest_api.services.VersionedModelService#validate(java.lang.Object)]] * * In order to extend this behavior, override the mentioned method instead. (that's why this is `final`) + * * @param name * @param version * @return @@ -292,6 +306,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit /** * Provides common validation (currently entity name validation). Override to extend for further specific validations. + * * @param item * @return */ diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index ca460dcc1..33bc06246 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -45,7 +45,7 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito } override def addConformanceRule(username: String, datasetName: String, - datasetVersion: Int, rule: ConformanceRule): Future[Option[Dataset]] = { + datasetVersion: Int, rule: ConformanceRule): Future[Option[(Dataset, Validation)]] = { update(username, datasetName, datasetVersion) { dataset => val existingRuleOrders = dataset.conformance.map(_.order).toSet if (!existingRuleOrders.contains(rule.order)) { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 6f8c7fffc..9b7351784 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -207,6 +207,8 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + // todo add/adjust check for each situation where validation with warning should get trough + s"PUT $apiUrl/{name}/{version}" can { "return 200" when { "a Dataset with the given name and version is the latest that exists" should { @@ -218,6 +220,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(datasetA1, datasetA2) + Seq("keyA", "keyB", "keyC").foreach {propName => propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) + )} + val exampleMappingCr = MappingConformanceRule(0, controlCheckpoint = true, mappingTable = "CurrencyMappingTable", @@ -247,7 +253,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ) val response = sendPut[Dataset, String](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) - assertCreated(response) + assertCreated(response) //v3 - prop def exist failing here val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetA/3") @@ -263,6 +269,21 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + "when properties are not backed by propDefs (undefined properties)" in { + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetA1) + // propdefs are empty + + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant + + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("keyA" -> List("There is no property definition for key 'keyA'."))) + } + "return 405" when { "a Dataset with the given name and version" should { "fail when version/name in the URL and payload is mismatched" in { @@ -299,11 +320,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), parent = Some(DatasetFactory.toParent(dataset1)) ) - val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, - properties = Some(Map("key1" -> "val1")), - conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), // untouched - parent = Some(DatasetFactory.toParent(dataset2)) - ) datasetFixture.add(dataset1, dataset2) val response = sendGet[String](s"$apiUrl/datasetA/audit-trail") @@ -362,7 +378,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } } - } s"POST $apiUrl/{name}/import" should { From 662119b8ecb6bb9bd924e8f6e2553c4d839a6276 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 7 Apr 2022 09:55:07 +0200 Subject: [PATCH 22/67] #1693 Validation-warnings as payload to caller in V3 - IT covered --- .../VersionedModelController.scala | 1 - .../DatasetControllerV3IntegrationSuite.scala | 31 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index 366c72ea5..87ffd7e72 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -28,7 +28,6 @@ import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService import za.co.absa.enceladus.model.menas.audit._ -import scala.concurrent.Future abstract class VersionedModelController[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 9b7351784..4168f87e1 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -64,11 +64,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), - PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val response = sendPost[Dataset, Dataset](apiUrl, bodyOpt = Some(dataset)) + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) + response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/dummyDs/1") @@ -223,6 +225,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA Seq("keyA", "keyB", "keyC").foreach {propName => propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) )} + // this will cause missing property 'keyD' to issue a warning if not present + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) + ) val exampleMappingCr = MappingConformanceRule(0, controlCheckpoint = true, @@ -252,8 +258,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA version = 2 // update references the last version ) - val response = sendPut[Dataset, String](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) - assertCreated(response) //v3 - prop def exist failing here + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) + assertCreated(response) + response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetA/3") @@ -415,13 +422,15 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), - PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + PropertyDefinitionFactory.getDummyPropertyDefinition("key2"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key3", essentiality = Essentiality.Recommended) ) - val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/2") + response.getBody shouldBe Validation.empty.withWarning("key3", "Property 'key3' is recommended to be present, but was not found!") val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) @@ -671,11 +680,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), - PropertyDefinitionFactory.getDummyPropertyDefinition("keyC") + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val response1 = sendPut[String, String](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) + val response1 = sendPut[String, Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) assertCreated(response1) + response1.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val headers1 = response1.getHeaders assert(headers1.getFirst("Location").endsWith("/api-v3/datasets/datasetA/2/properties")) @@ -831,8 +842,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ) datasetFixture.add(datasetV1) - val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) assertCreated(response) + // if, in the future, there can be a rule update resulting in a warning, let's reflect that here + response.getBody shouldBe Validation.empty val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetA/2/rules/1") // increased version in the url and added rule #1 From b0fd9555d5fb8eda20a809a0ced16849b87879c0 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 7 Apr 2022 13:37:28 +0200 Subject: [PATCH 23/67] #1693 API v3 impl for MTs, no tests yet --- .../v3/MappingTableControllerV3.scala | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala new file mode 100644 index 000000000..1049a9045 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.model._ +import za.co.absa.enceladus.model.menas._ +import za.co.absa.enceladus.rest_api.services.MappingTableService + +import java.util.concurrent.CompletableFuture +import javax.servlet.http.HttpServletRequest + +@RestController +@RequestMapping(Array("/api/mappingTable")) +class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableService) + extends VersionedModelControllerV3(mappingTableService) { + + import za.co.absa.enceladus.rest_api.utils.implicits._ + + import scala.concurrent.ExecutionContext.Implicits.global + + @GetMapping(path = Array("/{name}/{version}/defaults")) + @ResponseStatus(HttpStatus.OK) + def getDefaults(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Seq[DefaultValue]] = { + + forVersionExpression(name, version)(mappingTableService.getVersion).map { // "latest" version is accepted + case Some(entity) => entity.defaultMappingValue + case None => throw notFound() + } + } + + @PutMapping(path = Array("/{name}/{version}/defaults")) + @ResponseStatus(HttpStatus.CREATED) + def updateDefault(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody newDefaults: Array[DefaultValue], + request: HttpServletRequest + ): CompletableFuture[ResponseEntity[Validation]] = { + for { + existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion) + existingMt = existingMtOpt.getOrElse(throw notFound()) + updatedMtAndValidationOpt <- mappingTableService.updateDefaults(user.getUsername, name, existingMt.version, newDefaults.toList) + (updatedMt, validation) = updatedMtAndValidationOpt.getOrElse(throw notFound()) + response = createdWithNameVersionLocationBuilder(name, updatedMt.version, request, + stripLastSegments = 3, suffix = s"/defaults").body(validation) // stripping: /{name}/{version}/defaults + } yield response + } + + @PostMapping(path = Array("/{name}/{version}/defaults")) + @ResponseStatus(HttpStatus.CREATED) + def addDefault(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody newDefault: DefaultValue, + request: HttpServletRequest + ): CompletableFuture[ResponseEntity[Validation]] = { + for { + existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion) + existingMt = existingMtOpt.getOrElse(throw notFound()) + updatedMtAndValidationOpt <- mappingTableService.addDefault(user.getUsername, name, existingMt.version, newDefault) + (updatedMt, validation) = updatedMtAndValidationOpt.getOrElse(throw notFound()) + response = createdWithNameVersionLocationBuilder(name, updatedMt.version, request, + stripLastSegments = 3, suffix = s"/defaults").body(validation) // stripping: /{name}/{version}/defaults + } yield response + } + +} From d4804459870266c95de971ad137354b0821a42c0 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 7 Apr 2022 15:43:33 +0200 Subject: [PATCH 24/67] #1693 import validation check added to IT. --- .../v3/DatasetControllerV3IntegrationSuite.scala | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 4168f87e1..df14b7df2 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -209,8 +209,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo add/adjust check for each situation where validation with warning should get trough - s"PUT $apiUrl/{name}/{version}" can { "return 200" when { "a Dataset with the given name and version is the latest that exists" should { @@ -411,7 +409,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo 400 when validation of imported entity does not pass + "return 400" when { + "imported Dataset fails validation" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing + + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody shouldBe Validation.empty.withError("key2", "There is no property definition for key 'key2'.") + } + } "return 201" when { "there is a existing Dataset" should { @@ -698,7 +706,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } } - // todo: maybe pass through validation warnings on update? } // similar to put-properties validation From e1c904ebd04ca9937a2d6a95187993e4784bbffd Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 11 Apr 2022 11:07:43 +0200 Subject: [PATCH 25/67] #1693 api-v3 mapping table specific - IntegTest for GET/POST/PUT .../defaults --- .../v3/MappingTableControllerV3.scala | 3 +- .../v3/VersionedModelControllerV3.scala | 1 - ...ingTableControllerV3IntegrationSuite.scala | 190 ++++++++++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala index 1049a9045..070806dcc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -21,14 +21,13 @@ import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model._ -import za.co.absa.enceladus.model.menas._ import za.co.absa.enceladus.rest_api.services.MappingTableService import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest @RestController -@RequestMapping(Array("/api/mappingTable")) +@RequestMapping(Array("/api-v3/mapping-tables")) class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableService) extends VersionedModelControllerV3(mappingTableService) { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index b3ee9399d..39fd0009a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -29,7 +29,6 @@ import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.L import za.co.absa.enceladus.rest_api.services.VersionedModelService import java.net.URI -import java.util import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala new file mode 100644 index 000000000..aa35367e6 --- /dev/null +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -0,0 +1,190 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.integration.controllers.v3 + +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.model.{DefaultValue, Validation} +import za.co.absa.enceladus.model.test.factories.MappingTableFactory +import za.co.absa.enceladus.rest_api.integration.controllers.BaseRestApiTestV3 +import za.co.absa.enceladus.rest_api.integration.fixtures._ + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll with Matchers { + + @Autowired + private val mappingTableFixture: MappingTableFixtureService = null + + @Autowired + private val schemaFixture: SchemaFixtureService = null + + private val apiUrl = "/mapping-tables" + + // fixtures are cleared after each test + override def fixtures: List[FixtureService[_]] = List(mappingTableFixture, schemaFixture) + + // todo create/update to show that schema presence is checked + + // only MT-specific endpoints are covered here + s"GET $apiUrl/{name}/{version}/defaults" should { + "return 404" when { + "when the name/version does not exist" in { + mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) + + assertNotFound(sendGet[String](s"$apiUrl/notFoundMt/456/defaults")) + assertNotFound(sendGet[String](s"$apiUrl/mtA/456/defaults")) + assertNotFound(sendGet[String](s"$apiUrl/notFoundMt/latest/defaults")) + } + } + + "return 200" when { + "when there are no defaults" in { + mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) + + val response = sendGet[Array[DefaultValue]](s"$apiUrl/mtA/1/defaults") + + assertOk(response) + response.getBody shouldBe Seq() + } + + "when there are some defaults rules (version \"latest\")" in { + mappingTableFixture.add( + MappingTableFactory.getDummyMappingTable("mtA"), + MappingTableFactory.getDummyMappingTable("mtA", version = 2).copy(defaultMappingValue = List( + DefaultValue("columnX", "defaultXvalue"), + DefaultValue("columnY", "defaultYvalue") + )) + ) + + + val response = sendGet[Array[DefaultValue]](s"$apiUrl/mtA/latest/defaults") + assertOk(response) + response.getBody shouldBe Array(DefaultValue("columnX", "defaultXvalue"), DefaultValue("columnY", "defaultYvalue")) + } + } + } + + s"PUT $apiUrl/{name}/{version}/defaults" should { + "return 404" when { + "when the name/version does not exist" in { + mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) + + assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(Array()))) + } + } + + "return 400" when { + "when version is not the latest (only last version can be updated)" in { + val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1) + val mtAv2 = MappingTableFactory.getDummyMappingTable("mtA", version = 2) + val mtAv3 = MappingTableFactory.getDummyMappingTable("mtA", version = 3) + + mappingTableFixture.add(mtAv1, mtAv2, mtAv3) + + val response = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(Array())) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("version" -> + List("Version 2 of mtA is not the latest version, therefore cannot be edited") + )) + } + } + + "201 Created with location" when { + Seq( + ("empty defaults", Array.empty[DefaultValue]), + ("non-empty defaults", Array(DefaultValue("colA", "defaultA"))) + ).foreach { case (testCaseName, bothPayloadAndExpectedResult: Array[DefaultValue]) => + s"defaults are replaced with a new version ($testCaseName)" in { + val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) + mappingTableFixture.add(mtAv1) + + val response1 = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(bothPayloadAndExpectedResult)) + assertCreated(response1) + response1.getBody shouldBe Validation.empty + val headers1 = response1.getHeaders + assert(headers1.getFirst("Location").endsWith("/api-v3/mapping-tables/mtA/2/defaults")) + + val response2 = sendGet[Array[DefaultValue]](s"$apiUrl/mtA/2/defaults") + assertOk(response2) + val responseBody = response2.getBody + responseBody shouldBe bothPayloadAndExpectedResult // PUT is udd = 'anOldDefault' gets replaced, no trace of it + } + } + } + } + + s"POST $apiUrl/{name}/{version}/defaults" should { + "return 404" when { + "when the name/version does not exist" in { + mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) + + val aDefaultValue = DefaultValue("colA", "defaultA") + assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(aDefaultValue))) + } + } + + "return 400" when { + "when version is not the latest (only last version can be updated)" in { + val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1) + val mtAv2 = MappingTableFactory.getDummyMappingTable("mtA", version = 2) + val mtAv3 = MappingTableFactory.getDummyMappingTable("mtA", version = 3) + + mappingTableFixture.add(mtAv1, mtAv2, mtAv3) + + val response = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("version" -> + List("Version 2 of mtA is not the latest version, therefore cannot be edited") + )) + } + } + + "201 Created with location" when { + s"defaults are replaced with a new version" in { + val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) + mappingTableFixture.add(mtAv1) + + val response1 = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + assertCreated(response1) + response1.getBody shouldBe Validation.empty + val headers1 = response1.getHeaders + assert(headers1.getFirst("Location").endsWith("/api-v3/mapping-tables/mtA/2/defaults")) + + val response2 = sendGet[Array[DefaultValue]](s"$apiUrl/mtA/2/defaults") + assertOk(response2) + val responseBody = response2.getBody + val expectedDefaults = Array(DefaultValue("anOldDefault", "itsValue"), DefaultValue("colA", "defaultA")) // POST = adding, 'anOldDefault' is kept + responseBody shouldBe expectedDefaults + } + } + } + +} From 489ae12ca5d58157de2e5bf0f36d7059f96b1b47 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 11 Apr 2022 13:23:59 +0200 Subject: [PATCH 26/67] #1693 api-v3 mapping table specific - MT now validates if schema exists + IT --- .../v3/MappingTableControllerV3.scala | 4 +- .../services/v3/MappingTableServiceV3.scala | 49 +++++++++++++++++++ .../DatasetControllerV3IntegrationSuite.scala | 1 - ...ingTableControllerV3IntegrationSuite.scala | 49 +++++++++++++++++-- 4 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala index 070806dcc..bfa63b160 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -21,14 +21,14 @@ import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model._ -import za.co.absa.enceladus.rest_api.services.MappingTableService +import za.co.absa.enceladus.rest_api.services.v3.MappingTableServiceV3 import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest @RestController @RequestMapping(Array("/api-v3/mapping-tables")) -class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableService) +class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableServiceV3) extends VersionedModelControllerV3(mappingTableService) { import za.co.absa.enceladus.rest_api.utils.implicits._ diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala new file mode 100644 index 000000000..f6ecf8282 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import za.co.absa.enceladus.model._ +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository} +import za.co.absa.enceladus.rest_api.services.{MappingTableService, SchemaService, VersionedModelService} + +import scala.concurrent.Future + +@Service +class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTableMongoRepository, + datasetMongoRepository: DatasetMongoRepository, + schemaService: SchemaService) + extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) { + + import scala.concurrent.ExecutionContext.Implicits.global + + private def validateSchemaExists(schemaName: String, schemaVersion: Int): Future[Validation] = { + schemaService.getVersion(schemaName, schemaVersion).map { + case None => Validation.empty.withError("schema", s"Schema $schemaName v$schemaVersion not found!") + case Some(_) => Validation.empty + } + } + + override def validate(item: MappingTable): Future[Validation] = { + for { + originalValidation <- super.validate(item) + mtSchemaValidation <- validateSchemaExists(item.schemaName, item.schemaVersion) + } yield originalValidation.merge(mtSchemaValidation) + + } + +} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index df14b7df2..e7e9567f7 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -698,7 +698,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val headers1 = response1.getHeaders assert(headers1.getFirst("Location").endsWith("/api-v3/datasets/datasetA/2/properties")) - val response2 = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") assertOk(response2) val responseBody = response2.getBody diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index aa35367e6..f8f2a2469 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -22,10 +22,11 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.{DefaultValue, Validation} -import za.co.absa.enceladus.model.test.factories.MappingTableFactory +import za.co.absa.enceladus.model.{DefaultValue, MappingTable, Validation} +import za.co.absa.enceladus.model.test.factories.{MappingTableFactory, SchemaFactory} import za.co.absa.enceladus.rest_api.integration.controllers.BaseRestApiTestV3 import za.co.absa.enceladus.rest_api.integration.fixtures._ +import za.co.absa.enceladus.rest_api.integration.controllers.toExpected @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -43,9 +44,45 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be // fixtures are cleared after each test override def fixtures: List[FixtureService[_]] = List(mappingTableFixture, schemaFixture) - // todo create/update to show that schema presence is checked + s"POST $apiUrl" should { + "return 400" when { + "referenced schema does not exits" in { + val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchemaA", schemaVersion = 1) + + val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("schema" -> List("Schema mtSchemaA v1 not found!"))) + } + } + + "return 201" when { + "a MappingTables is created" in { + val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchema1", schemaVersion = 1) + schemaFixture.add(SchemaFactory.getDummySchema("mtSchema1")) + + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist + + val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/mapping-tables/mtA/1") - // only MT-specific endpoints are covered here + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. + val response2 = sendGet[MappingTable](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(mtA, actual) + + assert(actual == expected) + } + + } + } + + // only MT-specific endpoints are covered further on: s"GET $apiUrl/{name}/{version}/defaults" should { "return 404" when { "when the name/version does not exist" in { @@ -122,6 +159,8 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) mappingTableFixture.add(mtAv1) + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist + val response1 = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(bothPayloadAndExpectedResult)) assertCreated(response1) response1.getBody shouldBe Validation.empty @@ -172,6 +211,8 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) mappingTableFixture.add(mtAv1) + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist + val response1 = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) assertCreated(response1) response1.getBody shouldBe Validation.empty From d6ef40124af3944a497e25b691764a8b60cbd09e Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 11 Apr 2022 15:22:49 +0200 Subject: [PATCH 27/67] #1693 schema validation for DSs + IntegTest update --- .../services/v3/DatasetServiceV3.scala | 15 +++-- .../services/v3/HavingSchemaService.scala | 34 +++++++++++ .../DatasetControllerV3IntegrationSuite.scala | 57 ++++++++++++++----- 3 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index 33bc06246..dcc41b69d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -20,7 +20,7 @@ import org.springframework.stereotype.Service import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} -import za.co.absa.enceladus.rest_api.services.{DatasetService, PropertyDefinitionService} +import za.co.absa.enceladus.rest_api.services.{DatasetService, MappingTableService, PropertyDefinitionService, SchemaService} import scala.concurrent.Future @@ -29,19 +29,22 @@ import scala.concurrent.Future @Service class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, - datasetPropertyDefinitionService: PropertyDefinitionService) - extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) { + datasetPropertyDefinitionService: PropertyDefinitionService, + val schemaService: SchemaService) + extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) + with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global // general entity validation is extendable for V3 - here with properties validation override def validate(item: Dataset): Future[Validation] = { - // todo check schema presence same way as for import for { originalValidation <- super.validate(item) - dsSpecificValidation <- validateProperties(item.propertiesAsMap) - } yield originalValidation.merge(dsSpecificValidation) + propertiesValidation <- validateProperties(item.propertiesAsMap) + schemaValidation <- validateSchemaExists(item.schemaName, item.schemaVersion) + // todo validate CR rule existing MT + } yield originalValidation.merge(propertiesValidation).merge(schemaValidation) } override def addConformanceRule(username: String, datasetName: String, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala new file mode 100644 index 000000000..498721d8b --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import za.co.absa.enceladus.model.Validation +import za.co.absa.enceladus.rest_api.services.SchemaService + +import scala.concurrent.{ExecutionContext, Future} + +trait HavingSchemaService { + protected def schemaService: SchemaService + + def validateSchemaExists(schemaName: String, schemaVersion: Int) + (implicit executionContext: ExecutionContext): Future[Validation] = { + schemaService.getVersion(schemaName, schemaVersion).map { + case None => Validation.empty.withError("schema", s"Schema $schemaName v$schemaVersion not found!") + case Some(_) => Validation.empty + } + } + +} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index df14b7df2..3ced580c3 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -61,6 +61,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), @@ -85,6 +86,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "create a new version of Dataset" when { "the dataset is disabled (i.e. all version are disabled)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) datasetFixture.add(dataset1, dataset2) @@ -108,7 +110,19 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "return 400" when { + "dataset schema does not exits" in { + val dataset = DatasetFactory.getDummyDataset("dummyDs") + // there are schemas defined + + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("schema" -> List("Schema dummySchema v1 not found!"))) + } + "datasets properties are not backed by propDefs (undefined properties)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("undefinedProperty1" -> "value1"))) // propdefs are empty @@ -213,11 +227,12 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "a Dataset with the given name and version is the latest that exists" should { "update the dataset (with empty properties stripped)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version"), properties = Some(Map("keyA" -> "valA"))) val datasetA2 = DatasetFactory.getDummyDataset("datasetA", description = Some("second version"), properties = Some(Map("keyA" -> "valA")), version = 2) - datasetFixture.add(datasetA1, datasetA2) Seq("keyA", "keyB", "keyC").foreach {propName => propertyDefinitionFixture.add( @@ -274,22 +289,26 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - "when properties are not backed by propDefs (undefined properties)" in { - val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(datasetA1) - // propdefs are empty + "return 400" when { + "when properties are not backed by propDefs (undefined properties) and schema does not exist" in { + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetA1) + // propdefs are empty - val datasetA2 = DatasetFactory.getDummyDataset("datasetA", - description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant - val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) - assertBadRequest(response) - val responseBody = response.getBody - responseBody shouldBe Validation(Map("keyA" -> List("There is no property definition for key 'keyA'."))) - } + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe + Validation(Map( + "schema" -> List("Schema dummySchema v1 not found!"), + "keyA" -> List("There is no property definition for key 'keyA'.") + )) + } - "return 405" when { "a Dataset with the given name and version" should { "fail when version/name in the URL and payload is mismatched" in { val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) @@ -682,6 +701,8 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ("empty properties map", "{}", Some(Map.empty)) ).foreach { case (testCaseName, payload, expectedPropertiesSet) => s"properties are replaced with a new version ($testCaseName)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetV1) @@ -720,7 +741,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA // todo name validation - common for versioned entities "return 200" when { - "when properties are not backed by propDefs (undefined properties)" in { + "when properties are not backed by propDefs (undefined properties) and schema is missing" in { val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("undefinedProperty1" -> "someValue"))) datasetFixture.add(datasetV1) // propdefs are empty @@ -729,10 +750,14 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA assertOk(response) response.getBody shouldBe - Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) + Validation(Map( + "undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."), + "schema" -> List("Schema dummySchema v1 not found!") + )) } "when properties are not valid (based on propDefs) - mandatoriness check" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = None) // prop 'mandatoryA' not present datasetFixture.add(datasetV1) @@ -746,6 +771,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "when properties are not valid (based on propDefs) - property conformance" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("AorB" -> "c"))) datasetFixture.add(datasetV1) @@ -844,6 +870,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 201" when { "when conf rule is added" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( LiteralConformanceRule(order = 0,"column1", true, "ABC")) ) From a75ab85e351b45a0ce3ce7cafcad1e8a7285cdef Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 11 Apr 2022 17:21:01 +0200 Subject: [PATCH 28/67] #1693 Mt existence check for DS rules + reflecting in integTest --- .../services/v3/DatasetServiceV3.scala | 24 ++++- .../DatasetControllerV3IntegrationSuite.scala | 102 ++++++++++++------ 2 files changed, 90 insertions(+), 36 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index dcc41b69d..10078eda4 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -17,34 +17,48 @@ package za.co.absa.enceladus.rest_api.services.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.model.conformanceRule.ConformanceRule +import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} import za.co.absa.enceladus.rest_api.services.{DatasetService, MappingTableService, PropertyDefinitionService, SchemaService} import scala.concurrent.Future - // this DatasetService is a V3 difference wrapper - once V2 is removed, implementations can/should be merged @Service class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, datasetPropertyDefinitionService: PropertyDefinitionService, + mappingTableService: MappingTableService, val schemaService: SchemaService) extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global + def validateRules(item: Dataset): Future[Validation] = { + val validationsFutList: Seq[Future[Validation]] = item.conformance.map { + case r: MappingConformanceRule => + mappingTableService.getVersion(r.mappingTable, r.mappingTableVersion).map { + case Some(_) => Validation.empty //MT exists + case None => Validation.empty.withError("mapping-table", s"Mapping table ${r.mappingTable} v${r.mappingTableVersion} not found!") + } + case _ => Future.successful(Validation.empty) // no other validations besides mapping CRs + } + + Future.sequence(validationsFutList).map { listOfVals => + listOfVals.foldLeft(Validation.empty)(_ merge _) + } + } + // general entity validation is extendable for V3 - here with properties validation override def validate(item: Dataset): Future[Validation] = { - for { originalValidation <- super.validate(item) propertiesValidation <- validateProperties(item.propertiesAsMap) schemaValidation <- validateSchemaExists(item.schemaName, item.schemaVersion) - // todo validate CR rule existing MT - } yield originalValidation.merge(propertiesValidation).merge(schemaValidation) + rulesValidation <- validateRules(item) + } yield originalValidation.merge(propertiesValidation).merge(schemaValidation).merge(rulesValidation) } override def addConformanceRule(username: String, datasetName: String, diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 3ced580c3..216bbe9fd 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -27,7 +27,7 @@ import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, LiteralConfo import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType -import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.VersionList import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} @@ -49,19 +49,21 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + @Autowired + private val mappingTableFixture: MappingTableFixtureService = null + private val apiUrl = "/datasets" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture) + override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture, mappingTableFixture) s"POST $apiUrl" should { "return 201" when { "a Dataset is created" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) - - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), @@ -133,12 +135,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } } - // todo what to do if "the last dataset version is disabled" + // todo what to do if "the last dataset version is disabled"? } s"GET $apiUrl/{name}" should { "return 200" when { "a Dataset with the given name exists - so it gives versions" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, @@ -153,6 +156,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 404" when { "a Dataset with the given name does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(dataset) @@ -165,6 +169,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"GET $apiUrl/{name}/latest" should { "return 200" when { "a Dataset with the given name exists - gives latest version entity" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, parent = Some(DatasetFactory.toParent(datasetV1))) @@ -182,6 +187,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 404" when { "a Dataset with the given name does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(dataset) @@ -194,10 +200,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"GET $apiUrl/{name}/{version}" should { "return 200" when { "a Dataset with the given name and version exists - gives specified version of entity" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, description = Some("second")) val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3, description = Some("third")) - datasetFixture.add(datasetV1, datasetV2) + datasetFixture.add(datasetV1, datasetV2, datasetV3) val response = sendGet[Dataset](s"$apiUrl/datasetA/2") assertOk(response) @@ -223,6 +230,27 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + private val exampleMappingCr = MappingConformanceRule(0, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable", + mappingTableVersion = 9, //scalastyle:ignore magic.number + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + s"PUT $apiUrl/{name}/{version}" can { "return 200" when { "a Dataset with the given name and version is the latest that exists" should { @@ -243,26 +271,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val exampleMappingCr = MappingConformanceRule(0, - controlCheckpoint = true, - mappingTable = "CurrencyMappingTable", - mappingTableVersion = 9, //scalastyle:ignore magic.number - attributeMappings = Map("InputValue" -> "STRING_VAL"), - targetAttribute = "CCC", - outputColumn = "ConformedCCC", - isNullSafe = true, - mappingTableFilter = Some( - AndJoinedFilters(Set( - OrJoinedFilters(Set( - EqualsFilter("column1", "soughtAfterValue"), - EqualsFilter("column1", "alternativeSoughtAfterValue") - )), - DiffersFilter("column2", "anotherValue"), - NotFilter(IsNullFilter("col3")) - )) - ), - overrideMappingTableOwnFilter = Some(true) - ) + mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("CurrencyMappingTable", version = 9)) //scalastyle:ignore magic.number val datasetA3 = DatasetFactory.getDummyDataset("datasetA", description = Some("updated"), @@ -293,7 +302,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when properties are not backed by propDefs (undefined properties) and schema does not exist" in { val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetA1) - // propdefs are empty + // propdefs are empty, schemas not defined val datasetA2 = DatasetFactory.getDummyDataset("datasetA", description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant @@ -311,6 +320,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "a Dataset with the given name and version" should { "fail when version/name in the URL and payload is mismatched" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) datasetFixture.add(datasetA1) @@ -339,6 +349,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "there is a correct Dataset" should { "return an audit trail for the dataset" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA") val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), @@ -377,6 +388,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "there is a correct Dataset" should { "return the exported Dataset representation for the latest version" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), properties = Some(Map("key1" -> "val1", "key2" -> "val2")), @@ -517,6 +529,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "there is a correct Dataset version" should { "return the exported Dataset representation" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) val dataset3 = DatasetFactory.getDummyDataset(name = "dataset", version = 3, description = Some("showing non-latest export")) @@ -551,6 +564,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 404" when { "when the dataset of name/version does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetA) @@ -564,6 +578,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "any exiting latest dataset" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetA) val response = sendGet[UsedIn](s"$apiUrl/datasetA/latest/used-in") @@ -575,6 +590,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "for existing name+version for dataset" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2) datasetFixture.add(dataset2) val response = sendGet[UsedIn](s"$apiUrl/dataset/2/used-in") @@ -601,6 +617,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ("non-empty", Some(Map("key1" -> "val1", "key2" -> "val2"))) ).foreach { case (propertiesCaseName, propertiesData) => s"return dataset properties ($propertiesCaseName)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, properties = propertiesData) val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3, properties = Some(Map("other" -> "prop"))) @@ -620,6 +637,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "there is a latest Dataset version" should { s"return dataset properties" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) datasetFixture.add(datasetV1, datasetV2) @@ -645,6 +663,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 400" when { "when version is not the latest (only last version can be updated)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) @@ -660,6 +679,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "when properties are not backed by propDefs (undefined properties)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetV1) // propdefs are empty @@ -673,6 +693,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "when properties are not valid (based on propDefs)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetV1) @@ -702,7 +723,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ).foreach { case (testCaseName, payload, expectedPropertiesSet) => s"properties are replaced with a new version ($testCaseName)" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) datasetFixture.add(datasetV1) @@ -744,7 +764,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when properties are not backed by propDefs (undefined properties) and schema is missing" in { val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("undefinedProperty1" -> "someValue"))) datasetFixture.add(datasetV1) - // propdefs are empty + // propdefs are empty, schemas not defined val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") @@ -822,6 +842,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "when there are no conformance rules" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetV1) @@ -832,6 +853,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "when there are some conformance rules" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) datasetFixture.add(dsWithRules1) val response = sendGet[Array[ConformanceRule]](s"$apiUrl/datasetA/1/rules") @@ -844,6 +866,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"POST $apiUrl/{name}/{version}/rules" should { "return 404" when { "when the name+version does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetV1) @@ -855,9 +878,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 400" when { "when the there is a conflicting conf rule #" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( - LiteralConformanceRule(order = 0,"column1", true, "ABC")) - ) + LiteralConformanceRule(order = 0,"column1", true, "ABC") + )) datasetFixture.add(datasetV1) val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", @@ -868,6 +892,20 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + "return 400" when { + "when rule is not valid (missing MT)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", + bodyOpt = Some(exampleMcrRule0)) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("mapping-table", "Mapping table CurrencyMappingTable v9 not found!") + } + } + "return 201" when { "when conf rule is added" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) @@ -905,6 +943,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "when the rule with # does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) datasetFixture.add(dsWithRules1) val response = sendGet[String](s"$apiUrl/datasetA/1/rules/345") @@ -914,6 +953,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 200" when { "when there is a conformance rule with the order#" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) datasetFixture.add(dsWithRules1) val response = sendGet[ConformanceRule](s"$apiUrl/datasetA/1/rules/1") From 65b9f21fd2d14ca5aa949682d5518cb3723bdac6 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 12 Apr 2022 08:49:03 +0200 Subject: [PATCH 29/67] #1693 Mt existence check for DS rules + reflecting in integTest 2 --- .../rest_api/services/v3/MappingTableServiceV3.scala | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala index f6ecf8282..a97e9e11a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala @@ -26,18 +26,11 @@ import scala.concurrent.Future @Service class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, - schemaService: SchemaService) - extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) { + val schemaService: SchemaService) + extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global - private def validateSchemaExists(schemaName: String, schemaVersion: Int): Future[Validation] = { - schemaService.getVersion(schemaName, schemaVersion).map { - case None => Validation.empty.withError("schema", s"Schema $schemaName v$schemaVersion not found!") - case Some(_) => Validation.empty - } - } - override def validate(item: MappingTable): Future[Validation] = { for { originalValidation <- super.validate(item) @@ -45,5 +38,4 @@ class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTab } yield originalValidation.merge(mtSchemaValidation) } - } From 4426ff7efc2c30ca825d5fd44123fea77a05696e Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 14 Apr 2022 13:20:23 +0200 Subject: [PATCH 30/67] #1693 schema controller v3 - todo IT (original IT v2 works) --- .../controllers/SchemaController.scala | 4 +- .../controllers/v3/SchemaControllerV3.scala | 201 ++++++++++++++++++ .../rest_api/services/SchemaService.scala | 4 +- 3 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala index 9438a3d01..e12b4c10e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala @@ -146,9 +146,9 @@ class SchemaController @Autowired()( try { for { // the parsing of sparkStruct can fail, therefore we try to save it first before saving the attachment - update <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct) + (update, validation) <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct) _ <- attachmentService.uploadAttachment(menasAttachment) - } yield update.map(_._1) // v2 disregarding the validation + } yield Some(update) // v2 disregarding the validation; conforming to V2 Option[Entity] signature } catch { case e: SchemaParsingException => throw e.copy(schemaType = schemaType) //adding schema type } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala new file mode 100644 index 000000000..13bf97d60 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala @@ -0,0 +1,201 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import org.apache.spark.sql.types.StructType +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails +import org.springframework.web.bind.annotation._ +import org.springframework.web.multipart.MultipartFile +import za.co.absa.enceladus.model.{Schema, Validation} +import za.co.absa.enceladus.model.menas._ +import za.co.absa.enceladus.rest_api.controllers.SchemaController +import za.co.absa.enceladus.rest_api.exceptions.ValidationException +import za.co.absa.enceladus.rest_api.models.rest.exceptions.SchemaParsingException +import za.co.absa.enceladus.rest_api.repositories.RefCollection +import za.co.absa.enceladus.rest_api.services.{AttachmentService, SchemaRegistryService, SchemaService} +import za.co.absa.enceladus.rest_api.utils.SchemaType +import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor +import za.co.absa.enceladus.rest_api.utils.parsers.SchemaParser + +import java.util.Optional +import java.util.concurrent.CompletableFuture +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} +import scala.concurrent.Future +import scala.util.{Failure, Success, Try} + + +@RestController +@RequestMapping(Array("/api/schema")) +class SchemaControllerV3 @Autowired()( + schemaService: SchemaService, + attachmentService: AttachmentService, + sparkMenasConvertor: SparkMenasSchemaConvertor, + schemaRegistryService: SchemaRegistryService + ) + extends VersionedModelControllerV3(schemaService) { + + import za.co.absa.enceladus.rest_api.utils.implicits._ + + import scala.concurrent.ExecutionContext.Implicits.global + + @GetMapping(path = Array("/{name}/{version}/json"), produces = Array("application/json")) + @ResponseStatus(HttpStatus.OK) + def getJson(@PathVariable name: String, + @PathVariable version: String, + @RequestParam(defaultValue = "false") pretty: Boolean): CompletableFuture[String] = { + forVersionExpression(name, version) (schemaService.getVersion).map { + case Some(schema) => + // todo why is this a problem at all? why is this specific to json? + if (schema.fields.isEmpty) throw ValidationException( + Validation.empty.withError("schema-fields", s"Schema $name v$version exists, but has no fields!") + // or throw notFound() as v2? + ) + val sparkStruct = StructType(sparkMenasConvertor.convertMenasToSparkFields(schema.fields)) + if (pretty) sparkStruct.prettyJson else sparkStruct.json + case None => + throw notFound() + } + + } + + @GetMapping(path = Array("/{name}/{version}/original")) + @ResponseStatus(HttpStatus.OK) + def exportSchema(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + response: HttpServletResponse): CompletableFuture[Array[Byte]] = { + forVersionExpression(name, version)(attachmentService.getSchemaByNameAndVersion).map { attachment => + response.addHeader("mime-type", attachment.fileMIMEType) + attachment.fileContent + } + } + + @PostMapping(Array("/{name}/{version}/from-file")) + @ResponseStatus(HttpStatus.CREATED) + def handleFileUpload(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: Int, + @RequestParam file: MultipartFile, + @RequestParam format: Optional[String], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + + val fileContent = new String(file.getBytes) + + val schemaType = SchemaType.fromOptSchemaName(format.toScalaOption) + val sparkStruct = SchemaParser.getFactory(sparkMenasConvertor).getParser(schemaType).parse(fileContent) + + // for avro schema type, always force the same mime-type to be persisted + val mime = if (schemaType == SchemaType.Avro) { + SchemaController.avscContentType + } else { + file.getContentType + } + + val menasFile = MenasAttachment(refCollection = RefCollection.SCHEMA.name().toLowerCase, + refName = name, + refVersion = version + 1, // version is the current one, refVersion is the to-be-created one + attachmentType = MenasAttachment.ORIGINAL_SCHEMA_ATTACHMENT, + filename = file.getOriginalFilename, + fileContent = file.getBytes, + fileMIMEType = mime) + + uploadSchemaToMenas(principal.getUsername, menasFile, sparkStruct, schemaType).map { case (updatedSchema, validation) => + createdWithNameVersionLocationBuilder(name, updatedSchema.version, request, + stripLastSegments = 3).body(validation) // stripping: /{name}/{version}/from-file + } + } + + @PostMapping(Array("/{name}/{version}/from-remote-uri")) + @ResponseStatus(HttpStatus.CREATED) + def handleRemoteFile(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: Int, + @RequestParam remoteUrl: String, + @RequestParam format: Optional[String], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + + val schemaType: SchemaType.Value = SchemaType.fromOptSchemaName(format.toScalaOption) + val schemaResponse = schemaRegistryService.loadSchemaByUrl(remoteUrl) + val sparkStruct = SchemaParser.getFactory(sparkMenasConvertor).getParser(schemaType).parse(schemaResponse.fileContent) + + val menasFile = MenasAttachment(refCollection = RefCollection.SCHEMA.name().toLowerCase, + refName = name, + refVersion = version + 1, // version is the current one, refVersion is the to-be-created one + attachmentType = MenasAttachment.ORIGINAL_SCHEMA_ATTACHMENT, + filename = schemaResponse.url.getFile, + fileContent = schemaResponse.fileContent.getBytes, + fileMIMEType = schemaResponse.mimeType) + + uploadSchemaToMenas(principal.getUsername, menasFile, sparkStruct, schemaType).map { case (updatedSchema, validation) => + createdWithNameVersionLocationBuilder(name, updatedSchema.version, request, + stripLastSegments = 3).body(validation) // stripping: /{name}/{version}/from-remote-uri + } + } + + @PostMapping(Array("/{name}/{version}/from-registry")) + @ResponseStatus(HttpStatus.CREATED) + def handleSubject(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: Int, + @RequestParam subject: String, + @RequestParam format: Optional[String], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + + val schemaType: SchemaType.Value = SchemaType.fromOptSchemaName(format.toScalaOption) + + val valueSchemaResponse = Try { + schemaRegistryService.loadSchemaBySubjectName(s"$subject") + } match { + case Success(schemaResponse) => schemaResponse + case Failure(_) => schemaRegistryService.loadSchemaBySubjectName(s"$subject-value") // fallback to -value + } + + val valueSparkStruct = SchemaParser.getFactory(sparkMenasConvertor).getParser(schemaType).parse(valueSchemaResponse.fileContent) + + val menasFile = MenasAttachment(refCollection = RefCollection.SCHEMA.name().toLowerCase, + refName = name, + refVersion = version + 1, // version is the current one, refVersion is the to-be-created one + attachmentType = MenasAttachment.ORIGINAL_SCHEMA_ATTACHMENT, + filename = valueSchemaResponse.url.getFile, // only the value file gets saved as an attachment + fileContent = valueSchemaResponse.fileContent.getBytes, + fileMIMEType = valueSchemaResponse.mimeType) + + uploadSchemaToMenas(principal.getUsername, menasFile, valueSparkStruct, schemaType).map { case (updatedSchema, validation) => + createdWithNameVersionLocationBuilder(name, updatedSchema.version, request, + stripLastSegments = 3).body(validation) // stripping: /{name}/{version}/from-registry + } + } + + private def uploadSchemaToMenas(username: String, menasAttachment: MenasAttachment, sparkStruct: StructType, + schemaType: SchemaType.Value): Future[(Schema, Validation)] = { + try { + for { + // the parsing of sparkStruct can fail, therefore we try to save it first before saving the attachment + (updated, validation) <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct) + _ <- attachmentService.uploadAttachment(menasAttachment) + } yield (updated, validation) + } catch { + case e: SchemaParsingException => throw e.copy(schemaType = schemaType) // adding schema type + } + } + + +} + + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index 684f8907b..9583a54f6 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -39,10 +39,10 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, } yield UsedIn(Some(usedInD), Some(usedInM)) } - def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[Option[(Schema, Validation)]] = { + def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[(Schema, Validation)] = { super.update(username, schemaName, schemaVersion)({ oldSchema => oldSchema.copy(fields = sparkMenasConvertor.convertSparkToMenasFields(fields.fields).toList) - }) + }).map(_.getOrElse(throw new IllegalArgumentException("Failed to derive new schema from file!"))) } override def recreate(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { From fe6246c12f67d32213f035315f55727b8dca7719 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 14 Apr 2022 16:00:33 +0200 Subject: [PATCH 31/67] #1693 schema controller v3 - partial IT - POST, .../json, .../original, .../from-file --- .../controllers/v3/SchemaControllerV3.scala | 23 +- .../SchemaControllerV3IntegrationSuite.scala | 680 ++++++++++++++++++ 2 files changed, 691 insertions(+), 12 deletions(-) create mode 100644 rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala index 13bf97d60..8899afb4e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala @@ -41,7 +41,7 @@ import scala.util.{Failure, Success, Try} @RestController -@RequestMapping(Array("/api/schema")) +@RequestMapping(Array("/api-v3/schemas")) class SchemaControllerV3 @Autowired()( schemaService: SchemaService, attachmentService: AttachmentService, @@ -76,10 +76,10 @@ class SchemaControllerV3 @Autowired()( @GetMapping(path = Array("/{name}/{version}/original")) @ResponseStatus(HttpStatus.OK) - def exportSchema(@AuthenticationPrincipal principal: UserDetails, - @PathVariable name: String, - @PathVariable version: String, - response: HttpServletResponse): CompletableFuture[Array[Byte]] = { + def exportOriginalSchemaFile(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + response: HttpServletResponse): CompletableFuture[Array[Byte]] = { forVersionExpression(name, version)(attachmentService.getSchemaByNameAndVersion).map { attachment => response.addHeader("mime-type", attachment.fileMIMEType) attachment.fileContent @@ -92,12 +92,12 @@ class SchemaControllerV3 @Autowired()( @PathVariable name: String, @PathVariable version: Int, @RequestParam file: MultipartFile, - @RequestParam format: Optional[String], + @RequestParam format: String, request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { val fileContent = new String(file.getBytes) - val schemaType = SchemaType.fromOptSchemaName(format.toScalaOption) + val schemaType = SchemaType.fromSchemaName(format) val sparkStruct = SchemaParser.getFactory(sparkMenasConvertor).getParser(schemaType).parse(fileContent) // for avro schema type, always force the same mime-type to be persisted @@ -127,10 +127,10 @@ class SchemaControllerV3 @Autowired()( @PathVariable name: String, @PathVariable version: Int, @RequestParam remoteUrl: String, - @RequestParam format: Optional[String], + @RequestParam format: String, request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { - val schemaType: SchemaType.Value = SchemaType.fromOptSchemaName(format.toScalaOption) + val schemaType = SchemaType.fromSchemaName(format) val schemaResponse = schemaRegistryService.loadSchemaByUrl(remoteUrl) val sparkStruct = SchemaParser.getFactory(sparkMenasConvertor).getParser(schemaType).parse(schemaResponse.fileContent) @@ -154,11 +154,10 @@ class SchemaControllerV3 @Autowired()( @PathVariable name: String, @PathVariable version: Int, @RequestParam subject: String, - @RequestParam format: Optional[String], + @RequestParam format: String, request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { - val schemaType: SchemaType.Value = SchemaType.fromOptSchemaName(format.toScalaOption) - + val schemaType = SchemaType.fromSchemaName(format) val valueSchemaResponse = Try { schemaRegistryService.loadSchemaBySubjectName(s"$subject") } match { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala new file mode 100644 index 000000000..8add7bff9 --- /dev/null +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -0,0 +1,680 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.integration.controllers.v3 + +import com.github.tomakehurst.wiremock.WireMockServer +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder +import com.github.tomakehurst.wiremock.core.WireMockConfiguration +import org.apache.commons.io.IOUtils +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.{HttpStatus, MediaType} +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} +import za.co.absa.enceladus.model.{MappingTable, Schema, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} +import za.co.absa.enceladus.rest_api.integration.fixtures._ +import za.co.absa.enceladus.rest_api.models.SchemaApiFeatures +import za.co.absa.enceladus.rest_api.models.rest.RestResponse +import za.co.absa.enceladus.rest_api.models.rest.errors.{SchemaFormatError, SchemaParsingError} +import za.co.absa.enceladus.rest_api.repositories.RefCollection +import za.co.absa.enceladus.rest_api.utils.SchemaType +import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor +import za.co.absa.enceladus.restapi.TestResourcePath + +import java.io.File +import java.nio.file.{Files, Path} +import scala.collection.immutable.HashMap + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll with Matchers { + + private val port = 8877 // same port as in test/resources/application.conf in the `menas.schemaRegistry.baseUrl` key + private val wireMockServer = new WireMockServer(WireMockConfiguration.wireMockConfig().port(port)) + + override def beforeAll(): Unit = { + super.beforeAll() + wireMockServer.start() + } + + override def afterAll(): Unit = { + super.afterAll() + wireMockServer.stop() + } + + @Autowired + private val schemaFixture: SchemaFixtureService = null + + @Autowired + private val datasetFixture: DatasetFixtureService = null + + @Autowired + private val mappingTableFixture: MappingTableFixtureService = null + + @Autowired + private val attachmentFixture: AttachmentFixtureService = null + + @Autowired + private val convertor: SparkMenasSchemaConvertor = null + + private val apiUrl = "/schemas" + private val schemaRefCollection = RefCollection.SCHEMA.name().toLowerCase() + + override def fixtures: List[FixtureService[_]] = List(schemaFixture, attachmentFixture, datasetFixture, mappingTableFixture) + + s"POST $apiUrl" can { + "return 201" when { + "a Schema is created" in { + val schema = SchemaFactory.getDummySchema("schemaA") + + val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/1") + + val response2 = sendGet[Schema]("/schemas/schemaA/1") + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(schema, actual) + assert(actual == expected) + } + } + + "return 400" when { + "an enabled Schema with that name already exists" in { + val schema = SchemaFactory.getDummySchema() + schemaFixture.add(schema) + + val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + + assertBadRequest(response) + + val actual = response.getBody + val expected = Validation().withError("name", "entity with name already exists: 'dummyName'") + assert(actual == expected) + } + } + } + + // todo disable dataset - all versions/one version/ check the usage to prevent from disabling + // todo used-in implementation checks + + s"GET $apiUrl/{name}/{version}/json" should { + "return 404" when { + "no schema exists for the specified name" in { + val schema = SchemaFactory.getDummySchema(name = "schema1", version = 1) + schemaFixture.add(schema) + + val response = sendGet[String](s"$apiUrl/otherSchemaName/1/json") + + assertNotFound(response) + } + "no schema exists for the specified version" in { + val schema = SchemaFactory.getDummySchema(name = "schema1", version = 1) + schemaFixture.add(schema) + + val response = sendGet[String](s"$apiUrl/schema1/2/json") + + assertNotFound(response) + } + "the schema has no fields" in { // todo 404 or 400 failed valiadation??? + val schema = SchemaFactory.getDummySchema(name = "schemaA", version = 1) + schemaFixture.add(schema) + + val response = sendGet[Validation](s"$apiUrl/schemaA/1/json") + assertBadRequest(response) + response.getBody shouldBe + Validation.empty.withError("schema-fields", "Schema schemaA v1 exists, but has no fields!") + + } + "a non-boolean value is provided for the `pretty` query parameter" in { + val schema = SchemaFactory.getDummySchema(name = "schema1", version = 1, fields = List(SchemaFactory.getDummySchemaField())) + schemaFixture.add(schema) + + val response = sendGet[String](s"$apiUrl/schema1/1/json?pretty=tru") + + assertNotFound(response) + } + } + + "return 200" when { + "there is a Schema with the specified name and version" should { + "return the Spark Struct representation of a schema as a JSON (pretty=false by default)" in { + val schema = SchemaFactory.getDummySchema(name = "schema1", version = 1, fields = List(SchemaFactory.getDummySchemaField())) + schemaFixture.add(schema) + + val response = sendGet[String](s"$apiUrl/schema1/1/json") + + assertOk(response) + + val body = response.getBody + val expected = """{"type":"struct","fields":[{"name":"dummyFieldName","type":"string","nullable":true,"metadata":{}}]}""" + assert(body == expected) + } + "return the Spark Struct representation of a schema as a JSON (pretty=false explict)" in { + val schema = SchemaFactory.getDummySchema(name = "schema1", version = 1, fields = List(SchemaFactory.getDummySchemaField())) + schemaFixture.add(schema) + + val response = sendGet[String](s"$apiUrl/schema1/1/json?pretty=false") + + assertOk(response) + + val body = response.getBody + val expected = """{"type":"struct","fields":[{"name":"dummyFieldName","type":"string","nullable":true,"metadata":{}}]}""" + assert(body == expected) + } + "return the Spark Struct representation of a schema as a pretty JSON" in { + val schema = SchemaFactory.getDummySchema(name = "schema1", version = 1, fields = List(SchemaFactory.getDummySchemaField())) + schemaFixture.add(schema) + + val response = sendGet[String](s"$apiUrl/schema1/1/json?pretty=true") + + assertOk(response) + + val body = response.getBody.replace("\r\n", "\n") // this will make it work on Windows (CRLF->LF), too. + val expected = + """|{ + | "type" : "struct", + | "fields" : [ { + | "name" : "dummyFieldName", + | "type" : "string", + | "nullable" : true, + | "metadata" : { } + | } ] + |}""".stripMargin + assert(body == expected) + } + } + } + } + + s"GET $apiUrl/{name}/{version}/original" should { + "return 404" when { + "no Attachment exists for the specified name" in { + val attachment = AttachmentFactory.getDummyAttachment(refName = "schemaName", refVersion = 2, refCollection = schemaRefCollection) + attachmentFixture.add(attachment) + + val response = sendGet[Array[Byte]](s"$apiUrl/otherSchemaName/2/original") + + assertNotFound(response) + assert(!response.getHeaders.containsKey("mime-type")) + } + "no Attachment exists with a version up to the specified version" in { + val attachment = AttachmentFactory.getDummyAttachment(refName = "schemaName", refVersion = 2, refCollection = schemaRefCollection) + attachmentFixture.add(attachment) + + val response = sendGet[Array[Byte]](s"$apiUrl/schemaName/1/original") + + assertNotFound(response) + assert(!response.getHeaders.containsKey("mime-type")) + } + } + + "return 200" when { + val attachment1 = AttachmentFactory.getDummyAttachment( + refName = "schemaName", + refVersion = 1, + refCollection = schemaRefCollection, + fileContent = Array(1, 2, 3)) + val attachment2 = AttachmentFactory.getDummyAttachment( + refName = "schemaName", + refVersion = 2, + refCollection = schemaRefCollection, + fileContent = Array(2, 3, 4), + fileMIMEType = MediaType.APPLICATION_OCTET_STREAM_VALUE) + val attachment4 = AttachmentFactory.getDummyAttachment( + refName = "schemaName", + refVersion = 4, + refCollection = schemaRefCollection, + fileContent = Array(4, 5, 6), + fileMIMEType = MediaType.APPLICATION_JSON_VALUE) + val attachment5 = AttachmentFactory.getDummyAttachment( + refName = "schemaName", + refVersion = 5, + refCollection = schemaRefCollection, + fileContent = Array(5, 6, 7)) + "there are Attachments with previous and subsequent versions" should { + "return the byte array of the uploaded file with the nearest previous version" in { + attachmentFixture.add(attachment1, attachment2, attachment4, attachment5) + + val response = sendGet[Array[Byte]](s"$apiUrl/schemaName/3/original") + + assertOk(response) + assert(response.getHeaders.containsKey("mime-type")) + assert(response.getHeaders.get("mime-type").get(0) == MediaType.APPLICATION_OCTET_STREAM_VALUE) + + val body = response.getBody + assert(body.sameElements(attachment2.fileContent)) + } + } + "there is an Attachment with the exact version" should { + "return the byte array of the uploaded file with the exact version" in { + attachmentFixture.add(attachment1, attachment2, attachment4, attachment5) + + val response = sendGet[Array[Byte]](s"$apiUrl/schemaName/4/original") + + assertOk(response) + assert(response.getHeaders.containsKey("mime-type")) + assert(response.getHeaders.get("mime-type").get(0) == "application/json") + + val body = response.getBody + assert(body.sameElements(attachment4.fileContent)) + } + } + } + } + + s"POST $apiUrl/{name}/{version}/from-file" should { + "return 201" when { + "a copybook has no errors" should { + "return a new version of the schema" in { + val schema = SchemaFactory.getDummySchema("schemaA", version = 1) + schemaFixture.add(schema) + + val schemaParams = HashMap[String, String]("format" -> "copybook") + val responseUploaded = sendPostUploadFile[Validation]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) + assertCreated(responseUploaded) + val locationHeader = responseUploaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 3) + } + } + + "a JSON struct type schema has no errors" should { + "return a new version of the schema" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + val schemaParams = HashMap[String, Any]("format" -> "struct") + val responseUploaded = sendPostUploadFile[Validation]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) + assertCreated(responseUploaded) + val locationHeader = responseUploaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 2) + } + } + + "an avro schema has no errors" should { + "return a new version of the schema" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + val schemaParams = HashMap[String, Any]( + "name" -> schema.name, "version" -> schema.version, "format" -> "avro") + val responseUploaded = sendPostUploadFile[Schema]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.ok, schemaParams) + assertCreated(responseUploaded) + val locationHeader = responseUploaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 7) + } + } + } + + "return 400" when { + "no upload format type is specified" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + val schemaParams = HashMap.empty[String, Any] // v2 fallbacked on this, v3 forbids it + val response = sendPostUploadFile[String]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) + assertBadRequest(response) + response.getBody should include("Required String parameter 'format' is not present") + } + + "an empty upload format type is specified" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + val schemaParams = HashMap[String, Any]("format" -> "") // v2 fallbacked on this, v3 forbids it + val response = sendPostUploadFile[String]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) + assertBadRequest(response) + response.getBody should include("not a recognized schema format. Menas currently supports: struct, copybook, avro.") + } + + "a copybook with a syntax error" should { + "return a response containing a schema parsing error with syntax error specific fields" in { + val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "copybook") + val response = sendPostUploadFile[RestResponse]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.bogus, schemaParams) + val body = response.getBody + + assertBadRequest(response) + body.error match { + case Some(e: SchemaParsingError) => + assert(e.errorType == "schema_parsing") + assert(e.schemaType == SchemaType.Copybook) + assert(e.line.contains(22)) + assert(e.field.contains("")) + assert(body.message.contains("Syntax error in the copybook")) + case e => fail(s"Expected an instance of SchemaParsingError, got $e.") + } + } + } + + "a JSON struct type schema with a syntax error" should { + "return a response containing a schema parsing error returned by the StructType parser" in { + val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "struct") + val response = sendPostUploadFile[RestResponse]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) + val body = response.getBody + + assertBadRequest(response) + body.error match { + case Some(e: SchemaParsingError) => + assert(e.errorType == "schema_parsing") + assert(e.schemaType == SchemaType.Struct) + assert(body.message.contains("StructType serializer: Failed to convert the JSON string")) + case e => fail(s"Expected an instance of SchemaParsingError, got $e.") + } + } + } + + "an avro-schema with a syntax error" should { + "return a response containing a schema parsing error encountered during avro schema parsing" in { + val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "avro") + val response = sendPostUploadFile[RestResponse]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.bogus, schemaParams) + val body = response.getBody + + assertBadRequest(response) + body.error match { + case Some(e: SchemaParsingError) => + assert(e.errorType == "schema_parsing") + assert(e.schemaType == SchemaType.Avro) + assert(body.message.contains("Record has no fields")) + case e => fail(s"Expected an instance of SchemaParsingError, got $e.") + } + } + } + + "a wrong format has been specified" should { + "return a response containing a schema format error" in { + val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "foo") + val response = sendPostUploadFile[RestResponse]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) + val body = response.getBody + + assertBadRequest(response) + body.error match { + case Some(e: SchemaFormatError) => + assert(e.errorType == "schema_format") + assert(e.schemaType == "foo") + assert(body.message.contains("'foo' is not a recognized schema format.")) + case e => fail(s"Expected an instance of SchemaFormatError, got $e.") + } + } + } + } + + "return 404" when { + "a schema file is uploaded, but no schema exists for the specified name and version" in { + val schemaParams = HashMap[String, Any]( + "name" -> "dummy", "version" -> 1, "format" -> "copybook") + val responseUploaded = sendPostUploadFile[Schema]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) + assertNotFound(responseUploaded) + } + } + } + + import com.github.tomakehurst.wiremock.client.WireMock._ + + private def readTestResourceAsString(path: String): String = IOUtils.toString(getClass.getResourceAsStream(path)) + + /** + * will prepare the a response from file with correct `ContentType` + */ + private def readTestResourceAsResponseWithContentType(path: String): ResponseDefinitionBuilder = { + // this is crazy, but it works better than hardcoding mime-types + val filePath: Path = new File(getClass.getResource(path).toURI()).toPath + val mime = Option(Files.probeContentType(filePath)).getOrElse(MediaType.APPLICATION_OCTET_STREAM_VALUE) // default for e.g. cob + + val content = readTestResourceAsString(path) + import com.github.tomakehurst.wiremock.client.WireMock._ + okForContentType(mime, content) + } + + s"POST $apiUrl/{name}/{version}/from-remote-uri" should { + + val remoteFilePath = "/remote-test/someRemoteFile.ext" + val remoteUrl = s"http://localhost:$port$remoteFilePath" + + "return 201" when { + "a copybook has no errors" should { + "return a new version of the schema" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) + + val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) + val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + assertCreated(responseRemoteLoaded) + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 3) + } + } + + s"struct schema has no errors" should { + "return a new version of the schema" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) + + val params = HashMap("remoteUrl" -> remoteUrl, "format" -> "struct") + val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + assertCreated(responseRemoteLoaded) + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 2) + } + } + + "an avro schema has no errors" should { + "return a new version of the schema" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) + + val params = HashMap[String, Any]("format" -> "avro", "remoteUrl" -> remoteUrl) + val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + assertCreated(responseRemoteLoaded) + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 7) + } + } + } + + "return 400" when { + Seq( + (SchemaType.Copybook, TestResourcePath.Copybook.bogus, "Syntax error in the copybook"), + (SchemaType.Struct, TestResourcePath.Json.bogus, "StructType serializer: Failed to convert the JSON string"), + (SchemaType.Avro, TestResourcePath.Avro.bogus, "Record has no fields") + ).foreach { case (schemaType, testResourcePath, expectedErrorMessage) => + + s"a $schemaType with a syntax error" should { + "return a response containing a schema parsing error with syntax error specific fields" in { + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(testResourcePath))) + + val params = HashMap("name" -> "MySchema", "version" -> 1, "format" -> schemaType.toString, "remoteUrl" -> remoteUrl) + val response = sendPostRemoteFile[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) + val body = response.getBody + + assertBadRequest(response) + body.error match { + case Some(e: SchemaParsingError) => + assert(e.errorType == "schema_parsing") + assert(e.schemaType == schemaType) + assert(body.message.contains(expectedErrorMessage)) + case e => fail(s"Expected an instance of SchemaParsingError, got $e.") + } + } + } + } + + "a wrong format has been specified" should { + "return a response containing a schema format error" in { + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) + + val params = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "foo", "remoteUrl" -> remoteUrl) + val response = sendPostRemoteFile[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) + val body = response.getBody + + assertBadRequest(response) + body.error match { + case Some(e: SchemaFormatError) => + assert(e.errorType == "schema_format") + assert(e.schemaType == "foo") + assert(body.message.contains("'foo' is not a recognized schema format.")) + case e => fail(s"Expected an instance of SchemaFormatError, got $e.") + } + } + } + } + + "return 404" when { + "a schema file is loaded from remote url, but no schema exists for the specified name and version" in { + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) + + val params = HashMap[String, Any]("version" -> 1, "name" -> "dummy", "format" -> "copybook", "remoteUrl" -> remoteUrl) + val response = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + assertNotFound(response) + } + } + } + + s"POST $apiUrl/{name}/{version}/from-registry" should { + def subjectPath(subjectName: String) = s"/subjects/$subjectName/versions/latest/schema" + + "return 201" when { + "an avro schema has no errors" should { + "load schema by subject name as-is" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(subjectPath("myTopic1-value"))) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) + + val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic1-value") + val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) + assertCreated(responseRemoteLoaded) + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 7) + } + + "load schema by subject name -value fallback" in { + val schema = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(subjectPath("myTopic2"))) // will fail + .willReturn(notFound())) + + wireMockServer.stubFor(get(urlPathEqualTo(subjectPath("myTopic2-value"))) // fallback will kick in + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) + + val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic2") + val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) + assertCreated(responseRemoteLoaded) + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + assert(actual.name == schema.name) + assert(actual.version == schema.version + 1) + assert(actual.fields.length == 7) + } + } + } + } + +} From 373bf1d43338866835dc1e5be7cb2eb7d574df49 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 20 Apr 2022 09:56:06 +0200 Subject: [PATCH 32/67] #1693 PropertyDefinitionControllerV3 - impl + IT for create; VersionedModelControllerV3 now requires admin rights on specific endpoints (create, edit, import, delete) - TODO reflect in other datatypes --- .../v3/PropertyDefinitionControllerV3.scala | 27 ++++ .../v3/VersionedModelControllerV3.scala | 5 + .../controllers/BaseRestApiTest.scala | 7 +- ...finitionControllerV3IntegrationSuite.scala | 120 ++++++++++++++++++ 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala create mode 100644 rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala new file mode 100644 index 000000000..e134fe716 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService + +@RestController +@RequestMapping(path = Array("/api-v3/property-definitions/datasets"), produces = Array("application/json")) +class PropertyDefinitionControllerV3 @Autowired()(propertyDefService: PropertyDefinitionService) + extends VersionedModelControllerV3(propertyDefService) + +// super-class implementation is sufficient diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 39fd0009a..576745b7d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import com.mongodb.client.result.UpdateResult import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ @@ -96,6 +97,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PostMapping(Array("/{name}/import")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @RequestBody importObject: ExportableObject[C], @@ -121,6 +123,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PostMapping(Array("")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C, request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { @@ -138,6 +141,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PutMapping(Array("/{name}/{version}")) @ResponseStatus(HttpStatus.NO_CONTENT) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def edit(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: Int, @@ -159,6 +163,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @DeleteMapping(Array("/{name}", "/{name}/{version}")) @ResponseStatus(HttpStatus.OK) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def disable(@PathVariable name: String, @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { val v = if (version.isPresent) { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index ab2b8aa1e..049b26126 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -142,10 +142,15 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR } def sendPut[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), - bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { + bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { send(HttpMethod.PUT, urlPath, headers, bodyOpt) } + def sendPutByAdmin[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), + bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { + sendByAdmin(HttpMethod.PUT, urlPath, headers, bodyOpt) + } + def sendPutAsync[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): Future[ResponseEntity[T]] = { sendAsync(HttpMethod.PUT, urlPath, headers, bodyOpt) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala new file mode 100644 index 000000000..62c55cdd2 --- /dev/null +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -0,0 +1,120 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.integration.controllers.v3 + +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.model.Validation +import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.properties.propertyType.StringPropertyType +import za.co.absa.enceladus.model.test.factories.PropertyDefinitionFactory +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} +import za.co.absa.enceladus.rest_api.integration.fixtures._ + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll with Matchers { + + @Autowired + private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + + private val apiUrl = "/property-definitions/datasets" + + // fixtures are cleared after each test + override def fixtures: List[FixtureService[_]] = List(propertyDefinitionFixture) + + + private def minimalPdCreatePayload(name: String, suggestedValue: Option[String]) = { + val suggestedValuePart = suggestedValue match { + case Some(actualSuggestedValue) => s""","suggestedValue": "$actualSuggestedValue"""" + case _ => "" + } + + s"""{"name": "$name","propertyType": {"_t": "StringPropertyType"$suggestedValuePart}}""" + } + + private def invalidPayload(name: String) = + s"""{ + |"name": "$name", + |"propertyType": { + | "_t": "EnumPropertyType", + | "allowedValues": ["a", "b"], + | "suggestedValue": "invalidOptionC" + |} + |}""".stripMargin + + s"POST $apiUrl" can { + "return 201" when { + "PropertyDefinition is created" in { + val propertyDefinition = PropertyDefinitionFactory.getDummyPropertyDefinition() + val response = sendPostByAdmin[PropertyDefinition, Validation](apiUrl, bodyOpt = Some(propertyDefinition)) + assertCreated(response) + + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/property-definitions/datasets/dummyName/1") + + val response2 = sendGet[PropertyDefinition]("/property-definitions/datasets/dummyName/1") + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(propertyDefinition, actual) + assert(actual == expected) + } + Seq(Some("default1"), None).foreach { suggestedValue => + s"a PropertyDefinition is created with most of default values (suggestedValue=$suggestedValue)" in { + val propertyDefinition = minimalPdCreatePayload("smallPd", suggestedValue) + val response = sendPostByAdmin[String, Validation](apiUrl, bodyOpt = Some(propertyDefinition)) + assertCreated(response) + + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/property-definitions/datasets/smallPd/1") + + val response2 = sendGet[PropertyDefinition]("/property-definitions/datasets/smallPd/1") + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(PropertyDefinition("smallPd", propertyType = StringPropertyType(suggestedValue)), actual) + assert(actual == expected) + } + } + } + "return 400" when { + "an enabled PropertyDefinition with that name already exists" in { + val propertyDefinition = PropertyDefinitionFactory.getDummyPropertyDefinition() + propertyDefinitionFixture.add(propertyDefinition) + + val response = sendPostByAdmin[PropertyDefinition, Validation](apiUrl, bodyOpt = Some(propertyDefinition)) + assertBadRequest(response) + + val actual = response.getBody + val expected = Validation().withError("name", "entity with name already exists: 'dummyName'") + assert(actual == expected) + } + "an invalid PD payload is sent" in { + val response = sendPostByAdmin[String, String](apiUrl, bodyOpt = Some(invalidPayload("somePd1"))) + assertBadRequest(response) + + response.getBody shouldBe "The suggested value invalidOptionC cannot be used: Value 'invalidOptionC' is not one of the allowed values (a, b)." + } + } + } +} From 73bb8fd419396348b08c31cc28b8ec9c7fbd58dc Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 20 Apr 2022 11:12:13 +0200 Subject: [PATCH 33/67] #1693 admin rights required in VersionedModelControllerV3 -> reflected in DatasetControllerV3, MappingTableControllerV3, SchemaControllerV3. ITs fixed, but 403 tests missing --- .../controllers/v3/DatasetControllerV3.scala | 3 + .../v3/MappingTableControllerV3.scala | 3 + .../controllers/v3/SchemaControllerV3.scala | 5 +- .../controllers/BaseRestApiTest.scala | 46 ++++++++++++-- .../DatasetControllerV3IntegrationSuite.scala | 62 +++++++++---------- ...ingTableControllerV3IntegrationSuite.scala | 24 +++---- .../SchemaControllerV3IntegrationSuite.scala | 40 ++++++------ 7 files changed, 112 insertions(+), 71 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index b56cf937a..a72e97c22 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ @@ -46,6 +47,7 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) @PutMapping(Array("/{name}/{version}/properties")) @ResponseStatus(HttpStatus.OK) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def updateProperties(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: String, @@ -77,6 +79,7 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) @PostMapping(Array("/{name}/{version}/rules")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def addConformanceRule(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: String, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala index bfa63b160..e4fbb845b 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ @@ -48,6 +49,7 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @PutMapping(path = Array("/{name}/{version}/defaults")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def updateDefault(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: String, @@ -66,6 +68,7 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @PostMapping(path = Array("/{name}/{version}/defaults")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def addDefault(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: String, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala index 8899afb4e..a5e383bc6 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala @@ -18,6 +18,7 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import org.apache.spark.sql.types.StructType import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ @@ -33,7 +34,6 @@ import za.co.absa.enceladus.rest_api.utils.SchemaType import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import za.co.absa.enceladus.rest_api.utils.parsers.SchemaParser -import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.{HttpServletRequest, HttpServletResponse} import scala.concurrent.Future @@ -88,6 +88,7 @@ class SchemaControllerV3 @Autowired()( @PostMapping(Array("/{name}/{version}/from-file")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def handleFileUpload(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: Int, @@ -123,6 +124,7 @@ class SchemaControllerV3 @Autowired()( @PostMapping(Array("/{name}/{version}/from-remote-uri")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def handleRemoteFile(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: Int, @@ -150,6 +152,7 @@ class SchemaControllerV3 @Autowired()( @PostMapping(Array("/{name}/{version}/from-registry")) @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") def handleSubject(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: Int, diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 049b26126..1670e6ce7 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -117,6 +117,15 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR upload(urlPath, headers, fileParamName, fileName, parameters) } + def sendPostUploadFileByAdmin[T](urlPath: String, + fileName: String, + parameters: Map[String, Any], + fileParamName: String = "file", + headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { + upload(urlPath, headers, fileParamName, fileName, parameters, byAdmin = true) + } + def sendPostRemoteFile[T](urlPath: String, parameters: Map[String, Any], headers: HttpHeaders = new HttpHeaders()) @@ -126,6 +135,15 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR fromRemote(urlPath, headers, parameters) } + def sendPostRemoteFileByAdmin[T](urlPath: String, + parameters: Map[String, Any], + headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { + require(parameters.keySet.contains("remoteUrl"), s"parameters map must contain the 'remoteUrl' entry, but only $parameters was found") + + fromRemote(urlPath, headers, parameters, byAdmin = true) + } + def sendPostSubject[T](urlPath: String, parameters: Map[String, Any], headers: HttpHeaders = new HttpHeaders()) @@ -136,6 +154,16 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR fromRemote(urlPath, headers, parameters) } + def sendPostSubjectByAdmin[T](urlPath: String, + parameters: Map[String, Any], + headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { + require(parameters.keySet.contains("subject"), + s"parameters map must contain the 'subject', but only $parameters was found") + + fromRemote(urlPath, headers, parameters, byAdmin = true) + } + def sendPostAsync[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): Future[ResponseEntity[T]] = { sendAsync(HttpMethod.POST, urlPath, headers, bodyOpt) @@ -204,7 +232,8 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR headers: HttpHeaders = HttpHeaders.EMPTY, fileParamName: String, fileName: String, - additionalParams: Map[String, Any]) + additionalParams: Map[String, Any], + byAdmin: Boolean = false) (implicit ct: ClassTag[T]): ResponseEntity[T] = { val parameters = new LinkedMultiValueMap[String, Any] @@ -214,7 +243,11 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR } val url = s"$baseUrl/$urlPath" - headers.addAll(authHeaders) + if (byAdmin) { + headers.addAll(authHeadersAdmin) + } else { + headers.addAll(authHeaders) + } headers.setContentType(MediaType.MULTIPART_FORM_DATA) val clazz = ct.runtimeClass.asInstanceOf[Class[T]] @@ -225,7 +258,8 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR def fromRemote[T](urlPath: String, headers: HttpHeaders = HttpHeaders.EMPTY, - params: Map[String, Any]) + params: Map[String, Any], + byAdmin: Boolean = false) (implicit ct: ClassTag[T]): ResponseEntity[T] = { val parameters: MultiValueMap[String, String] = new LinkedMultiValueMap() @@ -234,7 +268,11 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR } val url = s"$baseUrl/$urlPath" - headers.addAll(authHeaders) + if (byAdmin) { + headers.addAll(authHeadersAdmin) + } else { + headers.addAll(authHeaders) + } headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED) val clazz = ct.runtimeClass.asInstanceOf[Class[T]] diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 35550887e..f65099b9e 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -71,7 +71,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + val response = sendPostByAdmin[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val locationHeader = response.getHeaders.getFirst("location") @@ -94,7 +94,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1, dataset2) val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create - val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) + val response = sendPostByAdmin[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) assertCreated(response) val locationHeaders = response.getHeaders.get("location").asScala locationHeaders should have size 1 @@ -116,7 +116,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset = DatasetFactory.getDummyDataset("dummyDs") // there are schemas defined - val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + val response = sendPostByAdmin[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertBadRequest(response) val responseBody = response.getBody @@ -128,7 +128,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("undefinedProperty1" -> "value1"))) // propdefs are empty - val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + val response = sendPostByAdmin[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertBadRequest(response) val responseBody = response.getBody @@ -280,7 +280,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA version = 2 // update references the last version ) - val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) + val response = sendPutByAdmin[Dataset, Validation](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) assertCreated(response) response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val locationHeader = response.getHeaders.getFirst("location") @@ -307,7 +307,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetA2 = DatasetFactory.getDummyDataset("datasetA", description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant - val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) + val response = sendPutByAdmin[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) assertBadRequest(response) val responseBody = response.getBody @@ -324,12 +324,12 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) datasetFixture.add(datasetA1) - val response = sendPut[Dataset, String](s"$apiUrl/datasetA/7", + val response = sendPutByAdmin[Dataset, String](s"$apiUrl/datasetA/7", bodyOpt = Some(DatasetFactory.getDummyDataset("datasetA", version = 5))) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody should include("version mismatch: 7 != 5") - val response2 = sendPut[Dataset, String](s"$apiUrl/datasetABC/4", + val response2 = sendPutByAdmin[Dataset, String](s"$apiUrl/datasetABC/4", bodyOpt = Some(DatasetFactory.getDummyDataset("datasetXYZ", version = 4))) response2.getStatusCode shouldBe HttpStatus.BAD_REQUEST response2.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") @@ -432,24 +432,21 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 400" when { "a Dataset with the given name" should { "fail when name in the URL and payload is mismatched" in { - val response = sendPost[String, String](s"$apiUrl/datasetABC/import", + val response = sendPostByAdmin[String, String](s"$apiUrl/datasetABC/import", bodyOpt = Some(importableDs)) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") } } - } - - "return 400" when { - "imported Dataset fails validation" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing + "imported Dataset fails validation" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing - val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPostByAdmin[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) - response.getStatusCode shouldBe HttpStatus.BAD_REQUEST - response.getBody shouldBe Validation.empty.withError("key2", "There is no property definition for key 'key2'.") - } + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody shouldBe Validation.empty.withError("key2", "There is no property definition for key 'key2'.") + } } "return 201" when { @@ -465,7 +462,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("key3", essentiality = Essentiality.Recommended) ) - val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPostByAdmin[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/2") @@ -495,7 +492,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("key2") ) - val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPostByAdmin[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version @@ -656,7 +653,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"PUT $apiUrl/{name}/{version}/properties" should { "return 404" when { "when the name+version does not exist" in { - val response = sendPut[Map[String, String], String](s"$apiUrl/notFoundDataset/456/properties", bodyOpt = Some(Map.empty)) + val response = sendPutByAdmin[Map[String, String], String](s"$apiUrl/notFoundDataset/456/properties", bodyOpt = Some(Map.empty)) assertNotFound(response) } } @@ -669,7 +666,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) datasetFixture.add(datasetV1, datasetV2, datasetV3) - val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) + val response = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) assertBadRequest(response) val responseBody = response.getBody @@ -684,7 +681,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(datasetV1) // propdefs are empty - val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + val response = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("undefinedProperty1" -> "someValue"))) assertBadRequest(response) @@ -702,13 +699,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) ) - val response1 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + val response1 = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("AorB" -> "a"))) // this is ok, but mandatoryA is missing assertBadRequest(response1) response1.getBody shouldBe Validation(Map("mandatoryA" -> List("Dataset property 'mandatoryA' is mandatory, but does not exist!"))) - val response2 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + val response2 = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("mandatoryA" -> "valueA", "AorB" -> "c"))) // mandatoryA is ok, but AorB has invalid value assertBadRequest(response2) @@ -733,7 +730,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val response1 = sendPut[String, Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) + val response1 = sendPutByAdmin[String, Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) assertCreated(response1) response1.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val headers1 = response1.getHeaders @@ -869,7 +866,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetV1) - val response = sendPost[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", + val response = sendPostByAdmin[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) assertNotFound(response) } @@ -883,21 +880,18 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA )) datasetFixture.add(datasetV1) - val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", + val response = sendPostByAdmin[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) assertBadRequest(response) response.getBody should include("Rule with order 0 cannot be added, another rule with this order already exists.") } - } - - "return 400" when { "when rule is not valid (missing MT)" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetV1) - val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", + val response = sendPostByAdmin[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleMcrRule0)) assertBadRequest(response) @@ -913,7 +907,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ) datasetFixture.add(datasetV1) - val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) + val response = sendPostByAdmin[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) assertCreated(response) // if, in the future, there can be a rule update resulting in a warning, let's reflect that here response.getBody shouldBe Validation.empty diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index f8f2a2469..6bcf40857 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -49,7 +49,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be "referenced schema does not exits" in { val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchemaA", schemaVersion = 1) - val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + val response = sendPostByAdmin[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) assertBadRequest(response) val responseBody = response.getBody @@ -64,7 +64,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist - val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + val response = sendPostByAdmin[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/mapping-tables/mtA/1") @@ -126,9 +126,9 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be "when the name/version does not exist" in { mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) - assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(Array()))) - assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(Array()))) - assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPutByAdmin[Array[DefaultValue], String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPutByAdmin[Array[DefaultValue], String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPutByAdmin[Array[DefaultValue], String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(Array()))) } } @@ -140,7 +140,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be mappingTableFixture.add(mtAv1, mtAv2, mtAv3) - val response = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(Array())) + val response = sendPutByAdmin[Array[DefaultValue], Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(Array())) assertBadRequest(response) val responseBody = response.getBody @@ -161,7 +161,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist - val response1 = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(bothPayloadAndExpectedResult)) + val response1 = sendPutByAdmin[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(bothPayloadAndExpectedResult)) assertCreated(response1) response1.getBody shouldBe Validation.empty val headers1 = response1.getHeaders @@ -182,9 +182,9 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) val aDefaultValue = DefaultValue("colA", "defaultA") - assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(aDefaultValue))) - assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(aDefaultValue))) - assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPostByAdmin[DefaultValue, String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPostByAdmin[DefaultValue, String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPostByAdmin[DefaultValue, String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(aDefaultValue))) } } @@ -196,7 +196,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be mappingTableFixture.add(mtAv1, mtAv2, mtAv3) - val response = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + val response = sendPostByAdmin[DefaultValue, Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) assertBadRequest(response) val responseBody = response.getBody @@ -213,7 +213,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist - val response1 = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + val response1 = sendPostByAdmin[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) assertCreated(response1) response1.getBody shouldBe Validation.empty val headers1 = response1.getHeaders diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 8add7bff9..2857701c7 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -87,7 +87,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a Schema is created" in { val schema = SchemaFactory.getDummySchema("schemaA") - val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") @@ -107,7 +107,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schema = SchemaFactory.getDummySchema() schemaFixture.add(schema) - val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) assertBadRequest(response) @@ -294,7 +294,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, String]("format" -> "copybook") - val responseUploaded = sendPostUploadFile[Validation]( + val responseUploaded = sendPostUploadFileByAdmin[Validation]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) assertCreated(responseUploaded) val locationHeader = responseUploaded.getHeaders.getFirst("location") @@ -316,7 +316,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, Any]("format" -> "struct") - val responseUploaded = sendPostUploadFile[Validation]( + val responseUploaded = sendPostUploadFileByAdmin[Validation]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) assertCreated(responseUploaded) val locationHeader = responseUploaded.getHeaders.getFirst("location") @@ -339,7 +339,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schemaParams = HashMap[String, Any]( "name" -> schema.name, "version" -> schema.version, "format" -> "avro") - val responseUploaded = sendPostUploadFile[Schema]( + val responseUploaded = sendPostUploadFileByAdmin[Schema]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.ok, schemaParams) assertCreated(responseUploaded) val locationHeader = responseUploaded.getHeaders.getFirst("location") @@ -362,7 +362,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap.empty[String, Any] // v2 fallbacked on this, v3 forbids it - val response = sendPostUploadFile[String]( + val response = sendPostUploadFileByAdmin[String]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) assertBadRequest(response) response.getBody should include("Required String parameter 'format' is not present") @@ -373,7 +373,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, Any]("format" -> "") // v2 fallbacked on this, v3 forbids it - val response = sendPostUploadFile[String]( + val response = sendPostUploadFileByAdmin[String]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) assertBadRequest(response) response.getBody should include("not a recognized schema format. Menas currently supports: struct, copybook, avro.") @@ -382,7 +382,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a copybook with a syntax error" should { "return a response containing a schema parsing error with syntax error specific fields" in { val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "copybook") - val response = sendPostUploadFile[RestResponse]( + val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.bogus, schemaParams) val body = response.getBody @@ -402,7 +402,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a JSON struct type schema with a syntax error" should { "return a response containing a schema parsing error returned by the StructType parser" in { val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "struct") - val response = sendPostUploadFile[RestResponse]( + val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) val body = response.getBody @@ -420,7 +420,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "an avro-schema with a syntax error" should { "return a response containing a schema parsing error encountered during avro schema parsing" in { val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "avro") - val response = sendPostUploadFile[RestResponse]( + val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.bogus, schemaParams) val body = response.getBody @@ -438,7 +438,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a wrong format has been specified" should { "return a response containing a schema format error" in { val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "foo") - val response = sendPostUploadFile[RestResponse]( + val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) val body = response.getBody @@ -458,7 +458,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a schema file is uploaded, but no schema exists for the specified name and version" in { val schemaParams = HashMap[String, Any]( "name" -> "dummy", "version" -> 1, "format" -> "copybook") - val responseUploaded = sendPostUploadFile[Schema]( + val responseUploaded = sendPostUploadFileByAdmin[Schema]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) assertNotFound(responseUploaded) } @@ -497,7 +497,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) - val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -521,7 +521,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) val params = HashMap("remoteUrl" -> remoteUrl, "format" -> "struct") - val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -545,7 +545,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "remoteUrl" -> remoteUrl) - val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -574,7 +574,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(testResourcePath))) val params = HashMap("name" -> "MySchema", "version" -> 1, "format" -> schemaType.toString, "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFile[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) + val response = sendPostRemoteFileByAdmin[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) val body = response.getBody assertBadRequest(response) @@ -595,7 +595,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) val params = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "foo", "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFile[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) + val response = sendPostRemoteFileByAdmin[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) val body = response.getBody assertBadRequest(response) @@ -616,7 +616,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) val params = HashMap[String, Any]("version" -> 1, "name" -> "dummy", "format" -> "copybook", "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val response = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertNotFound(response) } } @@ -635,7 +635,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic1-value") - val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) + val responseRemoteLoaded = sendPostSubjectByAdmin[Schema](s"$apiUrl/schemaA/1/from-registry", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -660,7 +660,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic2") - val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) + val responseRemoteLoaded = sendPostSubjectByAdmin[Schema](s"$apiUrl/schemaA/1/from-registry", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version From a2c5f407ad16d34bfcebe54bbafd8f85733401f2 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 20 Apr 2022 17:20:24 +0200 Subject: [PATCH 34/67] #1693 admin rights required in VersionedModelControllerV3 -> ITs with 403 cases for all types --- .../DatasetControllerV3IntegrationSuite.scala | 66 ++++++++++++++++++- ...ingTableControllerV3IntegrationSuite.scala | 28 +++++++- ...finitionControllerV3IntegrationSuite.scala | 10 +++ .../SchemaControllerV3IntegrationSuite.scala | 61 ++++++++++++----- 4 files changed, 148 insertions(+), 17 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index f65099b9e..654339bc5 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -135,6 +135,16 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } } + + "return 403" when { + s"admin auth is not used for POST" in { + val dataset = DatasetFactory.getDummyDataset("dummyDs") + + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + // todo what to do if "the last dataset version is disabled"? } @@ -336,6 +346,18 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } } + + "return 403" when { + s"admin auth is not used for PUT" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) + datasetFixture.add(datasetA1) + + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", description = Some("second version"), version = 2) + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } } s"GET $apiUrl/{name}/audit-trail" should { @@ -449,6 +471,22 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + "return 403" when { + s"admin auth is not used for POST" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("init version")) + datasetFixture.add(dataset1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) + + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + "return 201" when { "there is a existing Dataset" should { "a +1 version of dataset is added" in { @@ -649,7 +687,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"PUT $apiUrl/{name}/{version}/properties" should { "return 404" when { "when the name+version does not exist" in { @@ -713,6 +750,20 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + "return 403" when { + s"admin auth is not used for PUT" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("keyA")) + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("keyA" -> "valA"))) + + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + "201 Created with location" when { Seq( ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) @@ -899,6 +950,18 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + "return 403" when { + s"admin auth is not used for POST" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", + bodyOpt = Some(LiteralConformanceRule(order = 0,"column1", true, "ABC"))) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + "return 201" when { "when conf rule is added" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) @@ -956,4 +1019,5 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + // todo delete tests are missing } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index 6bcf40857..58e4aa744 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -20,10 +20,12 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.{DefaultValue, MappingTable, Validation} -import za.co.absa.enceladus.model.test.factories.{MappingTableFactory, SchemaFactory} +import za.co.absa.enceladus.model.test.factories.{MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.rest_api.integration.controllers.BaseRestApiTestV3 import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.integration.controllers.toExpected @@ -57,6 +59,14 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } + "return 403" when { + s"admin auth is not used for POST $apiUrl" in { + val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchemaA", schemaVersion = 1) + val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + "return 201" when { "a MappingTables is created" in { val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchema1", schemaVersion = 1) @@ -150,6 +160,14 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } + "return 403" when { + s"admin auth is not used for PUT" in { + // no need for fixture whip-up, auth check should precede further processing + val response = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(Array.empty[DefaultValue])) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + "201 Created with location" when { Seq( ("empty defaults", Array.empty[DefaultValue]), @@ -206,6 +224,14 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } + "return 403" when { + s"admin auth is not used for POST" in { + // no need for fixture whip-up, auth check should precede further processing + val response = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + "201 Created with location" when { s"defaults are replaced with a new version" in { val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 62c55cdd2..63670c4b9 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -20,6 +20,7 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.Validation @@ -97,6 +98,7 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w } } } + "return 400" when { "an enabled PropertyDefinition with that name already exists" in { val propertyDefinition = PropertyDefinitionFactory.getDummyPropertyDefinition() @@ -116,5 +118,13 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w response.getBody shouldBe "The suggested value invalidOptionC cannot be used: Value 'invalidOptionC' is not one of the allowed values (a, b)." } } + + "return 403" when { + s"admin auth is not used for POST $apiUrl" in { + val propertyDefinition = PropertyDefinitionFactory.getDummyPropertyDefinition() + val response = sendPost[PropertyDefinition, String](apiUrl, bodyOpt = Some(propertyDefinition)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 2857701c7..3305750e4 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -27,12 +27,10 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.{HttpStatus, MediaType} import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.menas.MenasReference -import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} -import za.co.absa.enceladus.model.{MappingTable, Schema, UsedIn, Validation} +import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, SchemaFactory} +import za.co.absa.enceladus.model.{Schema, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ -import za.co.absa.enceladus.rest_api.models.SchemaApiFeatures import za.co.absa.enceladus.rest_api.models.rest.RestResponse import za.co.absa.enceladus.rest_api.models.rest.errors.{SchemaFormatError, SchemaParsingError} import za.co.absa.enceladus.rest_api.repositories.RefCollection @@ -116,6 +114,14 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn assert(actual == expected) } } + + "return 403" when { + s"admin auth is not used for POST $apiUrl" in { + val schema = SchemaFactory.getDummySchema() + val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } } // todo disable dataset - all versions/one version/ check the usage to prevent from disabling @@ -337,8 +343,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schema = SchemaFactory.getDummySchema("schemaA") schemaFixture.add(schema) - val schemaParams = HashMap[String, Any]( - "name" -> schema.name, "version" -> schema.version, "format" -> "avro") + val schemaParams = HashMap[String, Any]("format" -> "avro") val responseUploaded = sendPostUploadFileByAdmin[Schema]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.ok, schemaParams) assertCreated(responseUploaded) @@ -381,7 +386,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a copybook with a syntax error" should { "return a response containing a schema parsing error with syntax error specific fields" in { - val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "copybook") + val schemaParams = HashMap[String, Any]("format" -> "copybook") val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.bogus, schemaParams) val body = response.getBody @@ -401,7 +406,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a JSON struct type schema with a syntax error" should { "return a response containing a schema parsing error returned by the StructType parser" in { - val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "struct") + val schemaParams = HashMap[String, Any]("format" -> "struct") val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) val body = response.getBody @@ -419,7 +424,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "an avro-schema with a syntax error" should { "return a response containing a schema parsing error encountered during avro schema parsing" in { - val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "avro") + val schemaParams = HashMap[String, Any]("format" -> "avro") val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.bogus, schemaParams) val body = response.getBody @@ -437,7 +442,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a wrong format has been specified" should { "return a response containing a schema format error" in { - val schemaParams = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "foo") + val schemaParams = HashMap[String, Any]("format" -> "foo") val response = sendPostUploadFileByAdmin[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) val body = response.getBody @@ -456,13 +461,21 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "return 404" when { "a schema file is uploaded, but no schema exists for the specified name and version" in { - val schemaParams = HashMap[String, Any]( - "name" -> "dummy", "version" -> 1, "format" -> "copybook") + val schemaParams = HashMap[String, Any]("format" -> "copybook") val responseUploaded = sendPostUploadFileByAdmin[Schema]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) assertNotFound(responseUploaded) } } + + "return 403" when { + s"admin auth is not used for POST $apiUrl/{name}/{version}/from-file" in { + val schemaParams = HashMap[String, Any]("format" -> "copybook") + val response = sendPostUploadFile[Schema]( + s"$apiUrl/irrelevantWhatSchema/123/from-file", TestResourcePath.Copybook.ok, schemaParams) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } } import com.github.tomakehurst.wiremock.client.WireMock._ @@ -573,7 +586,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) .willReturn(readTestResourceAsResponseWithContentType(testResourcePath))) - val params = HashMap("name" -> "MySchema", "version" -> 1, "format" -> schemaType.toString, "remoteUrl" -> remoteUrl) + val params = HashMap("format" -> schemaType.toString, "remoteUrl" -> remoteUrl) val response = sendPostRemoteFileByAdmin[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) val body = response.getBody @@ -594,7 +607,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) - val params = HashMap[String, Any]("version" -> 1, "name" -> "MySchema", "format" -> "foo", "remoteUrl" -> remoteUrl) + val params = HashMap[String, Any]("format" -> "foo", "remoteUrl" -> remoteUrl) val response = sendPostRemoteFileByAdmin[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) val body = response.getBody @@ -615,11 +628,20 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) - val params = HashMap[String, Any]("version" -> 1, "name" -> "dummy", "format" -> "copybook", "remoteUrl" -> remoteUrl) + val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) val response = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertNotFound(response) } } + + "return 403" when { + s"admin auth is not used for POST $apiUrl/{name}/{version}/from-remote-uri" in { + // no need for any mocking, auth check should precede further processing + val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) + val response = sendPostRemoteFile[Schema](s"$apiUrl/irrelevantWhatSchema/123/from-remote-uri", params) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } } s"POST $apiUrl/{name}/{version}/from-registry" should { @@ -675,6 +697,15 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } } + + "return 403" when { + s"admin auth is not used for POST $apiUrl/{name}/{version}/from-registry" in { + // no need for any mocking, auth check should precede further processing + val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopicABC") + val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/irrelevantWhatSchema/123/from-registry", params) + responseRemoteLoaded.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } } } From 4d9f623a454c6537570556282a52ae7fed46c900 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 22 Apr 2022 16:37:03 +0200 Subject: [PATCH 35/67] #1693 schema api v3: GET ... /json 400 with validation errors on empty fields + PUT/POST with schema with empty fields -> 400 validation error, too. + ITs reflecting this (this reqiured parallel SchemaServiceV3) --- .../controllers/v3/SchemaControllerV3.scala | 11 +-- .../v3/VersionedModelControllerV3.scala | 2 +- .../rest_api/services/SchemaService.scala | 4 +- .../services/VersionedModelService.scala | 1 + .../services/v3/SchemaServiceV3.scala | 96 +++++++++++++++++++ .../SchemaControllerV3IntegrationSuite.scala | 79 ++++++++++++++- 6 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala index a5e383bc6..58a8a1f75 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala @@ -29,6 +29,7 @@ import za.co.absa.enceladus.rest_api.controllers.SchemaController import za.co.absa.enceladus.rest_api.exceptions.ValidationException import za.co.absa.enceladus.rest_api.models.rest.exceptions.SchemaParsingException import za.co.absa.enceladus.rest_api.repositories.RefCollection +import za.co.absa.enceladus.rest_api.services.v3.SchemaServiceV3 import za.co.absa.enceladus.rest_api.services.{AttachmentService, SchemaRegistryService, SchemaService} import za.co.absa.enceladus.rest_api.utils.SchemaType import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor @@ -43,10 +44,10 @@ import scala.util.{Failure, Success, Try} @RestController @RequestMapping(Array("/api-v3/schemas")) class SchemaControllerV3 @Autowired()( - schemaService: SchemaService, - attachmentService: AttachmentService, - sparkMenasConvertor: SparkMenasSchemaConvertor, - schemaRegistryService: SchemaRegistryService + schemaService: SchemaServiceV3, + attachmentService: AttachmentService, + sparkMenasConvertor: SparkMenasSchemaConvertor, + schemaRegistryService: SchemaRegistryService ) extends VersionedModelControllerV3(schemaService) { @@ -61,10 +62,8 @@ class SchemaControllerV3 @Autowired()( @RequestParam(defaultValue = "false") pretty: Boolean): CompletableFuture[String] = { forVersionExpression(name, version) (schemaService.getVersion).map { case Some(schema) => - // todo why is this a problem at all? why is this specific to json? if (schema.fields.isEmpty) throw ValidationException( Validation.empty.withError("schema-fields", s"Schema $name v$version exists, but has no fields!") - // or throw notFound() as v2? ) val sparkStruct = StructType(sparkMenasConvertor.convertMenasToSparkFields(schema.fields)) if (pretty) sparkStruct.prettyJson else sparkStruct.json diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 576745b7d..4f139150d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -140,7 +140,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } @PutMapping(Array("/{name}/{version}")) - @ResponseStatus(HttpStatus.NO_CONTENT) + @ResponseStatus(HttpStatus.CREATED) @PreAuthorize("@authConstants.hasAdminRole(authentication)") def edit(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index 9583a54f6..9ae9b166f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -15,14 +15,14 @@ package za.co.absa.enceladus.rest_api.services +import org.apache.spark.sql.types.StructType import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} +import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import scala.concurrent.Future -import org.apache.spark.sql.types.StructType -import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor @Service class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index bb13f677f..55819db4b 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -314,6 +314,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit validateName(item.name) } + /** does not include za.co.absa.enceladus.rest_api.services.VersionedModelService#validate(java.lang.Object)*/ def validateForCreation(item: C): Future[Validation] = { isUniqueName(item.name).map { isUnique => if (isUnique) { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala new file mode 100644 index 000000000..59d7686bf --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.apache.spark.sql.types.StructType +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} +import za.co.absa.enceladus.rest_api.services.{SchemaService, VersionedModelService} +import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor +import scala.concurrent.ExecutionContext.Implicits.global + + +import scala.concurrent.Future + +@Service +class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, + mappingTableMongoRepository: MappingTableMongoRepository, + datasetMongoRepository: DatasetMongoRepository, + sparkMenasConvertor: SparkMenasSchemaConvertor) + extends VersionedModelService(schemaMongoRepository) { + + // same as v2 + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + for { + usedInD <- datasetMongoRepository.findRefEqual("schemaName", "schemaVersion", name, version) + usedInM <- mappingTableMongoRepository.findRefEqual("schemaName", "schemaVersion", name, version) + } yield UsedIn(Some(usedInD), Some(usedInM)) + } + + // same as v2 + def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[(Schema, Validation)] = { + super.update(username, schemaName, schemaVersion)({ oldSchema => + oldSchema.copy(fields = sparkMenasConvertor.convertSparkToMenasFields(fields.fields).toList) + }).map(_.getOrElse(throw new IllegalArgumentException("Failed to derive new schema from file!"))) + } + + override def validate(item: Schema): Future[Validation] = { + if (item.fields.isEmpty) { + // V3 disallows empty schema fields - V2 allowed it at first that to get updated by an attachment upload/remote-load + Future.successful(Validation.empty.withError("schema-fields","No fields found! There must be fields defined for actual usage.")) + } else { + Future.successful(Validation.empty) + } + } + + // same as V2, but fields from payload are used, too + override def update(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { + super.update(username, schema.name, schema.version) { latest => + latest.setDescription(schema.description).asInstanceOf[Schema].copy(fields = schema.fields) + } + } + + // same as V2, but fields from payload are used, too + override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { + val schema = Schema(name = newSchema.name, + description = newSchema.description, + fields = newSchema.fields + ) + super.create(schema, username) + } + + // same as v2 + override def recreate(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { + for { + latestVersion <- getLatestVersionNumber(schema.name) + update <- super.update(username, schema.name, latestVersion) { latest => + latest + .copy(fields = List()) + .setDescription(schema.description).asInstanceOf[Schema] + } + } yield update + } + + // same as v2 + override def importItem(item: Schema, username: String): Future[Option[(Schema, Validation)]] = { + getLatestVersionValue(item.name).flatMap { + case Some(version) => update(username, item.copy(version = version)) + case None => super.create(item.copy(version = 1), username) + } + } +} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 3305750e4..e2430fddb 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import org.springframework.http.{HttpStatus, MediaType} import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, SchemaFactory} -import za.co.absa.enceladus.model.{Schema, Validation} +import za.co.absa.enceladus.model.{Schema, SchemaField, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.RestResponse @@ -82,8 +82,10 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn s"POST $apiUrl" can { "return 201" when { - "a Schema is created" in { - val schema = SchemaFactory.getDummySchema("schemaA") + "a Schema is created (v1-payload has defined fields already)" in { + val schema = SchemaFactory.getDummySchema("schemaA", fields = List( + SchemaField("field1", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) + )) val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) @@ -91,6 +93,8 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/1") + response.getBody shouldBe Validation.empty + val response2 = sendGet[Schema]("/schemas/schemaA/1") assertOk(response2) @@ -101,8 +105,17 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } "return 400" when { + "a Schema is created (empty fields = warning)" in { + val schema = SchemaFactory.getDummySchema("schemaA") + val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + + assertBadRequest(response) + response.getBody shouldBe Validation.empty + .withError("schema-fields", "No fields found! There must be fields defined for actual usage.") + } "an enabled Schema with that name already exists" in { - val schema = SchemaFactory.getDummySchema() + val schema = SchemaFactory.getDummySchema(fields = List( + SchemaField("field1", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty))) schemaFixture.add(schema) val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) @@ -110,7 +123,9 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn assertBadRequest(response) val actual = response.getBody - val expected = Validation().withError("name", "entity with name already exists: 'dummyName'") + val expected = Validation.empty + .withError("name", "entity with name already exists: 'dummyName'") + assert(actual == expected) } } @@ -124,6 +139,60 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } + s"PUT $apiUrl/{name}/{version}" can { + "return 201" when { + "a Schema is updated (v1-payload has defined fields already)" in { + val schema1 = SchemaFactory.getDummySchema("schemaA", fields = List( + SchemaField("field1", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) + )) + schemaFixture.add(schema1) + + val schema2 = SchemaFactory.getDummySchema("schemaA", fields = List( + SchemaField("anotherField", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) + )) + val response = sendPutByAdmin[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) + + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/schemas/schemaA/2") + + response.getBody shouldBe Validation.empty + + val response2 = sendGet[Schema]("/schemas/schemaA/2") + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(schema2.copy(version = 2, parent = Some(SchemaFactory.toParent(schema1))), actual) + assert(actual == expected) + } + } + + "return 400" when { + "a Schema fails to update due to empty fields" in { + val schema1 = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema1) + + val schema2 = SchemaFactory.getDummySchema("schemaA") + val response = sendPutByAdmin[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) + + assertBadRequest(response) + response.getBody shouldBe Validation.empty + .withError("schema-fields", "No fields found! There must be fields defined for actual usage.") + } + } + + "return 403" when { + s"admin auth is not used for POST $apiUrl" in { + val schema1 = SchemaFactory.getDummySchema("schemaA") + schemaFixture.add(schema1) + + val schema = SchemaFactory.getDummySchema("schemaA") + val response = sendPut[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } + // todo disable dataset - all versions/one version/ check the usage to prevent from disabling // todo used-in implementation checks From f410702f0f126fe016c82e4e3abe248b5ee1e4d1 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 25 Apr 2022 10:45:48 +0200 Subject: [PATCH 36/67] #1693 dataset v2 delete IT asserted to make sure there are regressions to the implementation of V3 alongside of V2 --- .../DatasetApiIntegrationSuite.scala | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala index 837d7d1c2..98d9e6a47 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala @@ -437,4 +437,48 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl } } + s"DELETE $apiUrl/disable/{name}/{version}" can { + "return 200" when { + "a Dataset with the given name and version exists" should { + "disable only the dataset with the given name and version" in { + val dsA = DatasetFactory.getDummyDataset(name = "dsA", version = 1) + val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1) + datasetFixture.add(dsA, dsB) + + val response = sendDelete[Dataset, String](s"$apiUrl/disable/dsA/1") + + assertOk(response) + + val actual = response.getBody + val expected = """{"matchedCount":1,"modifiedCount":1,"upsertedId":null,"modifiedCountAvailable":true}""" + assert(actual == expected) + } + } + "multiple versions of the Dataset with the given name exist" should { + "disable the specified version of the Dataset" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2) + datasetFixture.add(dsA1, dsA2) + + val response = sendDelete[Dataset, String](s"$apiUrl/disable/dsA/1") + + assertOk(response) + + val actual = response.getBody + val expected = """{"matchedCount":1,"modifiedCount":1,"upsertedId":null,"modifiedCountAvailable":true}""" + assert(actual == expected) + } + } + + "no Dataset with the given name exists" should { + "disable nothing" in { + val response = sendDelete[Dataset, String](s"$apiUrl/disable/aDataset/1") + + assertNotFound(response) + // Beware that, sadly, V2 Schemas returns 200 on disable of non-existent entity while V2 Datasets returns 404 + // This is due to getUsedIn implementation (non) checking the entity existence. + } + } + } + } } From 27b2e6232d8367249f5da6735e0d0a1c6af2135a Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 25 Apr 2022 15:25:53 +0200 Subject: [PATCH 37/67] #1693 VersionedModelControllerV3 disable (DELETE .../{name}) + enable (PUT .../{name}) implemented. ITs cover dataset disabling - BaseRestApiTest omits entity for Delete calls + reflected in various places --- .../v3/VersionedModelControllerV3.scala | 34 ++++++--- .../models/rest/DisabledPayload.scala | 18 +++++ .../VersionedMongoRepository.scala | 9 +++ .../services/VersionedModelService.scala | 11 +++ .../controllers/BaseRestApiTest.scala | 8 +- .../DatasetApiIntegrationSuite.scala | 7 +- ...ropertyDefinitionApiIntegrationSuite.scala | 10 +-- .../SchemaApiFeaturesIntegrationSuite.scala | 32 ++++---- .../DatasetControllerV3IntegrationSuite.scala | 74 ++++++++++++++++++- .../SchemaControllerV3IntegrationSuite.scala | 2 + 10 files changed, 166 insertions(+), 39 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 4f139150d..a3ac06beb 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -27,6 +27,8 @@ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation} import za.co.absa.enceladus.rest_api.controllers.BaseController import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey +import za.co.absa.enceladus.rest_api.exceptions.NotFoundException +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload import za.co.absa.enceladus.rest_api.services.VersionedModelService import java.net.URI @@ -161,18 +163,32 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } - @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @PutMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) @PreAuthorize("@authConstants.hasAdminRole(authentication)") - def disable(@PathVariable name: String, - @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { - val v = if (version.isPresent) { - // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException - Some(version.get.toInt) - } else { - None + def enable(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String): CompletableFuture[DisabledPayload] = { + + versionedModelService.disableVersion(name, None).map { updateResult => // always disabling all version of the entity + if(updateResult.getMatchedCount >= 0) { + DisabledPayload(disabled = false) + } else { + throw NotFoundException(s"No versions for entity $name found to be enabled.") + } + } + } + + @DeleteMapping(Array("/{name}")) + @ResponseStatus(HttpStatus.OK) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") + def disable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { + versionedModelService.disableVersion(name, None).map { updateResult => // always disabling all version of the entity + if(updateResult.getMatchedCount >= 0) { + DisabledPayload(disabled = true) + } else { + throw NotFoundException(s"No versions for entity $name found to be disabled.") + } } - versionedModelService.disableVersion(name, v) } /** diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala new file mode 100644 index 000000000..009cefdb5 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.models.rest + +case class DisabledPayload(disabled: Boolean) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 51274b7ec..86971d785 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -133,6 +133,7 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab } + // for V3 usage: version = None def disableVersion(name: String, version: Option[Int], username: String): Future[UpdateResult] = { collection.updateMany(getNameVersionFilter(name, version), combine( set("disabled", true), @@ -140,6 +141,14 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab set("userDisabled", username))).toFuture() } + // V3 only + def enableAllVersions(name: String, username: String): Future[UpdateResult] = { + collection.updateMany(getNameVersionFilter(name, version = None), combine( + set("disabled", false), + set("dateDisabled", ZonedDateTime.now()), + set("userDisabled", username))).toFuture() + } + def isDisabled(name: String): Future[Boolean] = { val pipeline = Seq(filter(getNameFilter(name)), Aggregates.addFields(Field("enabled", BsonDocument("""{$toInt: {$not: "$disabled"}}"""))), diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index 55819db4b..6fef19d0a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -266,6 +266,17 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.findRefEqual(refNameCol, refVersionCol, name, version) } + /** + * Enables all versions of the entity by name. + * @param name + */ + def enableEntity(name: String): Future[UpdateResult] = { + val auth = SecurityContextHolder.getContext.getAuthentication + val principal = auth.getPrincipal.asInstanceOf[UserDetails] + + versionedMongoRepository.enableAllVersions(name, principal.getUsername) + } + def disableVersion(name: String, version: Option[Int]): Future[UpdateResult] = { val auth = SecurityContextHolder.getContext.getAuthentication val principal = auth.getPrincipal.asInstanceOf[UserDetails] diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 1670e6ce7..1206b1b15 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -184,13 +184,13 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR sendAsync(HttpMethod.PUT, urlPath, headers, bodyOpt) } - def sendDelete[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), - bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { + def sendDelete[T](urlPath: String, headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { send(HttpMethod.DELETE, urlPath, headers) } - def sendDeleteByAdmin[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), - bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { + def sendDeleteByAdmin[T](urlPath: String, headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { sendByAdmin(HttpMethod.DELETE, urlPath, headers) } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala index 98d9e6a47..4f13250da 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala @@ -23,7 +23,6 @@ import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule import za.co.absa.enceladus.model.dataFrameFilter._ -import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ @@ -445,7 +444,7 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1) datasetFixture.add(dsA, dsB) - val response = sendDelete[Dataset, String](s"$apiUrl/disable/dsA/1") + val response = sendDelete[String](s"$apiUrl/disable/dsA/1") assertOk(response) @@ -460,7 +459,7 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2) datasetFixture.add(dsA1, dsA2) - val response = sendDelete[Dataset, String](s"$apiUrl/disable/dsA/1") + val response = sendDelete[String](s"$apiUrl/disable/dsA/1") assertOk(response) @@ -472,7 +471,7 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl "no Dataset with the given name exists" should { "disable nothing" in { - val response = sendDelete[Dataset, String](s"$apiUrl/disable/aDataset/1") + val response = sendDelete[String](s"$apiUrl/disable/aDataset/1") assertNotFound(response) // Beware that, sadly, V2 Schemas returns 200 on disable of non-existent entity while V2 Datasets returns 404 diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala index c8b9ad592..7c451b08d 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala @@ -138,7 +138,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "otherPropertyDefinition", version = 1) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition") assertOk(response) @@ -153,7 +153,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition") assertOk(response) @@ -173,7 +173,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "otherPropertyDefinition", version = 1) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition/1") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition/1") assertOk(response) val actual = response.getBody @@ -191,7 +191,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](deleteUrl) + val response = sendDeleteByAdmin[String](deleteUrl) assertOk(response) val actual = response.getBody @@ -203,7 +203,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor "no PropertyDefinition with the given name exists" should { "disable nothing" in { - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition/1") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition/1") assertOk(response) val actual = response.getBody diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala index 6251e5f71..18935a7fb 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala @@ -223,7 +223,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "otherSchema", version = 1) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -238,7 +238,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -255,7 +255,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -272,7 +272,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -283,7 +283,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd } "no Schema with the given name exists" should { "disable nothing" in { - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -303,7 +303,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema") + val response = sendDelete[UsedIn](s"$apiUrl/disable/schema") assertBadRequest(response) @@ -320,7 +320,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema") + val response = sendDelete[UsedIn](s"$apiUrl/disable/schema") assertBadRequest(response) @@ -340,7 +340,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "otherSchema", version = 1) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -355,7 +355,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -372,7 +372,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -389,7 +389,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/2") + val response = sendDelete[String](s"$apiUrl/disable/schema/2") assertOk(response) @@ -406,7 +406,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -423,7 +423,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/2") + val response = sendDelete[String](s"$apiUrl/disable/schema/2") assertOk(response) @@ -434,7 +434,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd } "no Schema with the given name exists" should { "disable nothing" in { - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -455,7 +455,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema/1") + val response = sendDelete[UsedIn](s"$apiUrl/disable/schema/1") assertBadRequest(response) @@ -473,7 +473,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema/1") + val response = sendDelete[UsedIn](s"$apiUrl/disable/schema/1") assertBadRequest(response) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 654339bc5..5c0f2f1b6 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -32,6 +32,7 @@ import za.co.absa.enceladus.model.versionedModel.VersionList import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload import scala.collection.JavaConverters._ @@ -1019,5 +1020,76 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo delete tests are missing + // todo PUT to $apiUrl/{name} to enable test cases + // todo deal with historically partially-disabled entities + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a Dataset with the given name exists" should { + "disable the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2) + val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1) + datasetFixture.add(dsA1, dsA2, dsB) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated dataset unaffected + val responseB = sendGet[Dataset](s"$apiUrl/dsB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a Dataset with the given name exists and there hare mixed disabled states (historical)" should { + "disable all versions the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) + datasetFixture.add(dsA1, dsA2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + + "no Dataset with the given name exists" should { + "disable nothing" in { + val response = sendDeleteByAdmin[String](s"$apiUrl/aDataset") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + val response = sendDelete[Validation](s"$apiUrl/datasetA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index e2430fddb..a96a5a902 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -777,4 +777,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } + // todo DELETE for v3: 404 for not-found entities (v2 for schemas returned 200) + } From 1787692a49cbca92b8eb4ff677e85eb1ee98136b Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 26 Apr 2022 12:18:19 +0200 Subject: [PATCH 38/67] #1693 Dataset V3 API enable/disable ITs + impl fix --- .../v3/VersionedModelControllerV3.scala | 10 +- .../DatasetControllerV3IntegrationSuite.scala | 95 ++++++++++++++++--- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index a3ac06beb..b35f2f735 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -166,11 +166,9 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PutMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) @PreAuthorize("@authConstants.hasAdminRole(authentication)") - def enable(@AuthenticationPrincipal user: UserDetails, - @PathVariable name: String): CompletableFuture[DisabledPayload] = { - - versionedModelService.disableVersion(name, None).map { updateResult => // always disabling all version of the entity - if(updateResult.getMatchedCount >= 0) { + def enable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { + versionedModelService.enableEntity(name).map { updateResult => // always enabling all version of the entity + if(updateResult.getMatchedCount > 0) { DisabledPayload(disabled = false) } else { throw NotFoundException(s"No versions for entity $name found to be enabled.") @@ -183,7 +181,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PreAuthorize("@authConstants.hasAdminRole(authentication)") def disable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { versionedModelService.disableVersion(name, None).map { updateResult => // always disabling all version of the entity - if(updateResult.getMatchedCount >= 0) { + if(updateResult.getMatchedCount > 0) { DisabledPayload(disabled = true) } else { throw NotFoundException(s"No versions for entity $name found to be disabled.") diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 5c0f2f1b6..08ceaba75 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -274,9 +274,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA description = Some("second version"), properties = Some(Map("keyA" -> "valA")), version = 2) datasetFixture.add(datasetA1, datasetA2) - Seq("keyA", "keyB", "keyC").foreach {propName => propertyDefinitionFixture.add( - PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) - )} + Seq("keyA", "keyB", "keyC").foreach { propName => + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) + ) + } // this will cause missing property 'keyD' to issue a warning if not present propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) @@ -919,7 +921,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(datasetV1) val response = sendPostByAdmin[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", - bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + bodyOpt = Some(LiteralConformanceRule(0, "column1", true, value = "ABC"))) assertNotFound(response) } } @@ -928,12 +930,12 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when the there is a conflicting conf rule #" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( - LiteralConformanceRule(order = 0,"column1", true, "ABC") + LiteralConformanceRule(order = 0, "column1", true, "ABC") )) datasetFixture.add(datasetV1) val response = sendPostByAdmin[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", - bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + bodyOpt = Some(LiteralConformanceRule(0, "column1", true, value = "ABC"))) assertBadRequest(response) response.getBody should include("Rule with order 0 cannot be added, another rule with this order already exists.") @@ -958,7 +960,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(datasetV1) val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", - bodyOpt = Some(LiteralConformanceRule(order = 0,"column1", true, "ABC"))) + bodyOpt = Some(LiteralConformanceRule(order = 0, "column1", true, "ABC"))) response.getStatusCode shouldBe HttpStatus.FORBIDDEN } } @@ -967,7 +969,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when conf rule is added" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( - LiteralConformanceRule(order = 0,"column1", true, "ABC")) + LiteralConformanceRule(order = 0, "column1", true, "ABC")) ) datasetFixture.add(datasetV1) @@ -1020,8 +1022,77 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo PUT to $apiUrl/{name} to enable test cases - // todo deal with historically partially-disabled entities + s"PUT $apiUrl/{name}" can { + "return 200" when { + "a Dataset with the given name exists" should { + "enable the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = true) + val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1, disabled = true) + datasetFixture.add(dsA1, dsA2, dsB) + + val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions now enabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + + // unrelated dataset unaffected + val responseB = sendGet[Dataset](s"$apiUrl/dsB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe true + } + } + + "a Dataset with the given name exists and there have mixed disabled states (historical)" should { + "enable all versions the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) + datasetFixture.add(dsA1, dsA2) + + val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions enabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + } + } + } + + "return 404" when { + "no Dataset with the given name exists" should { + "enable nothing" in { + val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/aDataset") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + val response = sendDelete[Validation](s"$apiUrl/datasetA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } s"DELETE $apiUrl/{name}" can { "return 200" when { @@ -1052,7 +1123,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - "a Dataset with the given name exists and there hare mixed disabled states (historical)" should { + "a Dataset with the given name exists and there have mixed disabled states (historical)" should { "disable all versions the dataset with the given name" in { val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) @@ -1072,7 +1143,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA responseA2.getBody.disabled shouldBe true } } + } + "return 404" when { "no Dataset with the given name exists" should { "disable nothing" in { val response = sendDeleteByAdmin[String](s"$apiUrl/aDataset") From e4559b110c8b56fc7e16a82cab27b45470a67bf7 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 26 Apr 2022 12:32:53 +0200 Subject: [PATCH 39/67] #1693 Schema V3 API enable/disable simple cases - no usedIn checks, yet --- .../SchemaControllerV3IntegrationSuite.scala | 160 ++++++++++++++++-- 1 file changed, 147 insertions(+), 13 deletions(-) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index a96a5a902..07aff8bf8 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -31,7 +31,7 @@ import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, SchemaFacto import za.co.absa.enceladus.model.{Schema, SchemaField, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ -import za.co.absa.enceladus.rest_api.models.rest.RestResponse +import za.co.absa.enceladus.rest_api.models.rest.{DisabledPayload, RestResponse} import za.co.absa.enceladus.rest_api.models.rest.errors.{SchemaFormatError, SchemaParsingError} import za.co.absa.enceladus.rest_api.repositories.RefCollection import za.co.absa.enceladus.rest_api.utils.SchemaType @@ -63,22 +63,13 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn @Autowired private val schemaFixture: SchemaFixtureService = null - @Autowired - private val datasetFixture: DatasetFixtureService = null - - @Autowired - private val mappingTableFixture: MappingTableFixtureService = null - @Autowired private val attachmentFixture: AttachmentFixtureService = null - @Autowired - private val convertor: SparkMenasSchemaConvertor = null - private val apiUrl = "/schemas" private val schemaRefCollection = RefCollection.SCHEMA.name().toLowerCase() - override def fixtures: List[FixtureService[_]] = List(schemaFixture, attachmentFixture, datasetFixture, mappingTableFixture) + override def fixtures: List[FixtureService[_]] = List(schemaFixture, attachmentFixture) s"POST $apiUrl" can { "return 201" when { @@ -193,8 +184,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } - // todo disable dataset - all versions/one version/ check the usage to prevent from disabling - // todo used-in implementation checks + s"GET $apiUrl/{name}/{version}/json" should { "return 404" when { @@ -778,5 +768,149 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } // todo DELETE for v3: 404 for not-found entities (v2 for schemas returned 200) + s"PUT $apiUrl/{name}" can { + "return 200" when { + "a Schema with the given name exists" should { + "enable the Schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = true) + val schB = SchemaFactory.getDummySchema(name = "schB", version = 1, disabled = true) + schemaFixture.add(schA1, schA2, schB) + + val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions now enabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + + // unrelated schema unaffected + val responseB = sendGet[Schema](s"$apiUrl/schB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe true + } + } + + "a Schema with the given name exists and there have mixed disabled states (historical)" should { + "enable all versions the schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) + schemaFixture.add(schA1, schA2) + + val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions enabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + } + } + } + + "return 404" when { + "no Schema with the given name exists" should { + "enable nothing" in { + val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/aSchema") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + val schemaV1 = SchemaFactory.getDummySchema(name = "schemaA", version = 1) + schemaFixture.add(schemaV1) + + val response = sendDelete[Validation](s"$apiUrl/schemaA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } + + // todo disable schema - all versions/one version/ check the usage to prevent from disabling + // todo used-in implementation checks + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a Schema with the given name exists" should { + "disable the schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2) + val schB = SchemaFactory.getDummySchema(name = "schB", version = 1) + schemaFixture.add(schA1, schA2, schB) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated schema unaffected + val responseB = sendGet[Schema](s"$apiUrl/schB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a Schema with the given name exists and there have mixed disabled states (historical)" should { + "disable all versions the schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) + schemaFixture.add(schA1, schA2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 404" when { + "no Schema with the given name exists" should { + "disable nothing" in { + val response = sendDeleteByAdmin[String](s"$apiUrl/aSchema") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + val schemaV1 = SchemaFactory.getDummySchema(name = "schemaA", version = 1) + schemaFixture.add(schemaV1) + + val response = sendDelete[Validation](s"$apiUrl/schemaA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } } From cdbe7aecdc428385c315b669fe3411630bdcd40d Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 27 Apr 2022 14:42:56 +0200 Subject: [PATCH 40/67] #1693 VersionedModelControllerV3 - 2 levels of .../used-in exist ( /{name}/used-in & /{name}/{version}/used-in). The former is used in disable checking. IT cases added for Schema usedIns, Datasets with disabled entities --- .../PropertyDefinitionController.scala | 3 - .../controllers/RestExceptionHandler.scala | 9 +- .../v3/VersionedModelControllerV3.scala | 28 ++- .../EndpointDisabledException.scala | 20 ++ ...ed.scala => EntityDisabledException.scala} | 2 +- .../DatasetControllerV3IntegrationSuite.scala | 69 +++---- ...finitionControllerV3IntegrationSuite.scala | 2 + .../SchemaControllerV3IntegrationSuite.scala | 186 +++++++++++++++++- 8 files changed, 255 insertions(+), 64 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala rename rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/{EndpointDisabled.scala => EntityDisabledException.scala} (84%) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala index 11c26b063..c847ab54b 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala @@ -26,7 +26,6 @@ import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ -import za.co.absa.enceladus.rest_api.exceptions.EndpointDisabled import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService import za.co.absa.enceladus.model.ExportableObject import za.co.absa.enceladus.model.properties.PropertyDefinition @@ -63,8 +62,6 @@ class PropertyDefinitionController @Autowired()(propertyDefService: PropertyDefi val location: URI = new URI(s"/api/properties/datasets/${entity.name}/${entity.version}") ResponseEntity.created(location).body(entity) } - - // TODO: Location header would make sense for the underlying VersionedModelController.create, too. Issue #1611 } @GetMapping(Array("/{propertyName}")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala index 5ef13e650..b1fc91db7 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala @@ -62,11 +62,16 @@ class RestExceptionHandler { ResponseEntity.notFound().build[Any]() } - @ExceptionHandler(value = Array(classOf[EndpointDisabled])) - def handleEndpointDisabled(exception: EndpointDisabled): ResponseEntity[Any] = { + @ExceptionHandler(value = Array(classOf[EndpointDisabledException])) + def handleEndpointDisabled(exception: EndpointDisabledException): ResponseEntity[Any] = { ResponseEntity.status(HttpStatus.I_AM_A_TEAPOT).build[Any]() // Could change for LOCKED but I like this more } + @ExceptionHandler(value = Array(classOf[EntityDisabledException])) + def handleEntityDisabled(exception: EntityDisabledException): ResponseEntity[EntityDisabledException] = { + ResponseEntity.status(HttpStatus.BAD_REQUEST).body(exception) + } + @ExceptionHandler(value = Array(classOf[SchemaParsingException])) def handleBadRequestException(exception: SchemaParsingException): ResponseEntity[Any] = { val response = RestResponse(exception.message, Option(SchemaParsingError.fromException(exception))) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index b35f2f735..54f27803e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -27,7 +27,7 @@ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation} import za.co.absa.enceladus.rest_api.controllers.BaseController import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey -import za.co.absa.enceladus.rest_api.exceptions.NotFoundException +import za.co.absa.enceladus.rest_api.exceptions.{EntityDisabledException, NotFoundException, ValidationException} import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload import za.co.absa.enceladus.rest_api.services.VersionedModelService @@ -85,6 +85,12 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product forVersionExpression(name, version) { case (name, versionInt) => versionedModelService.getUsedIn(name, Some(versionInt)) } } + @GetMapping(Array("/{name}/used-in")) + @ResponseStatus(HttpStatus.OK) + def usedIn(@PathVariable name: String): CompletableFuture[UsedIn] = { + versionedModelService.getUsedIn(name, None) + } + @GetMapping(Array("/{name}/{version}/export")) @ResponseStatus(HttpStatus.OK) def exportSingleEntity(@PathVariable name: String, @PathVariable version: String): CompletableFuture[String] = { @@ -131,7 +137,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { versionedModelService.isDisabled(item.name).flatMap { isDisabled => if (isDisabled) { - versionedModelService.recreate(principal.getUsername, item) + Future.failed(EntityDisabledException(s"Entity ${item.name} is disabled. Enable it first (PUT) to push new versions (PUT).")) } else { versionedModelService.create(item, principal.getUsername) } @@ -155,10 +161,16 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } else if (version != item.version) { Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) } else { - versionedModelService.update(user.getUsername, item).map { - case Some((updatedEntity, validation)) => - createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) - case None => throw notFound() + versionedModelService.isDisabled(item.name).flatMap { isDisabled => + if (isDisabled) { + throw EntityDisabledException(s"Entity ${item.name} is disabled. Enable it first to create new versions.") + } else { + versionedModelService.update(user.getUsername, item).map { + case Some((updatedEntity, validation)) => + createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) + case None => throw notFound() + } + } } } } @@ -168,7 +180,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PreAuthorize("@authConstants.hasAdminRole(authentication)") def enable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { versionedModelService.enableEntity(name).map { updateResult => // always enabling all version of the entity - if(updateResult.getMatchedCount > 0) { + if (updateResult.getMatchedCount > 0) { DisabledPayload(disabled = false) } else { throw NotFoundException(s"No versions for entity $name found to be enabled.") @@ -181,7 +193,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PreAuthorize("@authConstants.hasAdminRole(authentication)") def disable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { versionedModelService.disableVersion(name, None).map { updateResult => // always disabling all version of the entity - if(updateResult.getMatchedCount > 0) { + if (updateResult.getMatchedCount > 0) { DisabledPayload(disabled = true) } else { throw NotFoundException(s"No versions for entity $name found to be disabled.") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala new file mode 100644 index 000000000..62f11853a --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala @@ -0,0 +1,20 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.exceptions + +// todo, there is no usage, remove? +case class EndpointDisabledException(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabled.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala similarity index 84% rename from rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabled.scala rename to rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala index fb745d5df..54a770d9d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabled.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala @@ -15,5 +15,5 @@ package za.co.absa.enceladus.rest_api.exceptions -case class EndpointDisabled(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) +case class EntityDisabledException(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 08ceaba75..45dad8596 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -30,6 +30,7 @@ import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.VersionList import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.EntityDisabledException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @@ -87,29 +88,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA assert(actual == expected) } - "create a new version of Dataset" when { - "the dataset is disabled (i.e. all version are disabled)" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) - val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) - datasetFixture.add(dataset1, dataset2) - - val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create - val response = sendPostByAdmin[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) - assertCreated(response) - val locationHeaders = response.getHeaders.get("location").asScala - locationHeaders should have size 1 - val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. - - val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) - assertOk(response2) - - val actual = response2.getBody - val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) - - assert(actual == expected) - } - } } "return 400" when { @@ -135,6 +113,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val responseBody = response.getBody responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } + "disabled entity with the name already exists" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", disabled = true) + datasetFixture.add(dataset1) + + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("a new version attempt")) + val response = sendPostByAdmin[Dataset, EntityDisabledException](apiUrl, bodyOpt = Some(dataset2)) + + assertBadRequest(response) + response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") + } } "return 403" when { @@ -146,7 +135,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - // todo what to do if "the last dataset version is disabled"? } s"GET $apiUrl/{name}" should { @@ -348,6 +336,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA response2.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") } } + "entity is disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", disabled = true) + datasetFixture.add(dataset1) + + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("ds update")) + val response = sendPutByAdmin[Dataset, EntityDisabledException](s"$apiUrl/dummyDs/1", bodyOpt = Some(dataset2)) + + assertBadRequest(response) + response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") + } } "return 403" when { @@ -592,25 +591,15 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"GET $apiUrl/{name}/{version}/used-in" should { - "return 404" when { - "when the dataset of latest version does not exist" in { - val response = sendGet[String](s"$apiUrl/notFoundDataset/latest/used-in") - assertNotFound(response) - } - } - + s"GET $apiUrl/{name}/used-in" should { "return 404" when { - "when the dataset of name/version does not exist" in { + "when the dataset of name does not exist" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetA) - val response = sendGet[String](s"$apiUrl/notFoundDataset/1/used-in") + val response = sendGet[String](s"$apiUrl/notFoundDataset/used-in") assertNotFound(response) - - val response2 = sendGet[String](s"$apiUrl/datasetA/7/used-in") - assertNotFound(response2) } } @@ -619,7 +608,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetA) - val response = sendGet[UsedIn](s"$apiUrl/datasetA/latest/used-in") + val response = sendGet[UsedIn](s"$apiUrl/datasetA/used-in") assertOk(response) response.getBody shouldBe UsedIn(None, None) @@ -627,11 +616,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "return 200" when { - "for existing name+version for dataset" in { + "for existing name for dataset" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2) datasetFixture.add(dataset2) - val response = sendGet[UsedIn](s"$apiUrl/dataset/2/used-in") + val response = sendGet[UsedIn](s"$apiUrl/dataset/used-in") assertOk(response) response.getBody shouldBe UsedIn(None, None) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 63670c4b9..ad3c17255 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -127,4 +127,6 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w } } } + + // todo used-in checks } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 07aff8bf8..23cb1c7ac 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -27,8 +27,9 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.{HttpStatus, MediaType} import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, SchemaFactory} -import za.co.absa.enceladus.model.{Schema, SchemaField, Validation} +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} +import za.co.absa.enceladus.model.{Schema, SchemaField, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.{DisabledPayload, RestResponse} @@ -63,13 +64,21 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn @Autowired private val schemaFixture: SchemaFixtureService = null + @Autowired + private val datasetFixture: DatasetFixtureService = null + + @Autowired + private val mappingTableFixture: MappingTableFixtureService = null + @Autowired private val attachmentFixture: AttachmentFixtureService = null private val apiUrl = "/schemas" private val schemaRefCollection = RefCollection.SCHEMA.name().toLowerCase() - override def fixtures: List[FixtureService[_]] = List(schemaFixture, attachmentFixture) + override def fixtures: List[FixtureService[_]] = List(schemaFixture, attachmentFixture, datasetFixture, mappingTableFixture) + + // todo the disabled state preventing post/put from creating a new entity => get rid of recreate s"POST $apiUrl" can { "return 201" when { @@ -185,7 +194,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } - s"GET $apiUrl/{name}/{version}/json" should { "return 404" when { "no schema exists for the specified name" in { @@ -204,7 +212,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn assertNotFound(response) } - "the schema has no fields" in { // todo 404 or 400 failed valiadation??? + "the schema has no fields" in { val schema = SchemaFactory.getDummySchema(name = "schemaA", version = 1) schemaFixture.add(schema) @@ -767,7 +775,67 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } - // todo DELETE for v3: 404 for not-found entities (v2 for schemas returned 200) + s"GET $apiUrl/{name}/used-in" should { + "return 200" when { + "there are used-in records" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", schemaName = "schema", schemaVersion = 1) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", schemaName = "schema", schemaVersion = 1, disabled = true) + datasetFixture.add(datasetA, datasetB) + + val mappingTableA = MappingTableFactory.getDummyMappingTable(name = "mappingA", schemaName = "schema", schemaVersion = 1) + val mappingTableB = MappingTableFactory.getDummyMappingTable(name = "mappingB", schemaName = "schema", schemaVersion = 1, disabled = true) + val mappingTableC = MappingTableFactory.getDummyMappingTable(name = "mappingC", schemaName = "schema", schemaVersion = 2) + mappingTableFixture.add(mappingTableA, mappingTableB, mappingTableC) + + + val response = sendGet[UsedIn](s"$apiUrl/schema/used-in") + assertOk(response) + + // datasetB and mappingB are disabled -> not reported + // mappingC is reported, even though it schema is tied to schema-v2, because disabling is done on the whole entity in API v3 + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1))), + mappingTables = Some(Seq(MenasReference(None, "mappingA", 1), MenasReference(None, "mappingC", 1))) + ) + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 200" when { + "there are used-in records for particular version" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", schemaName = "schema", schemaVersion = 1) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", schemaName = "schema", schemaVersion = 1, disabled = true) + datasetFixture.add(datasetA, datasetB) + + val mappingTableA = MappingTableFactory.getDummyMappingTable(name = "mappingA", schemaName = "schema", schemaVersion = 1) + val mappingTableB = MappingTableFactory.getDummyMappingTable(name = "mappingB", schemaName = "schema", schemaVersion = 1, disabled = true) + val mappingTableC = MappingTableFactory.getDummyMappingTable(name = "mappingC", schemaName = "schema", schemaVersion = 2) + mappingTableFixture.add(mappingTableA, mappingTableB, mappingTableC) + + + val response = sendGet[UsedIn](s"$apiUrl/schema/1/used-in") + assertOk(response) + + // datasetB and mappingB are disabled -> not reported + // mappingC is tied to schema v2 -> not reported + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1))), + mappingTables = Some(Seq(MenasReference(None, "mappingA", 1))) + ) + } + } + } + + s"PUT $apiUrl/{name}" can { "return 200" when { "a Schema with the given name exists" should { @@ -839,9 +907,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } - // todo disable schema - all versions/one version/ check the usage to prevent from disabling - // todo used-in implementation checks - s"DELETE $apiUrl/{name}" can { "return 200" when { "a Schema with the given name exists" should { @@ -871,7 +936,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } - "a Schema with the given name exists and there have mixed disabled states (historical)" should { + "a Schema with the given name exists and there have mixed (historical) disabled states " should { "disable all versions the schema with the given name" in { val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) @@ -891,6 +956,107 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn responseA2.getBody.disabled shouldBe true } } + "the Schema is only used in disabled Datasets" should { + "disable the Schema" in { + val dataset = DatasetFactory.getDummyDataset(schemaName = "schema", schemaVersion = 1, disabled = true) + datasetFixture.add(dataset) + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schema") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schema/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schema/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the Schema is only used in disabled MappingTables" should { + "disable the Schema" in { + val mappingTable = MappingTableFactory.getDummyMappingTable(schemaName = "schema", schemaVersion = 1, disabled = true) + mappingTableFixture.add(mappingTable) + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schema") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schema/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schema/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 400" when { + "the Schema is used by an enabled Dataset" should { + "return a list of the entities the Schema is used in" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", schemaName = "schema", schemaVersion = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, schemaName = "schema", schemaVersion = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3", schemaName = "anotherSchema", schemaVersion = 8) // moot + val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", schemaName = "schema", schemaVersion = 2, disabled = true) + datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) + + val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") + + assertBadRequest(response) + response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), Some(Seq())) + } + } + "the Schema is used by a enabled MappingTable" should { + "return a list of the entities the Schema is used in" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mapping1", schemaName = "schema", schemaVersion = 1, disabled = false) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mapping2", schemaName = "schema", schemaVersion = 2, disabled = false) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") + assertBadRequest(response) + + response.getBody shouldBe UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) + } + } + "the Schema is used by combination of MT and DS" should { + "return a list of the entities the Schema is used in" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mapping1", schemaName = "schema", schemaVersion = 1, disabled = false) + mappingTableFixture.add(mappingTable1) + + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", schemaName = "schema", schemaVersion = 2) + datasetFixture.add(dataset2) + + val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") + assertBadRequest(response) + + response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset2", 1))), Some(Seq(MenasReference(None, "mapping1", 1)))) + } + } } "return 404" when { From be02b5d8f9ad2ae51add4eaf17d2182b22d89be4 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 28 Apr 2022 13:46:26 +0200 Subject: [PATCH 41/67] #1693 usedIn + delete for propertyDefinitions ref'd in Datasets + ITs test cases. --- .../PropertyDefinitionController.scala | 6 +- .../v3/PropertyDefinitionControllerV3.scala | 6 +- .../VersionedMongoRepository.scala | 12 ++ .../rest_api/services/DatasetService.scala | 9 +- .../services/PropertyDefinitionService.scala | 6 +- .../rest_api/services/StatisticsService.scala | 4 +- .../services/v3/DatasetServiceV3.scala | 8 +- .../services/v3/HavingSchemaService.scala | 2 +- .../services/v3/MappingTableServiceV3.scala | 2 +- .../v3/PropertyDefinitionServiceV3.scala | 40 ++++ ...ingTableControllerV3IntegrationSuite.scala | 2 + ...finitionControllerV3IntegrationSuite.scala | 180 +++++++++++++++++- .../SchemaControllerV3IntegrationSuite.scala | 2 - 13 files changed, 251 insertions(+), 28 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala index c847ab54b..24dc896bc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala @@ -38,15 +38,15 @@ import scala.concurrent.ExecutionContext.Implicits.global */ @RestController @RequestMapping(path = Array("/api/properties/datasets"), produces = Array("application/json")) -class PropertyDefinitionController @Autowired()(propertyDefService: PropertyDefinitionService) - extends VersionedModelController(propertyDefService) { +class PropertyDefinitionController @Autowired()(propertyDefinitionService: PropertyDefinitionService) + extends VersionedModelController(propertyDefinitionService) { import za.co.absa.enceladus.rest_api.utils.implicits._ @GetMapping(Array("")) def getAllDatasetProperties(): CompletableFuture[Seq[PropertyDefinition]] = { logger.info("retrieving all dataset properties in full") - propertyDefService.getLatestVersions() + propertyDefinitionService.getLatestVersions() } @PostMapping(Array("")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala index e134fe716..c3fd01efe 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala @@ -17,11 +17,11 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.web.bind.annotation._ -import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService +import za.co.absa.enceladus.rest_api.services.v3.PropertyDefinitionServiceV3 @RestController @RequestMapping(path = Array("/api-v3/property-definitions/datasets"), produces = Array("application/json")) -class PropertyDefinitionControllerV3 @Autowired()(propertyDefService: PropertyDefinitionService) - extends VersionedModelControllerV3(propertyDefService) +class PropertyDefinitionControllerV3 @Autowired()(propertyDefinitionService: PropertyDefinitionServiceV3) + extends VersionedModelControllerV3(propertyDefinitionService) // super-class implementation is sufficient diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 86971d785..74ee5af44 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -175,6 +175,18 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab .toFuture() } + def findRefContainedAsKey(refNameCol: String, name: String): Future[Seq[MenasReference]] = { + + // `refNameCol` contains a map where the `name` is the key, so this is e.g. {"properties.keyName" : {$exists : true}} + val filter = Filters.and(getNotDisabledFilter, Filters.exists(s"$refNameCol.$name", true)) + + collection + .find[MenasReference](filter) + .projection(fields(include("name", "version"), computed("collection", collectionBaseName))) + .sort(Sorts.ascending("name", "version")) + .toFuture() + } + private def collectLatestVersions(postAggFilter: Option[Bson]): Future[Seq[C]] = { val pipeline = Seq( filter(Filters.notEqual("disabled", true)), diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 1b930e111..da0d5f3ac 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -17,8 +17,7 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.rest_api.repositories.DatasetMongoRepository -import za.co.absa.enceladus.rest_api.repositories.OozieRepository +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository, PropertyDefinitionMongoRepository} import za.co.absa.enceladus.rest_api.services.DatasetService.RuleValidationsAndFields import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, _} import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance @@ -39,7 +38,7 @@ import scala.util.{Failure, Success} @Service class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, - datasetPropertyDefinitionService: PropertyDefinitionService) + propertyDefinitionService: PropertyDefinitionService) extends VersionedModelService(datasetMongoRepository) { import scala.concurrent.ExecutionContext.Implicits.global @@ -224,7 +223,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository def validateProperties(properties: Map[String, String], forRun: Boolean = false): Future[Validation] = { - datasetPropertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => + propertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => val propDefsMap = Map(propDefs.map { propDef => (propDef.name, propDef) }: _*) // map(key, propDef) val existingPropsValidation = properties.toSeq.map { case (key, value) => validateExistingProperty(key, value, propDefsMap) } @@ -236,7 +235,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } def filterProperties(properties: Map[String, String], filter: PropertyDefinition => Boolean): Future[Map[String, String]] = { - datasetPropertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => + propertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => val filteredPropDefNames = propDefs.filter(filter).map(_.name).toSet properties.filterKeys(filteredPropDefNames.contains) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala index 7b4e5a039..cb5bc9b08 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala @@ -23,13 +23,15 @@ import za.co.absa.enceladus.model.properties.PropertyDefinition import scala.concurrent.Future -@Service +@Service("propertyDefinitionService") // by-name qualifier: V2 implementations use the base implementation, not v3 class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: PropertyDefinitionMongoRepository) extends VersionedModelService(propertyDefMongoRepository) { import scala.concurrent.ExecutionContext.Implicits.global - override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = Future.successful(UsedIn()) + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + Future.successful(UsedIn()) + } override def update(username: String, propertyDef: PropertyDefinition): Future[Option[(PropertyDefinition, Validation)]] = { super.update(username, propertyDef.name, propertyDef.version) { latest => diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala index 2af9a0c1e..878fdb85f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala @@ -23,10 +23,10 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @Component -class StatisticsService @Autowired() (propertyDefService: PropertyDefinitionService, datasetService: DatasetService){ +class StatisticsService @Autowired() (propertyDefinitionService: PropertyDefinitionService, datasetService: DatasetService){ //#TODO find optimizations #1897 def getPropertiesWithMissingCount(): Future[Seq[PropertyDefinitionStats]] = { - val propertyDefsFuture = propertyDefService.getLatestVersions() + val propertyDefsFuture = propertyDefinitionService.getLatestVersions() propertyDefsFuture .map { (props: Seq[PropertyDefinition]) => val propertiesWithMissingCounts: Seq[Future[PropertyDefinitionStats]] = props.map(propertyDef => diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index 10078eda4..933721b2a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -28,10 +28,10 @@ import scala.concurrent.Future @Service class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, - datasetPropertyDefinitionService: PropertyDefinitionService, - mappingTableService: MappingTableService, - val schemaService: SchemaService) - extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) + propertyDefinitionService: PropertyDefinitionServiceV3, + mappingTableService: MappingTableServiceV3, + val schemaService: SchemaServiceV3) + extends DatasetService(datasetMongoRepository, oozieRepository, propertyDefinitionService) with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala index 498721d8b..cb8104e15 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala @@ -21,7 +21,7 @@ import za.co.absa.enceladus.rest_api.services.SchemaService import scala.concurrent.{ExecutionContext, Future} trait HavingSchemaService { - protected def schemaService: SchemaService + protected def schemaService: SchemaServiceV3 def validateSchemaExists(schemaName: String, schemaVersion: Int) (implicit executionContext: ExecutionContext): Future[Validation] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala index a97e9e11a..9777c6227 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala @@ -26,7 +26,7 @@ import scala.concurrent.Future @Service class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, - val schemaService: SchemaService) + val schemaService: SchemaServiceV3) extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala new file mode 100644 index 000000000..ce3254417 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.{UsedIn, Validation} +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, PropertyDefinitionMongoRepository} +import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService + +import scala.concurrent.Future + +@Service +class PropertyDefinitionServiceV3 @Autowired()(propertyDefMongoRepository: PropertyDefinitionMongoRepository, + datasetMongoRepository: DatasetMongoRepository) + extends PropertyDefinitionService(propertyDefMongoRepository) { + + import scala.concurrent.ExecutionContext.Implicits.global + + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + for { + usedInD <- datasetMongoRepository.findRefContainedAsKey("properties", name) + } yield UsedIn(Some(usedInD), Some(Seq.empty)) + } + +} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index 58e4aa744..aebba6f7d 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -254,4 +254,6 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } + // todo used in & deletes for ties within datasets via MappingCR + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index ad3c17255..b4f2eca93 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -23,12 +23,14 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.Validation +import za.co.absa.enceladus.model.{UsedIn, Validation} +import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.properties.PropertyDefinition -import za.co.absa.enceladus.model.properties.propertyType.StringPropertyType -import za.co.absa.enceladus.model.test.factories.PropertyDefinitionFactory +import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -38,10 +40,13 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + @Autowired + private val datasetFixture: DatasetFixtureService = null + private val apiUrl = "/property-definitions/datasets" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(propertyDefinitionFixture) + override def fixtures: List[FixtureService[_]] = List(propertyDefinitionFixture, datasetFixture) private def minimalPdCreatePayload(name: String, suggestedValue: Option[String]) = { @@ -128,5 +133,170 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w } } - // todo used-in checks + s"GET $apiUrl/{name}/used-in" should { + "return 200" when { + "there are used-in records" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 1) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 2, description = Some("An update")) + val propDefB = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propB", version = 1) // moot + propertyDefinitionFixture.add(propDefA1, propDefA2, propDefB) + + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("propA" -> "something"))) + val datasetB1 = DatasetFactory.getDummyDataset(name = "datasetB", properties = Some(Map("propA" -> "something")), disabled = true) + val datasetC1 = DatasetFactory.getDummyDataset(name = "datasetC", properties = Some(Map("propA" -> "something else"))) + datasetFixture.add(datasetA1, datasetB1, datasetC1) + + val response = sendGet[UsedIn](s"$apiUrl/propA/used-in") + assertOk(response) + + // propDefB is moot. + // datasetB is not reported, because it is disabled + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), + mappingTables = Some(Seq()) + ) + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 200" when { + "there are used-in records for particular version" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 1) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 2, description = Some("An update")) + propertyDefinitionFixture.add(propDefA1, propDefA2) + + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("propA" -> "something"))) + val datasetB1 = DatasetFactory.getDummyDataset(name = "datasetB", properties = Some(Map("propA" -> "something")), disabled = true) + val datasetC1 = DatasetFactory.getDummyDataset(name = "datasetC", properties = Some(Map("propA" -> "something else"))) + datasetFixture.add(datasetA1, datasetB1, datasetC1) + + val response = sendGet[UsedIn](s"$apiUrl/propA/1/used-in") + assertOk(response) + + // same outcome as $apiUrl/{name}/used-in above -- because propDefs are not tied by version to datasets + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), + mappingTables = Some(Seq()) + ) + } + } + } + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a PropertyDefinition with the given name exists" should { + "disable the propertyDefinition with the given name" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 1) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 2) + val propDefB = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefB", version = 1) + propertyDefinitionFixture.add(propDefA1, propDefA2, propDefB) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/propDefA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated propDef unaffected + val responseB = sendGet[PropertyDefinition](s"$apiUrl/propDefB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a PropertyDefinition with the given name exists and there have mixed (historical) disabled states " should { + "disable all versions the propertyDefinition with the given name" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 1, disabled = true) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 2, disabled = false) + propertyDefinitionFixture.add(propDefA1, propDefA2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/propDefA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the PropertyDefinition is only used in disabled Datasets" should { + "disable the PropertyDefinition" in { + val propertyDefinition1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 1) + val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2) + propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) + + val dataset = DatasetFactory.getDummyDataset(disabled = true, properties = Some(Map("propertyDefinition" -> "value xyz"))) + datasetFixture.add(dataset) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/propertyDefinition") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinition/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinition/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 400" when { + "the PropertyDefinition is used by an enabled Dataset" should { + "return a list of the entities the PropertyDefinition is used in" in { + val propertyDefinition1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "keyA", version = 1) + val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "keyA", version = 2, propertyType = EnumPropertyType("x", "y", "z")) + val propertyDefinitionAsdf = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "keyASDF", version = 1) // moot support + propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2, propertyDefinitionAsdf) + + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", properties = Some(Map("keyA" -> "x"))) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, properties = Some(Map("keyA" -> "z"))) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3", properties = Some(Map("keyASDF" -> "ASDF"))) // moot + val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", properties = Some(Map("keyA" -> "x")), disabled = true) + datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) + + val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/keyA") + + assertBadRequest(response) + response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), Some(Seq())) + } + } + } + + "return 404" when { + "no PropertyDefinition with the given name exists" should { + "disable nothing" in { + val response = sendDeleteByAdmin[String](s"$apiUrl/aPropertyDefinition") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + val propertyDefinitionV1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinitionA", version = 1) + propertyDefinitionFixture.add(propertyDefinitionV1) + + val response = sendDelete[Validation](s"$apiUrl/propertyDefinitionA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 23cb1c7ac..855fd6a3b 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -78,8 +78,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn override def fixtures: List[FixtureService[_]] = List(schemaFixture, attachmentFixture, datasetFixture, mappingTableFixture) - // todo the disabled state preventing post/put from creating a new entity => get rid of recreate - s"POST $apiUrl" can { "return 201" when { "a Schema is created (v1-payload has defined fields already)" in { From 09c326e5f4e171a60b962865c744ab39c2ed5ce6 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 28 Apr 2022 14:32:02 +0200 Subject: [PATCH 42/67] #1693 PR review update 1 --- .../model/versionedModel/VersionList.scala | 18 ++++++++++++++++++ .../versionedModel/VersionedSummary.scala | 2 +- .../enceladus/rest_api/SpringFoxConfig.scala | 5 +++-- .../rest_api/services/DatasetService.scala | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala new file mode 100644 index 000000000..009a0bc5a --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.versionedModel + +case class VersionList(_id: String, versions: Seq[Int]) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index e9e931d59..d8ab45f16 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -17,4 +17,4 @@ package za.co.absa.enceladus.model.versionedModel case class VersionedSummary(_id: String, latestVersion: Int) -case class VersionList(_id: String, versions: Seq[Int]) + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala index b5980e65c..d57f4b30e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala @@ -43,12 +43,13 @@ class SpringFoxConfig extends ProjectMetadata { // api v2 regex("/api/dataset.*"), regex("/api/schema.*"), regex("/api/mappingTable.*"), regex("/api/properties.*"), - regex("/api/monitoring.*"),regex("/api/runs.*"), + regex("/api/monitoring.*"), regex("/api/runs.*"), regex("/api/user.*"), regex("/api/spark.*"), regex("/api/configuration.*"), // api v3 - regex("/api-v3/datasets.*") + regex("/api-v3/datasets.*"), regex("/api-v3/schemas.*"), + regex("/api-v3/mapping-tables.*"), regex("/api-v3/property-definitions.*") ) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 1b930e111..c852eebfc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -465,7 +465,7 @@ object DatasetService { * @param properties original properties * @return properties without empty-string value entries */ - def removeBlankProperties(properties: Map[String, String]): Map[String, String] = { + private def removeBlankProperties(properties: Map[String, String]): Map[String, String] = { properties.filter { case (_, propValue) => propValue.nonEmpty } } From 8baf2d336e3739a5b4eaf84477b05901b7d960f6 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 29 Apr 2022 10:09:56 +0200 Subject: [PATCH 43/67] #1693 PR review update: SchemaService(V3) update/create behavior is extensible via `updateFields` method - V3 uses this to reflect schema fields (actual name of the field :D). --- .../rest_api/services/SchemaService.scala | 26 ++++++--- .../services/v3/SchemaServiceV3.scala | 54 ++----------------- 2 files changed, 24 insertions(+), 56 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index 9ae9b166f..913cfa594 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -56,15 +56,29 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, } yield update } - override def update(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { - super.update(username, schema.name, schema.version) { latest => - latest.setDescription(schema.description).asInstanceOf[Schema] + /** + * This method applies only certain fields from `updateSchema` to the subject of this method. Here, for V2 API, + * only description field is applied, all other fields are disregarded - internally called at create/update + * @param current existing latest schema prior to changes + * @param update schema with create/update fields information + * @return + */ + protected def updateFields(current: Schema, update: Schema) : Schema = { + current.setDescription(update.description).asInstanceOf[Schema] + } + + /** final - override `updateWithFields` if needed */ + final override def update(username: String, update: Schema): Future[Option[(Schema, Validation)]] = { + super.update(username, update.name, update.version) { latest => + updateFields(latest, update) } } - override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { - val schema = Schema(name = newSchema.name, - description = newSchema.description) + /** final - override `updateWithFields` if needed */ + final override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { + val initSchema = Schema(name = newSchema.name, description = newSchema.description) + + val schema = updateFields(initSchema, newSchema) super.create(schema, username) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala index 59d7686bf..1622a7b6b 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala @@ -32,22 +32,7 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, sparkMenasConvertor: SparkMenasSchemaConvertor) - extends VersionedModelService(schemaMongoRepository) { - - // same as v2 - override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { - for { - usedInD <- datasetMongoRepository.findRefEqual("schemaName", "schemaVersion", name, version) - usedInM <- mappingTableMongoRepository.findRefEqual("schemaName", "schemaVersion", name, version) - } yield UsedIn(Some(usedInD), Some(usedInM)) - } - - // same as v2 - def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[(Schema, Validation)] = { - super.update(username, schemaName, schemaVersion)({ oldSchema => - oldSchema.copy(fields = sparkMenasConvertor.convertSparkToMenasFields(fields.fields).toList) - }).map(_.getOrElse(throw new IllegalArgumentException("Failed to derive new schema from file!"))) - } + extends SchemaService(schemaMongoRepository, mappingTableMongoRepository, datasetMongoRepository, sparkMenasConvertor) { override def validate(item: Schema): Future[Validation] = { if (item.fields.isEmpty) { @@ -58,39 +43,8 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, } } - // same as V2, but fields from payload are used, too - override def update(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { - super.update(username, schema.name, schema.version) { latest => - latest.setDescription(schema.description).asInstanceOf[Schema].copy(fields = schema.fields) - } - } - - // same as V2, but fields from payload are used, too - override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { - val schema = Schema(name = newSchema.name, - description = newSchema.description, - fields = newSchema.fields - ) - super.create(schema, username) - } - - // same as v2 - override def recreate(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { - for { - latestVersion <- getLatestVersionNumber(schema.name) - update <- super.update(username, schema.name, latestVersion) { latest => - latest - .copy(fields = List()) - .setDescription(schema.description).asInstanceOf[Schema] - } - } yield update - } - - // same as v2 - override def importItem(item: Schema, username: String): Future[Option[(Schema, Validation)]] = { - getLatestVersionValue(item.name).flatMap { - case Some(version) => update(username, item.copy(version = version)) - case None => super.create(item.copy(version = 1), username) - } + // V3 applies fields on create/update from the payload, too (V2 did not allow fields payload here, only via 'upload' + override protected def updateFields(current: Schema, update: Schema) : Schema = { + current.setDescription(update.description).asInstanceOf[Schema].copy(fields = update.fields) } } From a4740548093c4cfa811632c9815bc902b2071058 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 29 Apr 2022 10:20:40 +0200 Subject: [PATCH 44/67] #1693 cleanup --- .../enceladus/rest_api/services/v3/DatasetServiceV3.scala | 2 +- .../enceladus/rest_api/services/v3/SchemaServiceV3.scala | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index 933721b2a..3ee37ade5 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -20,7 +20,7 @@ import org.springframework.stereotype.Service import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} -import za.co.absa.enceladus.rest_api.services.{DatasetService, MappingTableService, PropertyDefinitionService, SchemaService} +import za.co.absa.enceladus.rest_api.services.DatasetService import scala.concurrent.Future diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala index 1622a7b6b..2d8b6bbaf 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala @@ -15,15 +15,12 @@ package za.co.absa.enceladus.rest_api.services.v3 -import org.apache.spark.sql.types.StructType import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} +import za.co.absa.enceladus.model.{Schema, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} -import za.co.absa.enceladus.rest_api.services.{SchemaService, VersionedModelService} +import za.co.absa.enceladus.rest_api.services.SchemaService import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor -import scala.concurrent.ExecutionContext.Implicits.global - import scala.concurrent.Future From c50a02447213203aac4918d93232ceaf784375c5 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 29 Apr 2022 15:09:15 +0200 Subject: [PATCH 45/67] #1693 MT in MCR check - usedIn fix (originally from JS). IT test-cases for usedIn and delete added. Custom alternative: ```scala // /** // * Finds references of MappingTable in Dataset's mapping conformance rules // * @param mtName // * @return // */ // def findRefByMappingTableInMappingConformanceRules(mtName: String, version: Option[Int]): Future[Seq[MenasReference]] = { // /* The gist of the find query that this method is based on; testable in a mongo client // { $and : [ // {...}, // non disabled // {"conformance": {"$elemMatch": {$and : [ // {"_t": "MappingConformanceRule"}, // {"mappingTable": "AnotherAwesomeMappingTable"} // mappingTableVersion not needed for match, but will be reported as part of the MenasReference // ]}}} // ]} // */ // val versionFilter = version.fold(Filters.and() /*empty filter*/ )(ver => Filters.eq("mappingTableVersion", ver)) // // val filter: Bson = Filters.and( // getNotDisabledFilter, // Filters.elemMatch("conformance", Filters.and( // Filters.eq("_t", "MappingConformanceRule"), // Filters.eq("mappingTable", mtName), // versionFilter // )) // ) // // collection // .find[MenasReference](filter) // .projection(fields(include("name", "version"), computed("collection", collectionBaseName))) // .sort(Sorts.ascending("name", "version")) // .toFuture() // } ``` --- .../repositories/DatasetMongoRepository.scala | 28 ++- ...ingTableControllerV3IntegrationSuite.scala | 204 +++++++++++++++++- .../SchemaControllerV3IntegrationSuite.scala | 1 - 3 files changed, 216 insertions(+), 17 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala index cf8c83a43..240d09752 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala @@ -15,18 +15,17 @@ package za.co.absa.enceladus.rest_api.repositories +import org.mongodb.scala.model.Projections._ +import org.mongodb.scala.model.{Filters, Sorts} import org.mongodb.scala.{Completed, MongoDatabase} import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository -import za.co.absa.enceladus.model.{Dataset, MappingTable, Schema} - -import scala.reflect.ClassTag -import za.co.absa.enceladus.model.menas.MenasReference -import org.mongodb.scala.model.Filters -import org.mongodb.scala.model.Projections._ import za.co.absa.enceladus.model +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.{Dataset, MappingTable, Schema} import scala.concurrent.Future +import scala.reflect.ClassTag object DatasetMongoRepository { val collectionBaseName: String = "dataset" @@ -77,13 +76,24 @@ class DatasetMongoRepository @Autowired()(mongoDb: MongoDatabase) * @return List of Menas references to Datasets, which contain the relevant conformance rules */ def containsMappingRuleRefEqual(refColVal: (String, Any)*): Future[Seq[MenasReference]] = { - - val equals = Filters.and(refColVal.map(col => Filters.eq(col._1, col._2)) :_*) - val filter = Filters.elemMatch("conformance", equals) + // The gist of the find query that this method is based on; testable in a mongo client + // { $and : [ + // {... non disabled filter here...}, + // {"conformance": {"$elemMatch": {$and : [ + // {"mappingTable": "AnotherAwesomeMappingTable"}, // from refColVal + // {"mappingTableVersion": 1} // from refColVal + // ]}}} + // ]} + + val equalConditionsFilter = Filters.and(refColVal.map { + case (key, value) => Filters.eq(key, value) + } :_*) + val filter = Filters.and(getNotDisabledFilter, Filters.elemMatch("conformance", equalConditionsFilter)) collection .find[MenasReference](filter) .projection(fields(include("name", "version"), computed("collection", collectionBaseName))) + .sort(Sorts.ascending("name", "version")) .toFuture() } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index aebba6f7d..cbb4a823f 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -23,12 +23,14 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.properties.PropertyDefinition -import za.co.absa.enceladus.model.{DefaultValue, MappingTable, Validation} -import za.co.absa.enceladus.model.test.factories.{MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.rest_api.integration.controllers.BaseRestApiTestV3 +import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule +import za.co.absa.enceladus.model.dataFrameFilter._ +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, SchemaFactory} +import za.co.absa.enceladus.model.{DefaultValue, MappingTable, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ -import za.co.absa.enceladus.rest_api.integration.controllers.toExpected +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -41,10 +43,13 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be @Autowired private val schemaFixture: SchemaFixtureService = null + @Autowired + private val datasetFixture: DatasetFixtureService = null + private val apiUrl = "/mapping-tables" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(mappingTableFixture, schemaFixture) + override def fixtures: List[FixtureService[_]] = List(mappingTableFixture, schemaFixture, datasetFixture) s"POST $apiUrl" should { "return 400" when { @@ -254,6 +259,191 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } - // todo used in & deletes for ties within datasets via MappingCR + private def mcr(mtName: String, mtVersion: Int, index: Int = 0) = MappingConformanceRule(index, + controlCheckpoint = true, + mappingTable = mtName, + mappingTableVersion = mtVersion, + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + + s"GET $apiUrl/{name}/used-in" should { + "return 200" when { + "there are used-in records" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable",1))) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable",1)), disabled = true) + val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) + datasetFixture.add(datasetA, datasetB, datasetC) + + val response = sendGet[UsedIn](s"$apiUrl/mappingTable/used-in") + assertOk(response) + + // datasetB is disabled -> not reported + // datasetC is reported, because this is a version-less check + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), + mappingTables = None + ) + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 200" when { + "there are used-in records for particular version" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable",1))) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable",1)), disabled = true) + val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) + datasetFixture.add(datasetA, datasetB, datasetC) + + val response = sendGet[UsedIn](s"$apiUrl/mappingTable/1/used-in") + assertOk(response) + + // datasetB is disabled -> not reported + // datasetC is not reported, because it depends on v2 of the MT + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1))), + mappingTables = None // todo this unfortunately differs from other usedIns, they give Some(Seq.empty) + ) + } + } + } + + // todo deletes for ties within datasets via MappingCR + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a MappingTable with the given name exists" should { + "disable the mappingTable with the given name" in { + val mtA1 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 1) + val mtA2 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 2) + val mtB = MappingTableFactory.getDummyMappingTable(name = "mtB", version = 1) + mappingTableFixture.add(mtA1, mtA2, mtB) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/mtA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[MappingTable](s"$apiUrl/mtA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[MappingTable](s"$apiUrl/mtA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated mappingTable unaffected + val responseB = sendGet[MappingTable](s"$apiUrl/mtB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a MappingTable with the given name exists and there have mixed (historical) disabled states " should { + "disable all versions the mappingTable with the given name" in { + val mtA1 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 1, disabled = true) + val mtA2 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 2, disabled = false) + mappingTableFixture.add(mtA1, mtA2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/mtA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[MappingTable](s"$apiUrl/mtA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[MappingTable](s"$apiUrl/mtA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the MappingTable is only used in disabled Datasets" should { + "disable the MappingTable" in { + val dataset = DatasetFactory.getDummyDataset(conformance = List(mcr("mappingTable", 1)), disabled = true) + datasetFixture.add(dataset) + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/mappingTable") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[MappingTable](s"$apiUrl/mappingTable/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[MappingTable](s"$apiUrl/mappingTable/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 400" when { + "the MappingTable is used by an enabled Dataset" should { + "return a list of the entities the MappingTable is used in" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", conformance = List(mcr("mappingTable", 1))) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, conformance = List(mcr("mappingTable", 2))) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3",conformance = List(mcr("anotherMappingTable", 8))) // moot + val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", conformance = List(mcr("mappingTable", 2)), disabled = true) + datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) + + val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/mappingTable") + + assertBadRequest(response) + response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + } + } + } + + "return 404" when { + "no MappingTable with the given name exists" should { + "disable nothing" in { + val response = sendDeleteByAdmin[String](s"$apiUrl/aMappingTable") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + val mappingTableV1 = MappingTableFactory.getDummyMappingTable(name = "mappingTableA", version = 1) + mappingTableFixture.add(mappingTableV1) + + val response = sendDelete[Validation](s"$apiUrl/mappingTableA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 855fd6a3b..247b11b05 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -833,7 +833,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } - s"PUT $apiUrl/{name}" can { "return 200" when { "a Schema with the given name exists" should { From 32e48d0f40cd896801f29cee4d975e11906ed89c Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 29 Apr 2022 16:43:31 +0200 Subject: [PATCH 46/67] #1693 UsedIn - empty/nonEmpty normalization + UnitTest. V2 alternated between None and Some(Seq.empty) in various places, V3 consistently returns None for no references groups -> reflected in V3 ITs. + cleanup --- .../za/co/absa/enceladus/model/UsedIn.scala | 39 ++++++++-- .../co/absa/enceladus/model/UsedInTest.scala | 73 +++++++++++++++++++ .../services/v3/DatasetServiceV3.scala | 6 +- .../services/v3/MappingTableServiceV3.scala | 4 + .../v3/PropertyDefinitionServiceV3.scala | 3 +- .../services/v3/SchemaServiceV3.scala | 9 ++- .../DatasetControllerV3IntegrationSuite.scala | 2 - ...ingTableControllerV3IntegrationSuite.scala | 16 +++- ...finitionControllerV3IntegrationSuite.scala | 6 +- .../SchemaControllerV3IntegrationSuite.scala | 5 +- 10 files changed, 144 insertions(+), 19 deletions(-) create mode 100644 data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala index 02e1c6cfd..decf6c23a 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala @@ -15,15 +15,42 @@ package za.co.absa.enceladus.model +import com.fasterxml.jackson.annotation.JsonIgnore import za.co.absa.enceladus.model.menas.MenasReference -case class UsedIn( - datasets: Option[Seq[MenasReference]] = None, - mappingTables: Option[Seq[MenasReference]] = None -) { +case class UsedIn(datasets: Option[Seq[MenasReference]] = None, + mappingTables: Option[Seq[MenasReference]] = None) { - def nonEmpty: Boolean = { - datasets.exists(_.nonEmpty) || mappingTables.exists(_.nonEmpty) + /** + * Should any of the original UsedIn equal to Some(Seq.empty), it will be None after normalization + */ + @JsonIgnore + lazy val normalized: UsedIn = { + def normalizeOne(field: Option[Seq[MenasReference]]) = field match { + case None => None + case Some(x) if x.isEmpty => None + case otherNonEmpty => otherNonEmpty + } + + val normalizedDs = normalizeOne(datasets) + val normalizedMt = normalizeOne(mappingTables) + + (normalizedDs, normalizedMt) match { + case (`datasets`, `mappingTables`) => this // no normalization needed + case _ => UsedIn(normalizedDs, normalizedMt) + } + } + + val isEmpty: Boolean = { + normalized.datasets == None && normalized.mappingTables == None } + val nonEmpty: Boolean = !isEmpty +} + +object UsedIn { + /** + * Normalized + */ + val empty = UsedIn(None, None) } diff --git a/data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala b/data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala new file mode 100644 index 000000000..6b4e72551 --- /dev/null +++ b/data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import za.co.absa.enceladus.model.menas.MenasReference + +class UsedInTest extends AnyFlatSpec with Matchers { + + private val exampleRef = MenasReference(Some("collection1"), "entity1", 1) + + "UsedIn" should "correctly evaluate .nonEmpty" in { + UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).nonEmpty shouldBe true + UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).nonEmpty shouldBe true + UsedIn(Some(Seq(exampleRef)), None).nonEmpty shouldBe true + + UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).nonEmpty shouldBe true + UsedIn(None, Some(Seq(exampleRef))).nonEmpty shouldBe true + + UsedIn(Some(Seq.empty), Some(Seq.empty)).nonEmpty shouldBe false + UsedIn(None, Some(Seq.empty)).nonEmpty shouldBe false + UsedIn(Some(Seq.empty), None).nonEmpty shouldBe false + UsedIn(None, None).nonEmpty shouldBe false + } + + it should "correctly evaluate .empty" in { + UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).isEmpty shouldBe false + UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).isEmpty shouldBe false + UsedIn(Some(Seq(exampleRef)), None).isEmpty shouldBe false + + UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).isEmpty shouldBe false + UsedIn(None, Some(Seq(exampleRef))).isEmpty shouldBe false + + UsedIn(Some(Seq.empty), Some(Seq.empty)).isEmpty shouldBe true + UsedIn(None, Some(Seq.empty)).isEmpty shouldBe true + UsedIn(Some(Seq.empty), None).isEmpty shouldBe true + UsedIn(None, None).isEmpty shouldBe true + } + + it should "normalize" in { + UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).normalized shouldBe UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))) + + UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).normalized shouldBe UsedIn(Some(Seq(exampleRef)), None) + UsedIn(Some(Seq(exampleRef)), None).normalized shouldBe UsedIn(Some(Seq(exampleRef)), None) + + UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).normalized shouldBe UsedIn(None, Some(Seq(exampleRef))) + UsedIn(None, Some(Seq(exampleRef))).normalized shouldBe UsedIn(None, Some(Seq(exampleRef))) + + UsedIn(Some(Seq.empty), Some(Seq.empty)).normalized shouldBe UsedIn(None, None) + UsedIn(None, None).normalized shouldBe UsedIn(None, None) + + + + + } + + + +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index 3ee37ade5..c804171f5 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -18,7 +18,7 @@ package za.co.absa.enceladus.rest_api.services.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} -import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} import za.co.absa.enceladus.rest_api.services.DatasetService @@ -73,6 +73,10 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito } } + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + super.getUsedIn(name, version).map(_.normalized) + } + } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala index 9777c6227..12fb346f8 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala @@ -36,6 +36,10 @@ class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTab originalValidation <- super.validate(item) mtSchemaValidation <- validateSchemaExists(item.schemaName, item.schemaVersion) } yield originalValidation.merge(mtSchemaValidation) + } + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + super.getUsedIn(name, version).map(_.normalized) } + } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala index ce3254417..bab32b4df 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala @@ -34,7 +34,8 @@ class PropertyDefinitionServiceV3 @Autowired()(propertyDefMongoRepository: Prope override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { for { usedInD <- datasetMongoRepository.findRefContainedAsKey("properties", name) - } yield UsedIn(Some(usedInD), Some(Seq.empty)) + optionalUsedInD = if (usedInD.isEmpty) None else Some(usedInD) + } yield UsedIn(optionalUsedInD, None) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala index 2d8b6bbaf..6dc6dae12 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala @@ -17,7 +17,7 @@ package za.co.absa.enceladus.rest_api.services.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.model.{Schema, Validation} +import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} import za.co.absa.enceladus.rest_api.services.SchemaService import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor @@ -31,6 +31,8 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, sparkMenasConvertor: SparkMenasSchemaConvertor) extends SchemaService(schemaMongoRepository, mappingTableMongoRepository, datasetMongoRepository, sparkMenasConvertor) { + import scala.concurrent.ExecutionContext.Implicits.global + override def validate(item: Schema): Future[Validation] = { if (item.fields.isEmpty) { // V3 disallows empty schema fields - V2 allowed it at first that to get updated by an attachment upload/remote-load @@ -44,4 +46,9 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, override protected def updateFields(current: Schema, update: Schema) : Schema = { current.setDescription(update.description).asInstanceOf[Schema].copy(fields = update.fields) } + + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + super.getUsedIn(name, version).map(_.normalized) + } + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 45dad8596..fd33e7d45 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -35,8 +35,6 @@ import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload -import scala.collection.JavaConverters._ - @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index cbb4a823f..b7140c5c9 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -324,13 +324,25 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be // datasetC is not reported, because it depends on v2 of the MT response.getBody shouldBe UsedIn( datasets = Some(Seq(MenasReference(None, "datasetA", 1))), - mappingTables = None // todo this unfortunately differs from other usedIns, they give Some(Seq.empty) + mappingTables = None + ) + } + + "there are no used-in records for particular version" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + mappingTableFixture.add(mappingTable1) + + val response = sendGet[UsedIn](s"$apiUrl/mappingTable/1/used-in") + assertOk(response) + + response.getBody shouldBe UsedIn( + datasets = None, + mappingTables = None ) } } } - // todo deletes for ties within datasets via MappingCR s"DELETE $apiUrl/{name}" can { "return 200" when { "a MappingTable with the given name exists" should { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index b4f2eca93..53e29e4e8 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -153,7 +153,7 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w // datasetB is not reported, because it is disabled response.getBody shouldBe UsedIn( datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), - mappingTables = Some(Seq()) + mappingTables = None ) } } @@ -177,7 +177,7 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w // same outcome as $apiUrl/{name}/used-in above -- because propDefs are not tied by version to datasets response.getBody shouldBe UsedIn( datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), - mappingTables = Some(Seq()) + mappingTables = None ) } } @@ -275,7 +275,7 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/keyA") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), Some(Seq())) + response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) } } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 247b11b05..fc77f4aed 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -36,7 +36,6 @@ import za.co.absa.enceladus.rest_api.models.rest.{DisabledPayload, RestResponse} import za.co.absa.enceladus.rest_api.models.rest.errors.{SchemaFormatError, SchemaParsingError} import za.co.absa.enceladus.rest_api.repositories.RefCollection import za.co.absa.enceladus.rest_api.utils.SchemaType -import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import za.co.absa.enceladus.restapi.TestResourcePath import java.io.File @@ -1017,7 +1016,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), Some(Seq())) + response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) } } "the Schema is used by a enabled MappingTable" should { @@ -1033,7 +1032,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) + response.getBody shouldBe UsedIn(None, Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) } } "the Schema is used by combination of MT and DS" should { From 1c241d17c826bb2814cdbf4f2f7d0c61191651ec Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 3 May 2022 13:30:55 +0200 Subject: [PATCH 47/67] #1693 DatasetService.updateProperties[V3] -> split between DatasetService and DatasetServiceV3 with renaming. VersionList removed in favor of VersionedSummary everywhere. - explanatory comments --- .../model/versionedModel/VersionList.scala | 18 ------------ .../controllers/DatasetController.scala | 2 +- .../VersionedModelController.scala | 2 +- .../v3/VersionedModelControllerV3.scala | 6 ++-- .../VersionedMongoRepository.scala | 17 +++++++---- .../rest_api/services/DatasetService.scala | 29 +++++-------------- .../services/VersionedModelService.scala | 15 +++++----- .../services/v3/DatasetServiceV3.scala | 18 ++++++++++++ .../rest_api/utils/implicits/package.scala | 2 +- .../DatasetControllerV3IntegrationSuite.scala | 6 ++-- .../DatasetRepositoryIntegrationSuite.scala | 12 ++++---- .../services/DatasetServiceTest.scala | 2 +- 12 files changed, 60 insertions(+), 69 deletions(-) delete mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala deleted file mode 100644 index 009a0bc5a..000000000 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionList.scala +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2018 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.enceladus.model.versionedModel - -case class VersionList(_id: String, versions: Seq[Int]) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index dc11c0933..fcdd69e91 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -113,7 +113,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def replaceProperties(@AuthenticationPrincipal principal: UserDetails, @PathVariable datasetName: String, @RequestBody newProperties: Optional[Map[String, String]]): CompletableFuture[ResponseEntity[Option[Dataset]]] = { - datasetService.updatePropertiesV2(principal.getUsername, datasetName, newProperties.toScalaOption).map { + datasetService.updateProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map { case None => throw notFound() case Some(dataset) => val location: URI = new URI(s"/api/dataset/${dataset.name}/${dataset.version}") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index 87ffd7e72..e129a8b82 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -39,7 +39,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @GetMapping(Array("/list", "/list/{searchQuery}")) @ResponseStatus(HttpStatus.OK) def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { - versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption) + versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) } @GetMapping(Array("/searchSuggestions")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index b3ee9399d..a7d385d0e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -47,13 +47,13 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("")) @ResponseStatus(HttpStatus.OK) def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { - versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption) + versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) } @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionsList(@PathVariable name: String): CompletableFuture[VersionList] = { - versionedModelService.getAllVersionsValues(name) map { + def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[VersionedSummary] = { + versionedModelService.getLatestVersionSummary(name) map { case Some(entity) => entity case None => throw notFound() } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 51274b7ec..95f68b210 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -27,7 +27,7 @@ import org.mongodb.scala.model.Updates._ import org.mongodb.scala.model._ import org.mongodb.scala.result.UpdateResult import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionList} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} import scala.concurrent.Future import scala.reflect.ClassTag @@ -61,7 +61,7 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.distinct[String]("name", getNotDisabledFilter).toFuture().map(_.sorted) } - def getLatestVersionsSummary(searchQuery: Option[String] = None): Future[Seq[VersionedSummary]] = { + def getLatestVersionsSummarySearch(searchQuery: Option[String] = None): Future[Seq[VersionedSummary]] = { val searchFilter = searchQuery match { case Some(search) => Filters.regex("name", search, "i") case None => Filters.expr(true) @@ -84,21 +84,26 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.find(getNameVersionFilter(name, Some(version))).headOption() } - def getLatestVersionValue(name: String): Future[Option[Int]] = { + def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { val pipeline = Seq( filter(getNameFilter(name)), Aggregates.group("$name", Accumulators.max("latestVersion", "$version")) ) - collection.aggregate[VersionedSummary](pipeline).headOption().map(_.map(_.latestVersion)) + collection.aggregate[VersionedSummary](pipeline).headOption() } - def getAllVersionsValues(name: String): Future[Option[VersionList]] = { + def getLatestVersionValue(name: String): Future[Option[Int]] = { + getLatestVersionSummary(name).map(_.map(_.latestVersion)) + } + + def getAllVersionsValues(name: String): Future[Seq[Int]] = { val pipeline = Seq( filter(getNameFilter(name)), Aggregates.sort(Sorts.ascending("version")), Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array ) - collection.aggregate[VersionList](pipeline).headOption().map(_.map(vlist => VersionList("versions", vlist.versions))) + collection.aggregate[Seq[Int]](pipeline).headOption().map(_.getOrElse(Seq.empty) + ) } def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index c852eebfc..e707c98e1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -53,7 +53,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository .setHDFSPath(dataset.hdfsPath) .setHDFSPublishPath(dataset.hdfsPublishPath) .setConformance(dataset.conformance) - .setProperties(removeBlankProperties(dataset.properties)) + .setProperties(removeBlankPropertiesOpt(dataset.properties)) .setDescription(dataset.description).asInstanceOf[Dataset] }) } @@ -113,7 +113,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository schemaName = newDataset.schemaName, schemaVersion = newDataset.schemaVersion, conformance = List(), - properties = removeBlankProperties(newDataset.properties)) + properties = removeBlankPropertiesOpt(newDataset.properties)) super.create(dataset, username) } @@ -124,28 +124,13 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } - def updateProperties(username: String, datasetName: String, datasetVersion: Int, - updatedProperties: Map[String, String]): Future[Option[(Dataset, Validation)]] = { - for { - successfulValidation <- validateProperties(updatedProperties).flatMap { - case validation if !validation.isValid => Future.failed(ValidationException(validation)) // warnings are ok for update - case validation => Future.successful(validation) // empty or with warnings - } - - // updateFuture includes latest-check and version increase - update <- updateFuture(username, datasetName, datasetVersion) { latest => - Future.successful(latest.copy(properties = Some(removeBlankProperties(updatedProperties)))) - } - } yield update - } - // kept for API v2 usage only - def updatePropertiesV2(username: String, datasetName: String, - updatedProperties: Option[Map[String, String]]): Future[Option[Dataset]] = { + def updateProperties(username: String, datasetName: String, + updatedProperties: Option[Map[String, String]]): Future[Option[Dataset]] = { for { latestVersion <- getLatestVersionNumber(datasetName) update <- update(username, datasetName, latestVersion) { latest => - latest.copy(properties = removeBlankProperties(updatedProperties)) + latest.copy(properties = removeBlankPropertiesOpt(updatedProperties)) } } yield update.map(_._1) // v2 does not expect validation on update } @@ -453,7 +438,7 @@ object DatasetService { * @param properties original properties * @return properties without empty-string value entries */ - def removeBlankProperties(properties: Option[Map[String, String]]): Option[Map[String, String]] = { + private[services] def removeBlankPropertiesOpt(properties: Option[Map[String, String]]): Option[Map[String, String]] = { properties.map { removeBlankProperties } @@ -465,7 +450,7 @@ object DatasetService { * @param properties original properties * @return properties without empty-string value entries */ - private def removeBlankProperties(properties: Map[String, String]): Map[String, String] = { + private[services] def removeBlankProperties(properties: Map[String, String]): Map[String, String] = { properties.filter { case (_, propValue) => propValue.nonEmpty } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index bb13f677f..eb6ef28cc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -21,7 +21,7 @@ import org.springframework.security.core.context.SecurityContextHolder import org.springframework.security.core.userdetails.UserDetails import za.co.absa.enceladus.model.{ModelVersion, Schema, UsedIn, Validation} import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionList} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} import za.co.absa.enceladus.rest_api.exceptions._ import za.co.absa.enceladus.rest_api.repositories.VersionedMongoRepository import za.co.absa.enceladus.model.menas.audit._ @@ -38,8 +38,8 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit private[services] val logger = LoggerFactory.getLogger(this.getClass) - def getLatestVersionsSummary(searchQuery: Option[String]): Future[Seq[VersionedSummary]] = { - versionedMongoRepository.getLatestVersionsSummary(searchQuery) + def getLatestVersionsSummarySearch(searchQuery: Option[String]): Future[Seq[VersionedSummary]] = { + versionedMongoRepository.getLatestVersionsSummarySearch(searchQuery) } def getLatestVersions(): Future[Seq[C]] = { @@ -58,10 +58,6 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getAllVersions(name) } - def getAllVersionsValues(name: String): Future[Option[VersionList]] = { - versionedMongoRepository.getAllVersionsValues(name) - } - def getLatestVersion(name: String): Future[Option[C]] = { versionedMongoRepository.getLatestVersionValue(name).flatMap({ case Some(version) => getVersion(name, version) @@ -80,6 +76,10 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getLatestVersionValue(name) } + def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { + versionedMongoRepository.getLatestVersionSummary(name) + } + def exportSingleItem(name: String, version: Int): Future[String] = { getVersion(name, version).flatMap({ case Some(item) => Future(item.exportItem()) @@ -202,6 +202,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit } private[rest_api] def create(item: C, username: String): Future[Option[(C, Validation)]] = { + // individual validations are deliberately not run in parallel - the latter may not be needed if the former fails for { validation <- for { generalValidation <- validate(item) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index 10078eda4..f5dff3127 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -19,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.rest_api.exceptions.ValidationException import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} +import za.co.absa.enceladus.rest_api.services.DatasetService._ import za.co.absa.enceladus.rest_api.services.{DatasetService, MappingTableService, PropertyDefinitionService, SchemaService} import scala.concurrent.Future @@ -53,6 +55,7 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito // general entity validation is extendable for V3 - here with properties validation override def validate(item: Dataset): Future[Validation] = { + // individual validations are deliberately not run in parallel - the latter may not be needed if the former fails for { originalValidation <- super.validate(item) propertiesValidation <- validateProperties(item.propertiesAsMap) @@ -73,6 +76,21 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito } } + def updateProperties(username: String, datasetName: String, datasetVersion: Int, + updatedProperties: Map[String, String]): Future[Option[(Dataset, Validation)]] = { + for { + successfulValidation <- validateProperties(updatedProperties).flatMap { + case validation if !validation.isValid => Future.failed(ValidationException(validation)) // warnings are ok for update + case validation => Future.successful(validation) // empty or with warnings + } + + // updateFuture includes latest-check and version increase + update <- updateFuture(username, datasetName, datasetVersion) { latest => + Future.successful(latest.copy(properties = Some(removeBlankProperties(updatedProperties)))) + } + } yield update + } + } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala index 00b517746..897bc0d17 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala @@ -60,7 +60,7 @@ package object implicits { classOf[Run], classOf[Schema], classOf[SchemaField], classOf[SplineReference], classOf[RunSummary], classOf[RunDatasetNameGroupedSummary], classOf[RunDatasetVersionGroupedSummary], classOf[RuntimeConfig], classOf[OozieSchedule], classOf[OozieScheduleInstance], classOf[ScheduleTiming], classOf[DataFormat], - classOf[UserInfo], classOf[VersionedSummary], classOf[VersionList], classOf[MenasAttachment], classOf[MenasReference], + classOf[UserInfo], classOf[VersionedSummary], classOf[MenasAttachment], classOf[MenasReference], classOf[PropertyDefinition], classOf[PropertyType], classOf[Essentiality], classOf[LandingPageInformation], classOf[TodaysRunsStatistics], classOf[DataFrameFilter] diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 216bbe9fd..51b0c7b25 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.model.versionedModel.VersionList +import za.co.absa.enceladus.model.versionedModel.VersionedSummary import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -148,9 +148,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA parent = Some(DatasetFactory.toParent(datasetV1))) datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[VersionList](s"$apiUrl/datasetA") + val response = sendGet[VersionedSummary](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == VersionList("versions", Seq(1, 2))) + assert(response.getBody == VersionedSummary("datasetA", 2)) } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 4dc11ee16..9c3f4d3f2 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -477,14 +477,14 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { "DatasetMongoRepository::getLatestVersions" should { "return an empty Seq" when { "no datasets exist and search query is provided" in { - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some("abc"))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("abc"))) assert(actual.isEmpty) } "only disabled dataset exists" in { val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", version = 1, disabled = true, dateDisabled = Option(DatasetFactory.dummyZonedDateTime), userDisabled = Option("user")) datasetFixture.add(dataset1) - assert(await(datasetMongoRepository.getLatestVersionsSummary(Some("dataset1"))).isEmpty) + assert(await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("dataset1"))).isEmpty) } } @@ -496,7 +496,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val dataset5 = DatasetFactory.getDummyDataset(name = "abc", version = 1) datasetFixture.add(dataset2, dataset3, dataset4, dataset5) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some("dataset2"))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("dataset2"))) val expected = Seq(dataset3).map(DatasetFactory.toSummary) assert(actual == expected) @@ -508,7 +508,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val dataset5 = DatasetFactory.getDummyDataset(name = "abc", version = 1) datasetFixture.add(dataset2, dataset3, dataset4, dataset5) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some("tas"))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("tas"))) val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummary) assert(actual == expected) @@ -544,7 +544,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val abc1 = DatasetFactory.getDummyDataset(name = "abc", version = 1) datasetFixture.add(dataset1ver1, dataset1ver2, dataset2ver1, abc1) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some(""))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some(""))) val expected = Seq(abc1, dataset1ver2, dataset2ver1).map(DatasetFactory.toSummary) assert(actual == expected) @@ -561,7 +561,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val dataset1ver2 = DatasetFactory.getDummyDataset(name = "dataset1", version = 2) datasetFixture.add(dataset1ver2) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(None)) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(None)) val expected = Seq(dataset1ver2, dataset2ver2).map(DatasetFactory.toSummary) assert(actual == expected) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala index 48914328b..7afcb53c6 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala @@ -305,7 +305,7 @@ class DatasetServiceTest extends VersionedModelServiceTest[Dataset] with Matcher ) val dataset = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(properties)) - DatasetService.removeBlankProperties(dataset.properties) shouldBe Some(Map("propKey1" -> "someValue")) + DatasetService.removeBlankPropertiesOpt(dataset.properties) shouldBe Some(Map("propKey1" -> "someValue")) } test("DatasetService.replacePrefixIfFound replaces field prefixes") { From 7fe2a93aa6df378d41131dd122e8be231a41912e Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 4 May 2022 09:55:42 +0200 Subject: [PATCH 48/67] #1693 PR review update: VersionedModelControllerV3 - GET /{name}/export mapping removed -> reflected in IT; small updates --- .../v3/VersionedModelControllerV3.scala | 18 +++------ .../DatasetControllerV3IntegrationSuite.scala | 39 ------------------- 2 files changed, 6 insertions(+), 51 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 74c742867..8060cfc3c 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -36,6 +36,10 @@ import javax.servlet.http.HttpServletRequest import scala.concurrent.Future import scala.util.{Failure, Success, Try} +object VersionedModelControllerV3 { + val LatestVersionKey = "latest" +} + abstract class VersionedModelControllerV3[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -89,12 +93,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product forVersionExpression(name, version)(versionedModelService.exportSingleItem) } - @GetMapping(Array("/{name}/export")) - @ResponseStatus(HttpStatus.OK) - def exportLatestEntity(@PathVariable name: String): CompletableFuture[String] = { - versionedModelService.exportLatestItem(name) // todo: remove in favor of the above? (that supports /{name}/latest/export) - } - @PostMapping(Array("/{name}/import")) @ResponseStatus(HttpStatus.CREATED) @PreAuthorize("@authConstants.hasAdminRole(authentication)") @@ -207,8 +205,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product val location: URI = ServletUriComponentsBuilder.fromRequest(request) .path(s"$strippingPrefix/{name}/{version}$suffix") .buildAndExpand(name, version.toString) - .normalize() // will normalize `/one/two/../three` into `/one/tree` - .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 + .normalize // will normalize `/one/two/../three` into `/one/tree` + .toUri // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918 @@ -216,7 +214,3 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } - -object VersionedModelControllerV3 { - val LatestVersionKey = "latest" -} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 8e512df88..059118540 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -399,45 +399,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"GET $apiUrl/{name}/export" should { - "return 404" when { - "when the name does not exist" in { - val response = sendGet[String](s"$apiUrl/notFoundDataset/export") - assertNotFound(response) - } - } - - "return 200" when { - "there is a correct Dataset" should { - "return the exported Dataset representation for the latest version" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") - val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), - properties = Some(Map("key1" -> "val1", "key2" -> "val2")), - conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) - ) - datasetFixture.add(dataset1, dataset2) - val response = sendGet[String](s"$apiUrl/dataset/export") - - assertOk(response) - - val body = response.getBody - assert(body == - """{"metadata":{"exportVersion":1},"item":{ - |"name":"dataset", - |"description":"Hi, I am the latest version", - |"hdfsPath":"/dummy/path", - |"hdfsPublishPath":"/dummy/publish/path", - |"schemaName":"dummySchema", - |"schemaVersion":1, - |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], - |"properties":{"key2":"val2","key1":"val1"} - |}}""".stripMargin.replaceAll("[\\r\\n]", "")) - } - } - } - } - s"POST $apiUrl/{name}/import" should { val importableDs = """{"metadata":{"exportVersion":1},"item":{ From 56a9f0c99265695e572c477b820d1e7f1f50e3ad Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 4 May 2022 14:45:01 +0200 Subject: [PATCH 49/67] #1693 VersionedMongoRepository - unused method `getAllVersionsValues` removed, some ITs and comments added --- .../VersionedMongoRepository.scala | 20 +++--- .../DatasetRepositoryIntegrationSuite.scala | 66 ++++++++++++++++++- 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 95f68b210..90f9dd6e8 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -84,6 +84,9 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.find(getNameVersionFilter(name, Some(version))).headOption() } + /** + * Beware that this method ignores the disabled flag of the entities + */ def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { val pipeline = Seq( filter(getNameFilter(name)), @@ -92,20 +95,13 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).headOption() } + /** + * Beware that this method ignores the disabled flag of the entities + */ def getLatestVersionValue(name: String): Future[Option[Int]] = { getLatestVersionSummary(name).map(_.map(_.latestVersion)) } - def getAllVersionsValues(name: String): Future[Seq[Int]] = { - val pipeline = Seq( - filter(getNameFilter(name)), - Aggregates.sort(Sorts.ascending("version")), - Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array - ) - collection.aggregate[Seq[Int]](pipeline).headOption().map(_.getOrElse(Seq.empty) - ) - } - def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { val filter = if (inclDisabled) getNameFilter(name) else getNameFilterEnabled(name) collection @@ -175,8 +171,8 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab val pipeline = Seq( filter(Filters.notEqual("disabled", true)), Aggregates.group("$name", - Accumulators.max("latestVersion", "$version"), - Accumulators.last("doc","$$ROOT")), + Accumulators.max("latestVersion", "$version"), + Accumulators.last("doc", "$$ROOT")), Aggregates.replaceRoot("$doc")) ++ postAggFilter.map(Aggregates.filter) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 9c3f4d3f2..087381db9 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.integration.repositories import com.mongodb.MongoWriteException import org.junit.runner.RunWith +import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -32,11 +33,12 @@ import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance import za.co.absa.enceladus.model.menas.scheduler.ScheduleTiming import za.co.absa.enceladus.model.menas.scheduler.RuntimeConfig import za.co.absa.enceladus.model.menas.scheduler.dataFormats.ParquetDataFormat +import za.co.absa.enceladus.model.versionedModel.VersionedSummary @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { +class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers { @Autowired private val datasetFixture: DatasetFixtureService = null @@ -474,7 +476,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { } } - "DatasetMongoRepository::getLatestVersions" should { + "DatasetMongoRepository::getLatestVersionsSummarySearch" should { "return an empty Seq" when { "no datasets exist and search query is provided" in { val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("abc"))) @@ -568,6 +570,66 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { } } + "DatasetMongoRepository::getLatestVersionSummary" should { + "returns no summary" when { + "no datasets exist by the name" in { + val actual = await(datasetMongoRepository.getLatestVersionSummary("notExistingName")) + actual shouldBe None + } + } + "returns even the disabled dataset" when { + "only disabled dataset exists" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) + datasetFixture.add(dataset1, dataset2) + val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) + actual shouldBe Some(VersionedSummary("datasetA", 2)) // warning: currently, this method reports the disabled, too + } + } + + "return give correct version summary" when { + "dataset versions exist" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(dataset1, dataset2, dataset3) + + val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) + actual shouldBe Some(VersionedSummary("datasetA", 3)) + } + } + } + + "DatasetMongoRepository::getLatestVersionValue" should { + "returns no latest version" when { + "no datasets exist by the name" in { + val actual = await(datasetMongoRepository.getLatestVersionValue("notExistingName")) + actual shouldBe None + } + } + "returns even the disabled dataset version" when { + "only disabled dataset exists" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) + datasetFixture.add(dataset1, dataset2) + val actual = await(datasetMongoRepository.getLatestVersionValue("datasetA")) + actual shouldBe Some(2) // warning: currently, this method reports the disabled, too + } + } + + "return gives correct latest version" when { + "dataset versions exist" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(dataset1, dataset2, dataset3) + + val actual = await(datasetMongoRepository.getLatestVersionValue("datasetA")) + actual shouldBe Some(3) + } + } + } + "DatasetMongoRepository::distinctCount" should { "return 0" when { "no datasets exists" in { From a4aac7097d505b0568592dd0d9d334a29ceddf24 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 4 May 2022 15:28:01 +0200 Subject: [PATCH 50/67] #1693 for V3 API: VersionedSummary -> NamedLatestVersion: impl + IT --- .../versionedModel/NamedLatestVersion.scala | 18 ++++++++++++++++++ .../versionedModel/VersionedSummary.scala | 4 +++- .../v3/VersionedModelControllerV3.scala | 7 ++++--- .../DatasetControllerV3IntegrationSuite.scala | 6 +++--- 4 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala new file mode 100644 index 000000000..8c44eff11 --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.versionedModel + +case class NamedLatestVersion(name: String, version: Int) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index d8ab45f16..909a193dc 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -15,6 +15,8 @@ package za.co.absa.enceladus.model.versionedModel -case class VersionedSummary(_id: String, latestVersion: Int) +case class VersionedSummary(_id: String, latestVersion: Int) { + def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion) +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 8060cfc3c..11241d3b2 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -50,15 +50,16 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product // todo maybe offset/limit? @GetMapping(Array("")) @ResponseStatus(HttpStatus.OK) - def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedLatestVersion]] = { versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) + .map(_.map(_.toNamedLatestVersion)) } @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[VersionedSummary] = { + def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedLatestVersion] = { versionedModelService.getLatestVersionSummary(name) map { - case Some(entity) => entity + case Some(entity) => entity.toNamedLatestVersion case None => throw notFound() } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 059118540..97854b83f 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.model.versionedModel.VersionedSummary +import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -158,9 +158,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA parent = Some(DatasetFactory.toParent(datasetV1))) datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[VersionedSummary](s"$apiUrl/datasetA") + val response = sendGet[NamedLatestVersion](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == VersionedSummary("datasetA", 2)) + assert(response.getBody == NamedLatestVersion("datasetA", 2)) } } From 5d24819ae14faf62096e590e7d10870ad096236b Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 4 May 2022 09:55:42 +0200 Subject: [PATCH 51/67] #1693 PR review update: VersionedModelControllerV3 - GET /{name}/export mapping removed -> reflected in IT; small updates --- .../v3/VersionedModelControllerV3.scala | 18 +++------ .../DatasetControllerV3IntegrationSuite.scala | 39 ------------------- 2 files changed, 6 insertions(+), 51 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index a7d385d0e..c277e65f2 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -36,6 +36,10 @@ import javax.servlet.http.HttpServletRequest import scala.concurrent.Future import scala.util.{Failure, Success, Try} +object VersionedModelControllerV3 { + val LatestVersionKey = "latest" +} + abstract class VersionedModelControllerV3[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -89,12 +93,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product forVersionExpression(name, version)(versionedModelService.exportSingleItem) } - @GetMapping(Array("/{name}/export")) - @ResponseStatus(HttpStatus.OK) - def exportLatestEntity(@PathVariable name: String): CompletableFuture[String] = { - versionedModelService.exportLatestItem(name) // todo: remove in favor of the above? (that supports /{name}/latest/export) - } - @PostMapping(Array("/{name}/import")) @ResponseStatus(HttpStatus.CREATED) def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @@ -203,8 +201,8 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product val location: URI = ServletUriComponentsBuilder.fromRequest(request) .path(s"$strippingPrefix/{name}/{version}$suffix") .buildAndExpand(name, version.toString) - .normalize() // will normalize `/one/two/../three` into `/one/tree` - .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 + .normalize // will normalize `/one/two/../three` into `/one/tree` + .toUri // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918 @@ -212,7 +210,3 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } } - -object VersionedModelControllerV3 { - val LatestVersionKey = "latest" -} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 51b0c7b25..2c3b9da46 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -377,45 +377,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"GET $apiUrl/{name}/export" should { - "return 404" when { - "when the name does not exist" in { - val response = sendGet[String](s"$apiUrl/notFoundDataset/export") - assertNotFound(response) - } - } - - "return 200" when { - "there is a correct Dataset" should { - "return the exported Dataset representation for the latest version" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val dataset1 = DatasetFactory.getDummyDataset(name = "dataset") - val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, description = Some("Hi, I am the latest version"), - properties = Some(Map("key1" -> "val1", "key2" -> "val2")), - conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) - ) - datasetFixture.add(dataset1, dataset2) - val response = sendGet[String](s"$apiUrl/dataset/export") - - assertOk(response) - - val body = response.getBody - assert(body == - """{"metadata":{"exportVersion":1},"item":{ - |"name":"dataset", - |"description":"Hi, I am the latest version", - |"hdfsPath":"/dummy/path", - |"hdfsPublishPath":"/dummy/publish/path", - |"schemaName":"dummySchema", - |"schemaVersion":1, - |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], - |"properties":{"key2":"val2","key1":"val1"} - |}}""".stripMargin.replaceAll("[\\r\\n]", "")) - } - } - } - } - s"POST $apiUrl/{name}/import" should { val importableDs = """{"metadata":{"exportVersion":1},"item":{ From 70406946bdda2c39806db79fb6786d04c29ec6d0 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 4 May 2022 14:45:01 +0200 Subject: [PATCH 52/67] #1693 VersionedMongoRepository - unused method `getAllVersionsValues` removed, some ITs and comments added --- .../VersionedMongoRepository.scala | 20 +++--- .../DatasetRepositoryIntegrationSuite.scala | 66 ++++++++++++++++++- 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 95f68b210..90f9dd6e8 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -84,6 +84,9 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.find(getNameVersionFilter(name, Some(version))).headOption() } + /** + * Beware that this method ignores the disabled flag of the entities + */ def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { val pipeline = Seq( filter(getNameFilter(name)), @@ -92,20 +95,13 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).headOption() } + /** + * Beware that this method ignores the disabled flag of the entities + */ def getLatestVersionValue(name: String): Future[Option[Int]] = { getLatestVersionSummary(name).map(_.map(_.latestVersion)) } - def getAllVersionsValues(name: String): Future[Seq[Int]] = { - val pipeline = Seq( - filter(getNameFilter(name)), - Aggregates.sort(Sorts.ascending("version")), - Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array - ) - collection.aggregate[Seq[Int]](pipeline).headOption().map(_.getOrElse(Seq.empty) - ) - } - def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { val filter = if (inclDisabled) getNameFilter(name) else getNameFilterEnabled(name) collection @@ -175,8 +171,8 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab val pipeline = Seq( filter(Filters.notEqual("disabled", true)), Aggregates.group("$name", - Accumulators.max("latestVersion", "$version"), - Accumulators.last("doc","$$ROOT")), + Accumulators.max("latestVersion", "$version"), + Accumulators.last("doc", "$$ROOT")), Aggregates.replaceRoot("$doc")) ++ postAggFilter.map(Aggregates.filter) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 9c3f4d3f2..087381db9 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.integration.repositories import com.mongodb.MongoWriteException import org.junit.runner.RunWith +import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -32,11 +33,12 @@ import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance import za.co.absa.enceladus.model.menas.scheduler.ScheduleTiming import za.co.absa.enceladus.model.menas.scheduler.RuntimeConfig import za.co.absa.enceladus.model.menas.scheduler.dataFormats.ParquetDataFormat +import za.co.absa.enceladus.model.versionedModel.VersionedSummary @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { +class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers { @Autowired private val datasetFixture: DatasetFixtureService = null @@ -474,7 +476,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { } } - "DatasetMongoRepository::getLatestVersions" should { + "DatasetMongoRepository::getLatestVersionsSummarySearch" should { "return an empty Seq" when { "no datasets exist and search query is provided" in { val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("abc"))) @@ -568,6 +570,66 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { } } + "DatasetMongoRepository::getLatestVersionSummary" should { + "returns no summary" when { + "no datasets exist by the name" in { + val actual = await(datasetMongoRepository.getLatestVersionSummary("notExistingName")) + actual shouldBe None + } + } + "returns even the disabled dataset" when { + "only disabled dataset exists" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) + datasetFixture.add(dataset1, dataset2) + val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) + actual shouldBe Some(VersionedSummary("datasetA", 2)) // warning: currently, this method reports the disabled, too + } + } + + "return give correct version summary" when { + "dataset versions exist" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(dataset1, dataset2, dataset3) + + val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) + actual shouldBe Some(VersionedSummary("datasetA", 3)) + } + } + } + + "DatasetMongoRepository::getLatestVersionValue" should { + "returns no latest version" when { + "no datasets exist by the name" in { + val actual = await(datasetMongoRepository.getLatestVersionValue("notExistingName")) + actual shouldBe None + } + } + "returns even the disabled dataset version" when { + "only disabled dataset exists" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) + datasetFixture.add(dataset1, dataset2) + val actual = await(datasetMongoRepository.getLatestVersionValue("datasetA")) + actual shouldBe Some(2) // warning: currently, this method reports the disabled, too + } + } + + "return gives correct latest version" when { + "dataset versions exist" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(dataset1, dataset2, dataset3) + + val actual = await(datasetMongoRepository.getLatestVersionValue("datasetA")) + actual shouldBe Some(3) + } + } + } + "DatasetMongoRepository::distinctCount" should { "return 0" when { "no datasets exists" in { From e0fda73af58ca77435aed3d29e9c5daae7df2c7f Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 4 May 2022 15:28:01 +0200 Subject: [PATCH 53/67] #1693 for V3 API: VersionedSummary -> NamedLatestVersion: impl + IT --- .../versionedModel/NamedLatestVersion.scala | 18 ++++++++++++++++++ .../versionedModel/VersionedSummary.scala | 4 +++- .../v3/VersionedModelControllerV3.scala | 7 ++++--- .../DatasetControllerV3IntegrationSuite.scala | 6 +++--- 4 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala new file mode 100644 index 000000000..8c44eff11 --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.versionedModel + +case class NamedLatestVersion(name: String, version: Int) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index d8ab45f16..909a193dc 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -15,6 +15,8 @@ package za.co.absa.enceladus.model.versionedModel -case class VersionedSummary(_id: String, latestVersion: Int) +case class VersionedSummary(_id: String, latestVersion: Int) { + def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion) +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index c277e65f2..41aea585e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -50,15 +50,16 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product // todo maybe offset/limit? @GetMapping(Array("")) @ResponseStatus(HttpStatus.OK) - def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedLatestVersion]] = { versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) + .map(_.map(_.toNamedLatestVersion)) } @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[VersionedSummary] = { + def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedLatestVersion] = { versionedModelService.getLatestVersionSummary(name) map { - case Some(entity) => entity + case Some(entity) => entity.toNamedLatestVersion case None => throw notFound() } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 2c3b9da46..e0f523000 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.model.versionedModel.VersionedSummary +import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -148,9 +148,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA parent = Some(DatasetFactory.toParent(datasetV1))) datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[VersionedSummary](s"$apiUrl/datasetA") + val response = sendGet[NamedLatestVersion](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == VersionedSummary("datasetA", 2)) + assert(response.getBody == NamedLatestVersion("datasetA", 2)) } } From 1da8e8c17a3fd9d077901ab8c867859ba2f3d727 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 5 May 2022 12:39:29 +0200 Subject: [PATCH 54/67] #1693 PR review updates: renamings, comments, removed unused, formatting, --- .../controllers/v3/MappingTableControllerV3.scala | 11 ++++++----- .../rest_api/controllers/v3/SchemaControllerV3.scala | 4 ++-- .../enceladus/rest_api/services/SchemaService.scala | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala index e4fbb845b..f8a14e5c6 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -50,11 +50,11 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @PutMapping(path = Array("/{name}/{version}/defaults")) @ResponseStatus(HttpStatus.CREATED) @PreAuthorize("@authConstants.hasAdminRole(authentication)") - def updateDefault(@AuthenticationPrincipal user: UserDetails, - @PathVariable name: String, - @PathVariable version: String, - @RequestBody newDefaults: Array[DefaultValue], - request: HttpServletRequest + def updateDefaults(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody newDefaults: Array[DefaultValue], + request: HttpServletRequest ): CompletableFuture[ResponseEntity[Validation]] = { for { existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion) @@ -75,6 +75,7 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @RequestBody newDefault: DefaultValue, request: HttpServletRequest ): CompletableFuture[ResponseEntity[Validation]] = { + // request processing as above in PUT except for: mappingTableService.{updateDefaults -> addDefault} being used for { existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion) existingMt = existingMtOpt.getOrElse(throw notFound()) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala index 58a8a1f75..9031c71e1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala @@ -30,7 +30,7 @@ import za.co.absa.enceladus.rest_api.exceptions.ValidationException import za.co.absa.enceladus.rest_api.models.rest.exceptions.SchemaParsingException import za.co.absa.enceladus.rest_api.repositories.RefCollection import za.co.absa.enceladus.rest_api.services.v3.SchemaServiceV3 -import za.co.absa.enceladus.rest_api.services.{AttachmentService, SchemaRegistryService, SchemaService} +import za.co.absa.enceladus.rest_api.services.{AttachmentService, SchemaRegistryService} import za.co.absa.enceladus.rest_api.utils.SchemaType import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import za.co.absa.enceladus.rest_api.utils.parsers.SchemaParser @@ -60,7 +60,7 @@ class SchemaControllerV3 @Autowired()( def getJson(@PathVariable name: String, @PathVariable version: String, @RequestParam(defaultValue = "false") pretty: Boolean): CompletableFuture[String] = { - forVersionExpression(name, version) (schemaService.getVersion).map { + forVersionExpression(name, version)(schemaService.getVersion).map { case Some(schema) => if (schema.fields.isEmpty) throw ValidationException( Validation.empty.withError("schema-fields", s"Schema $name v$version exists, but has no fields!") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index 913cfa594..a4f5eb763 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -67,14 +67,14 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, current.setDescription(update.description).asInstanceOf[Schema] } - /** final - override `updateWithFields` if needed */ + /** final - override `updateFields` if needed */ final override def update(username: String, update: Schema): Future[Option[(Schema, Validation)]] = { super.update(username, update.name, update.version) { latest => updateFields(latest, update) } } - /** final - override `updateWithFields` if needed */ + /** final - override `updateFields` if needed */ final override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { val initSchema = Schema(name = newSchema.name, description = newSchema.description) From 9c3f663bc391b0c632d1cea42f99c764374bba3d Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 5 May 2022 15:32:24 +0200 Subject: [PATCH 55/67] #1693 PR review updates: `@PreAuthorize("@authConstants.hasAdminRole(authentication)")` is now limited to changing endpoints of PropertyDefinitionControllerV3 (previously it was incorrectly used for all changing endpoints. IT updated. And specific check for the adminRole have been added to PropertyDefinitionControllerV3IntegrationSuite. --- .../controllers/v3/DatasetControllerV3.scala | 3 - .../v3/MappingTableControllerV3.scala | 2 - .../v3/PropertyDefinitionControllerV3.scala | 61 ++++- .../controllers/v3/SchemaControllerV3.scala | 3 - .../v3/VersionedModelControllerV3.scala | 5 - .../DatasetControllerV3IntegrationSuite.scala | 108 ++------ ...ingTableControllerV3IntegrationSuite.scala | 48 +--- ...finitionControllerV3IntegrationSuite.scala | 246 +++++++++++++++++- .../SchemaControllerV3IntegrationSuite.scala | 92 ++----- 9 files changed, 361 insertions(+), 207 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index a72e97c22..b56cf937a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} -import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ @@ -47,7 +46,6 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) @PutMapping(Array("/{name}/{version}/properties")) @ResponseStatus(HttpStatus.OK) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def updateProperties(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: String, @@ -79,7 +77,6 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) @PostMapping(Array("/{name}/{version}/rules")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def addConformanceRule(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: String, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala index f8a14e5c6..efb83c268 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -49,7 +49,6 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @PutMapping(path = Array("/{name}/{version}/defaults")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def updateDefaults(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: String, @@ -68,7 +67,6 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @PostMapping(path = Array("/{name}/{version}/defaults")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def addDefault(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: String, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala index e134fe716..e80c86778 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala @@ -15,13 +15,70 @@ package za.co.absa.enceladus.rest_api.controllers.v3 +import com.mongodb.client.result.UpdateResult import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.access.prepost.PreAuthorize +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.{ExportableObject, Validation} import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService +import java.util.Optional +import java.util.concurrent.CompletableFuture +import javax.servlet.http.HttpServletRequest + @RestController @RequestMapping(path = Array("/api-v3/property-definitions/datasets"), produces = Array("application/json")) class PropertyDefinitionControllerV3 @Autowired()(propertyDefService: PropertyDefinitionService) - extends VersionedModelControllerV3(propertyDefService) + extends VersionedModelControllerV3(propertyDefService) { + + // super-class implementation is sufficient, but the following changing endpoints need admin-auth + + @PostMapping(Array("/{name}/import")) + @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") + override def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @RequestBody importObject: ExportableObject[PropertyDefinition], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + super.importSingleEntity(principal, name, importObject, request) + } + + @PostMapping(Array("")) + @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") + override def create(@AuthenticationPrincipal principal: UserDetails, + @RequestBody item: PropertyDefinition, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + + super.create(principal, item, request) + } + + @PutMapping(Array("/{name}/{version}")) + @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") + override def edit(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: Int, + @RequestBody item: PropertyDefinition, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + + super.edit(user, name, version, item, request) + } + + @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + @PreAuthorize("@authConstants.hasAdminRole(authentication)") + override def disable(@PathVariable name: String, + @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { + + super.disable(name, version) + } + + // todo add "enable" with preAuth check when available, too + +} -// super-class implementation is sufficient diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala index 9031c71e1..ed7c3e8df 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/SchemaControllerV3.scala @@ -87,7 +87,6 @@ class SchemaControllerV3 @Autowired()( @PostMapping(Array("/{name}/{version}/from-file")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def handleFileUpload(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: Int, @@ -123,7 +122,6 @@ class SchemaControllerV3 @Autowired()( @PostMapping(Array("/{name}/{version}/from-remote-uri")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def handleRemoteFile(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: Int, @@ -151,7 +149,6 @@ class SchemaControllerV3 @Autowired()( @PostMapping(Array("/{name}/{version}/from-registry")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def handleSubject(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @PathVariable version: Int, diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 11241d3b2..578a4a4f1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers.v3 import com.mongodb.client.result.UpdateResult import org.springframework.http.{HttpStatus, ResponseEntity} -import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ @@ -96,7 +95,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PostMapping(Array("/{name}/import")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @PathVariable name: String, @RequestBody importObject: ExportableObject[C], @@ -122,7 +120,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PostMapping(Array("")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C, request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { @@ -140,7 +137,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @PutMapping(Array("/{name}/{version}")) @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def edit(@AuthenticationPrincipal user: UserDetails, @PathVariable name: String, @PathVariable version: Int, @@ -162,7 +158,6 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @DeleteMapping(Array("/{name}", "/{name}/{version}")) @ResponseStatus(HttpStatus.OK) - @PreAuthorize("@authConstants.hasAdminRole(authentication)") def disable(@PathVariable name: String, @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { val v = if (version.isPresent) { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 97854b83f..1ed1ff4c1 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -71,7 +71,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val response = sendPostByAdmin[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertCreated(response) response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val locationHeader = response.getHeaders.getFirst("location") @@ -94,7 +94,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1, dataset2) val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create - val response = sendPostByAdmin[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) + val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) assertCreated(response) val locationHeaders = response.getHeaders.get("location").asScala locationHeaders should have size 1 @@ -116,7 +116,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset = DatasetFactory.getDummyDataset("dummyDs") // there are schemas defined - val response = sendPostByAdmin[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertBadRequest(response) val responseBody = response.getBody @@ -128,23 +128,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("undefinedProperty1" -> "value1"))) // propdefs are empty - val response = sendPostByAdmin[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) assertBadRequest(response) val responseBody = response.getBody responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } } - - "return 403" when { - s"admin auth is not used for POST" in { - val dataset = DatasetFactory.getDummyDataset("dummyDs") - - val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - // todo what to do if "the last dataset version is disabled"? } @@ -290,7 +280,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA version = 2 // update references the last version ) - val response = sendPutByAdmin[Dataset, Validation](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) assertCreated(response) response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val locationHeader = response.getHeaders.getFirst("location") @@ -317,7 +307,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetA2 = DatasetFactory.getDummyDataset("datasetA", description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant - val response = sendPutByAdmin[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) assertBadRequest(response) val responseBody = response.getBody @@ -334,30 +324,18 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) datasetFixture.add(datasetA1) - val response = sendPutByAdmin[Dataset, String](s"$apiUrl/datasetA/7", + val response = sendPut[Dataset, String](s"$apiUrl/datasetA/7", bodyOpt = Some(DatasetFactory.getDummyDataset("datasetA", version = 5))) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody should include("version mismatch: 7 != 5") - val response2 = sendPutByAdmin[Dataset, String](s"$apiUrl/datasetABC/4", + val response2 = sendPut[Dataset, String](s"$apiUrl/datasetABC/4", bodyOpt = Some(DatasetFactory.getDummyDataset("datasetXYZ", version = 4))) response2.getStatusCode shouldBe HttpStatus.BAD_REQUEST response2.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") } } } - - "return 403" when { - s"admin auth is not used for PUT" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) - datasetFixture.add(datasetA1) - - val datasetA2 = DatasetFactory.getDummyDataset("datasetA", description = Some("second version"), version = 2) - val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } s"GET $apiUrl/{name}/audit-trail" should { @@ -415,7 +393,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 400" when { "a Dataset with the given name" should { "fail when name in the URL and payload is mismatched" in { - val response = sendPostByAdmin[String, String](s"$apiUrl/datasetABC/import", + val response = sendPost[String, String](s"$apiUrl/datasetABC/import", bodyOpt = Some(importableDs)) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") @@ -425,29 +403,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing - val response = sendPostByAdmin[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody shouldBe Validation.empty.withError("key2", "There is no property definition for key 'key2'.") } } - "return 403" when { - s"admin auth is not used for POST" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence - val dataset1 = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("init version")) - datasetFixture.add(dataset1) - - propertyDefinitionFixture.add( - PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), - PropertyDefinitionFactory.getDummyPropertyDefinition("key2") - ) - - val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - "return 201" when { "there is a existing Dataset" should { "a +1 version of dataset is added" in { @@ -461,7 +423,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("key3", essentiality = Essentiality.Recommended) ) - val response = sendPostByAdmin[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/2") @@ -491,7 +453,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("key2") ) - val response = sendPostByAdmin[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version @@ -651,7 +613,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA s"PUT $apiUrl/{name}/{version}/properties" should { "return 404" when { "when the name+version does not exist" in { - val response = sendPutByAdmin[Map[String, String], String](s"$apiUrl/notFoundDataset/456/properties", bodyOpt = Some(Map.empty)) + val response = sendPut[Map[String, String], String](s"$apiUrl/notFoundDataset/456/properties", bodyOpt = Some(Map.empty)) assertNotFound(response) } } @@ -664,7 +626,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) datasetFixture.add(datasetV1, datasetV2, datasetV3) - val response = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) assertBadRequest(response) val responseBody = response.getBody @@ -679,7 +641,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(datasetV1) // propdefs are empty - val response = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("undefinedProperty1" -> "someValue"))) assertBadRequest(response) @@ -697,13 +659,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) ) - val response1 = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + val response1 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("AorB" -> "a"))) // this is ok, but mandatoryA is missing assertBadRequest(response1) response1.getBody shouldBe Validation(Map("mandatoryA" -> List("Dataset property 'mandatoryA' is mandatory, but does not exist!"))) - val response2 = sendPutByAdmin[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + val response2 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(Map("mandatoryA" -> "valueA", "AorB" -> "c"))) // mandatoryA is ok, but AorB has invalid value assertBadRequest(response2) @@ -711,20 +673,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - "return 403" when { - s"admin auth is not used for PUT" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(datasetV1) - - propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("keyA")) - val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", - bodyOpt = Some(Map("keyA" -> "valA"))) - - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - "201 Created with location" when { Seq( ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) @@ -742,7 +690,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) ) - val response1 = sendPutByAdmin[String, Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) + val response1 = sendPut[String, Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) assertCreated(response1) response1.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") val headers1 = response1.getHeaders @@ -878,7 +826,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetV1) - val response = sendPostByAdmin[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", + val response = sendPost[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) assertNotFound(response) } @@ -892,7 +840,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA )) datasetFixture.add(datasetV1) - val response = sendPostByAdmin[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", + val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) assertBadRequest(response) @@ -903,7 +851,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetV1) - val response = sendPostByAdmin[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleMcrRule0)) assertBadRequest(response) @@ -911,18 +859,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - "return 403" when { - s"admin auth is not used for POST" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") - datasetFixture.add(datasetV1) - - val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", - bodyOpt = Some(LiteralConformanceRule(order = 0,"column1", true, "ABC"))) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - "return 201" when { "when conf rule is added" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) @@ -931,7 +867,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA ) datasetFixture.add(datasetV1) - val response = sendPostByAdmin[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) assertCreated(response) // if, in the future, there can be a rule update resulting in a warning, let's reflect that here response.getBody shouldBe Validation.empty diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index 58e4aa744..7dece37b2 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -51,7 +51,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be "referenced schema does not exits" in { val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchemaA", schemaVersion = 1) - val response = sendPostByAdmin[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) assertBadRequest(response) val responseBody = response.getBody @@ -59,14 +59,6 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } - "return 403" when { - s"admin auth is not used for POST $apiUrl" in { - val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchemaA", schemaVersion = 1) - val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - "return 201" when { "a MappingTables is created" in { val mtA = MappingTableFactory.getDummyMappingTable("mtA", schemaName = "mtSchema1", schemaVersion = 1) @@ -74,7 +66,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist - val response = sendPostByAdmin[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) + val response = sendPost[MappingTable, Validation](apiUrl, bodyOpt = Some(mtA)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/mapping-tables/mtA/1") @@ -136,9 +128,9 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be "when the name/version does not exist" in { mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) - assertNotFound(sendPutByAdmin[Array[DefaultValue], String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(Array()))) - assertNotFound(sendPutByAdmin[Array[DefaultValue], String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(Array()))) - assertNotFound(sendPutByAdmin[Array[DefaultValue], String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(Array()))) + assertNotFound(sendPut[Array[DefaultValue], String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(Array()))) } } @@ -150,7 +142,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be mappingTableFixture.add(mtAv1, mtAv2, mtAv3) - val response = sendPutByAdmin[Array[DefaultValue], Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(Array())) + val response = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(Array())) assertBadRequest(response) val responseBody = response.getBody @@ -160,14 +152,6 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } - "return 403" when { - s"admin auth is not used for PUT" in { - // no need for fixture whip-up, auth check should precede further processing - val response = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(Array.empty[DefaultValue])) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - "201 Created with location" when { Seq( ("empty defaults", Array.empty[DefaultValue]), @@ -179,7 +163,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist - val response1 = sendPutByAdmin[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(bothPayloadAndExpectedResult)) + val response1 = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(bothPayloadAndExpectedResult)) assertCreated(response1) response1.getBody shouldBe Validation.empty val headers1 = response1.getHeaders @@ -200,9 +184,9 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("mtA")) val aDefaultValue = DefaultValue("colA", "defaultA") - assertNotFound(sendPostByAdmin[DefaultValue, String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(aDefaultValue))) - assertNotFound(sendPostByAdmin[DefaultValue, String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(aDefaultValue))) - assertNotFound(sendPostByAdmin[DefaultValue, String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/notFoundMt/456/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/mtA/456/defaults", bodyOpt = Some(aDefaultValue))) + assertNotFound(sendPost[DefaultValue, String](s"$apiUrl/notFoundMt/latest/defaults", bodyOpt = Some(aDefaultValue))) } } @@ -214,7 +198,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be mappingTableFixture.add(mtAv1, mtAv2, mtAv3) - val response = sendPostByAdmin[DefaultValue, Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + val response = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/2/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) assertBadRequest(response) val responseBody = response.getBody @@ -224,14 +208,6 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } - "return 403" when { - s"admin auth is not used for POST" in { - // no need for fixture whip-up, auth check should precede further processing - val response = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - "201 Created with location" when { s"defaults are replaced with a new version" in { val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) @@ -239,7 +215,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist - val response1 = sendPostByAdmin[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + val response1 = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) assertCreated(response1) response1.getBody shouldBe Validation.empty val headers1 = response1.getHeaders diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 63670c4b9..633666d76 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -25,8 +25,9 @@ import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.Validation import za.co.absa.enceladus.model.properties.PropertyDefinition -import za.co.absa.enceladus.model.properties.propertyType.StringPropertyType +import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} import za.co.absa.enceladus.model.test.factories.PropertyDefinitionFactory +import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -127,4 +128,247 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w } } } + + s"GET $apiUrl/{name}" should { + "return 200" when { + "a propDef with the given name exists - so it gives versions" in { + val pdV1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", version = 1) + val pdV2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", + version = 2, parent = Some(PropertyDefinitionFactory.toParent(pdV1))) + propertyDefinitionFixture.add(pdV1, pdV2) + + val response = sendGet[NamedLatestVersion](s"$apiUrl/pdA") + assertOk(response) + assert(response.getBody == NamedLatestVersion("pdA", 2)) + } + } + + "return 404" when { + "a propDef with the given name does not exist" in { + val pd = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", version = 1) + propertyDefinitionFixture.add(pd) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName") + assertNotFound(response) + } + } + } + + s"GET $apiUrl/{name}/{version}" should { + "return 200" when { + "a PropertyDefinition with the given name and version exists - gives specified version of entity" in { + val pdV1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", version = 1) + val pdV2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", version = 2, description = Some("second")) + val pdV3 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", version = 3, description = Some("third")) + propertyDefinitionFixture.add(pdV1, pdV2, pdV3) + + val response = sendGet[PropertyDefinition](s"$apiUrl/pdA/2") + assertOk(response) + + val actual = response.getBody + val expected = toExpected(pdV2, actual) + + assert(actual == expected) + } + } + + "return 404" when { + "a PropertyDefinition with the given name/version does not exist" in { + val pd = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "pdA", version = 1) + propertyDefinitionFixture.add(pd) + + val response = sendGet[String](s"$apiUrl/anotherPropertyDefinitionName/1") + assertNotFound(response) + + val response2 = sendGet[String](s"$apiUrl/pdA/7") + assertNotFound(response2) + } + } + } + + s"PUT $apiUrl/{name}/{version}" can { + "return 200" when { + "a PropertyDefinition with the given name and version is the latest that exists" should { + "update the propertyDefinition" in { + val propertyDefinitionA1 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA") + val propertyDefinitionA2 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", + description = Some("second version"), version = 2) + propertyDefinitionFixture.add(propertyDefinitionA1, propertyDefinitionA2) + + val propertyDefinitionA3 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", + description = Some("updated"), + propertyType = EnumPropertyType("a", "b"), + version = 2 // update references the last version + ) + + val response = sendPutByAdmin[PropertyDefinition, Validation](s"$apiUrl/propertyDefinitionA/2", bodyOpt = Some(propertyDefinitionA3)) + assertCreated(response) + response.getBody shouldBe Validation.empty + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/property-definitions/datasets/propertyDefinitionA/3") + + val response2 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinitionA/3") + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(propertyDefinitionA3.copy(version = 3, parent = Some(PropertyDefinitionFactory.toParent(propertyDefinitionA2))), actual) + assert(actual == expected) + } + } + } + + "return 400" when { + "a PropertyDefinition with the given name and version" should { + "fail when version/name in the URL and payload is mismatched" in { + val propertyDefinitionA1 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", description = Some("init version")) + propertyDefinitionFixture.add(propertyDefinitionA1) + + val response = sendPutByAdmin[PropertyDefinition, String](s"$apiUrl/propertyDefinitionA/7", + bodyOpt = Some(PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", version = 5))) + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody should include("version mismatch: 7 != 5") + + val response2 = sendPutByAdmin[PropertyDefinition, String](s"$apiUrl/propertyDefinitionABC/4", + bodyOpt = Some(PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionXYZ", version = 4))) + response2.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response2.getBody should include("name mismatch: 'propertyDefinitionABC' != 'propertyDefinitionXYZ'") + } + } + } + + "return 403" when { + s"admin auth is not used" in { + val propertyDefinitionA1 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA") + propertyDefinitionFixture.add(propertyDefinitionA1) + + val propertyDefinitionA2 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", + description = Some("updated"), + propertyType = EnumPropertyType("a", "b"), + version = 1 // update references the last version + ) + + val response = sendPut[PropertyDefinition, String](s"$apiUrl/propertyDefinitionA/1", bodyOpt = Some(propertyDefinitionA2)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + + "return 404" when { + "a propDef with the given name does not exist" in { + val propertyDefinitionA2 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA") + + val response = sendPutByAdmin[PropertyDefinition, String](s"$apiUrl/propertyDefinitionA/1", bodyOpt = Some(propertyDefinitionA2)) + assertNotFound(response) + } + } + + } + + s"POST $apiUrl/{name}/import" should { + val importablePd = + """{"todo":{"exportVersion":1},"item":{ + |"name":"propertyDefinitionXYZ", + |"description":"Hi, I am the import", + |"propertyType":{"_t":"StringPropertyType"}, + |"putIntoInfoFile":false, + |"essentiality":{"_t":"Optional"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "") + + "return 400" when { + "a PropertyDefinition with the given name" should { + "fail when name in the URL and payload is mismatched" in { + val response = sendPostByAdmin[String, String](s"$apiUrl/propertyDefinitionABC/import", + bodyOpt = Some(importablePd)) + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody should include("name mismatch: 'propertyDefinitionABC' != 'propertyDefinitionXYZ'") + } + } + } + + "return 403" when { + s"admin auth is not used" in { + val response = sendPost[String, Validation](s"$apiUrl/propertyDefinitionXYZ/import", bodyOpt = Some(importablePd)) + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + + "return 201" when { + "there is a existing PropertyDefinition" should { + "a +1 version of propertyDefinition is added" in { + val propertyDefinition1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinitionXYZ", description = Some("init version")) + propertyDefinitionFixture.add(propertyDefinition1) + + val response = sendPostByAdmin[String, Validation](s"$apiUrl/propertyDefinitionXYZ/import", bodyOpt = Some(importablePd)) + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/property-definitions/datasets/propertyDefinitionXYZ/2") + response.getBody shouldBe Validation.empty + + val response2 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinitionXYZ/2") + assertOk(response2) + + val actual = response2.getBody + val expectedPdBase = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinitionXYZ", version = 2, + description = Some("Hi, I am the import"), + parent = Some(PropertyDefinitionFactory.toParent(propertyDefinition1)) + ) + val expected = toExpected(expectedPdBase, actual) + + assert(actual == expected) + } + } + + "there is no such PropertyDefinition, yet" should { + "a the version of propertyDefinition created" in { + val response = sendPostByAdmin[String, String](s"$apiUrl/propertyDefinitionXYZ/import", bodyOpt = Some(importablePd)) + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/property-definitions/datasets/propertyDefinitionXYZ/1") // this is the first version + + val response2 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinitionXYZ/1") + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinitionXYZ", + description = Some("Hi, I am the import")) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + } + } + + s"GET $apiUrl/{name}/{version}/export" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundPropertyDefinition/2/export") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct PropertyDefinition version" should { + "return the exported PropertyDefinition representation" in { + val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2, description = Some("v2 here")) + val propertyDefinition3 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 3, description = Some("showing non-latest export")) + propertyDefinitionFixture.add(propertyDefinition2, propertyDefinition3) + val response = sendGet[String](s"$apiUrl/propertyDefinition/2/export") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"metadata":{"exportVersion":1},"item":{ + |"name":"propertyDefinition", + |"description":"v2 here", + |"propertyType":{"_t":"StringPropertyType","suggestedValue":null}, + |"putIntoInfoFile":false, + |"essentiality":{"_t":"Optional"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + } + + // todo delete } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index e2430fddb..e5f948571 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -87,7 +87,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn SchemaField("field1", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) )) - val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") @@ -107,7 +107,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "return 400" when { "a Schema is created (empty fields = warning)" in { val schema = SchemaFactory.getDummySchema("schemaA") - val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) assertBadRequest(response) response.getBody shouldBe Validation.empty @@ -118,7 +118,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn SchemaField("field1", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty))) schemaFixture.add(schema) - val response = sendPostByAdmin[Schema, Validation](apiUrl, bodyOpt = Some(schema)) + val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) assertBadRequest(response) @@ -129,14 +129,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn assert(actual == expected) } } - - "return 403" when { - s"admin auth is not used for POST $apiUrl" in { - val schema = SchemaFactory.getDummySchema() - val response = sendPost[Schema, Validation](apiUrl, bodyOpt = Some(schema)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } s"PUT $apiUrl/{name}/{version}" can { @@ -150,7 +142,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schema2 = SchemaFactory.getDummySchema("schemaA", fields = List( SchemaField("anotherField", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) )) - val response = sendPutByAdmin[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) + val response = sendPut[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") @@ -173,24 +165,13 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema1) val schema2 = SchemaFactory.getDummySchema("schemaA") - val response = sendPutByAdmin[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) + val response = sendPut[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) assertBadRequest(response) response.getBody shouldBe Validation.empty .withError("schema-fields", "No fields found! There must be fields defined for actual usage.") } } - - "return 403" when { - s"admin auth is not used for POST $apiUrl" in { - val schema1 = SchemaFactory.getDummySchema("schemaA") - schemaFixture.add(schema1) - - val schema = SchemaFactory.getDummySchema("schemaA") - val response = sendPut[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } // todo disable dataset - all versions/one version/ check the usage to prevent from disabling @@ -369,7 +350,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, String]("format" -> "copybook") - val responseUploaded = sendPostUploadFileByAdmin[Validation]( + val responseUploaded = sendPostUploadFile[Validation]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) assertCreated(responseUploaded) val locationHeader = responseUploaded.getHeaders.getFirst("location") @@ -391,7 +372,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, Any]("format" -> "struct") - val responseUploaded = sendPostUploadFileByAdmin[Validation]( + val responseUploaded = sendPostUploadFile[Validation]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) assertCreated(responseUploaded) val locationHeader = responseUploaded.getHeaders.getFirst("location") @@ -413,7 +394,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, Any]("format" -> "avro") - val responseUploaded = sendPostUploadFileByAdmin[Schema]( + val responseUploaded = sendPostUploadFile[Schema]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.ok, schemaParams) assertCreated(responseUploaded) val locationHeader = responseUploaded.getHeaders.getFirst("location") @@ -436,7 +417,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap.empty[String, Any] // v2 fallbacked on this, v3 forbids it - val response = sendPostUploadFileByAdmin[String]( + val response = sendPostUploadFile[String]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) assertBadRequest(response) response.getBody should include("Required String parameter 'format' is not present") @@ -447,7 +428,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn schemaFixture.add(schema) val schemaParams = HashMap[String, Any]("format" -> "") // v2 fallbacked on this, v3 forbids it - val response = sendPostUploadFileByAdmin[String]( + val response = sendPostUploadFile[String]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) assertBadRequest(response) response.getBody should include("not a recognized schema format. Menas currently supports: struct, copybook, avro.") @@ -456,7 +437,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a copybook with a syntax error" should { "return a response containing a schema parsing error with syntax error specific fields" in { val schemaParams = HashMap[String, Any]("format" -> "copybook") - val response = sendPostUploadFileByAdmin[RestResponse]( + val response = sendPostUploadFile[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.bogus, schemaParams) val body = response.getBody @@ -476,7 +457,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a JSON struct type schema with a syntax error" should { "return a response containing a schema parsing error returned by the StructType parser" in { val schemaParams = HashMap[String, Any]("format" -> "struct") - val response = sendPostUploadFileByAdmin[RestResponse]( + val response = sendPostUploadFile[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) val body = response.getBody @@ -494,7 +475,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "an avro-schema with a syntax error" should { "return a response containing a schema parsing error encountered during avro schema parsing" in { val schemaParams = HashMap[String, Any]("format" -> "avro") - val response = sendPostUploadFileByAdmin[RestResponse]( + val response = sendPostUploadFile[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Avro.bogus, schemaParams) val body = response.getBody @@ -512,7 +493,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "a wrong format has been specified" should { "return a response containing a schema format error" in { val schemaParams = HashMap[String, Any]("format" -> "foo") - val response = sendPostUploadFileByAdmin[RestResponse]( + val response = sendPostUploadFile[RestResponse]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.bogus, schemaParams) val body = response.getBody @@ -531,20 +512,11 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "return 404" when { "a schema file is uploaded, but no schema exists for the specified name and version" in { val schemaParams = HashMap[String, Any]("format" -> "copybook") - val responseUploaded = sendPostUploadFileByAdmin[Schema]( + val responseUploaded = sendPostUploadFile[Schema]( s"$apiUrl/schemaA/1/from-file", TestResourcePath.Copybook.ok, schemaParams) assertNotFound(responseUploaded) } } - - "return 403" when { - s"admin auth is not used for POST $apiUrl/{name}/{version}/from-file" in { - val schemaParams = HashMap[String, Any]("format" -> "copybook") - val response = sendPostUploadFile[Schema]( - s"$apiUrl/irrelevantWhatSchema/123/from-file", TestResourcePath.Copybook.ok, schemaParams) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } import com.github.tomakehurst.wiremock.client.WireMock._ @@ -579,7 +551,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) - val responseRemoteLoaded = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -603,7 +575,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) val params = HashMap("remoteUrl" -> remoteUrl, "format" -> "struct") - val responseRemoteLoaded = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -627,7 +599,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "remoteUrl" -> remoteUrl) - val responseRemoteLoaded = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -656,7 +628,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(testResourcePath))) val params = HashMap("format" -> schemaType.toString, "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFileByAdmin[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) + val response = sendPostRemoteFile[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) val body = response.getBody assertBadRequest(response) @@ -677,7 +649,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) val params = HashMap[String, Any]("format" -> "foo", "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFileByAdmin[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) + val response = sendPostRemoteFile[RestResponse](s"$apiUrl/schemaA/1/from-remote-uri", params) val body = response.getBody assertBadRequest(response) @@ -698,19 +670,10 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFileByAdmin[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val response = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) assertNotFound(response) } } - - "return 403" when { - s"admin auth is not used for POST $apiUrl/{name}/{version}/from-remote-uri" in { - // no need for any mocking, auth check should precede further processing - val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) - val response = sendPostRemoteFile[Schema](s"$apiUrl/irrelevantWhatSchema/123/from-remote-uri", params) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } s"POST $apiUrl/{name}/{version}/from-registry" should { @@ -726,7 +689,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic1-value") - val responseRemoteLoaded = sendPostSubjectByAdmin[Schema](s"$apiUrl/schemaA/1/from-registry", params) + val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -751,7 +714,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic2") - val responseRemoteLoaded = sendPostSubjectByAdmin[Schema](s"$apiUrl/schemaA/1/from-registry", params) + val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) assertCreated(responseRemoteLoaded) val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -766,15 +729,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } } - - "return 403" when { - s"admin auth is not used for POST $apiUrl/{name}/{version}/from-registry" in { - // no need for any mocking, auth check should precede further processing - val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopicABC") - val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/irrelevantWhatSchema/123/from-registry", params) - responseRemoteLoaded.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } } From 99ee67b0e7cb94cf49e4a26b4edaf5a02a0a8491 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 5 May 2022 15:51:30 +0200 Subject: [PATCH 56/67] #1693 mergefix: admin-auth for PropDefs only. --- .../v3/PropertyDefinitionControllerV3.scala | 11 ++--- .../DatasetControllerV3IntegrationSuite.scala | 47 ++++--------------- ...ingTableControllerV3IntegrationSuite.scala | 20 ++------ .../SchemaControllerV3IntegrationSuite.scala | 41 +++++----------- 4 files changed, 29 insertions(+), 90 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala index d7982eba5..8b1c0d6f1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala @@ -24,6 +24,8 @@ import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.{ExportableObject, Validation} +import za.co.absa.enceladus.rest_api.exceptions.NotFoundException +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload import za.co.absa.enceladus.rest_api.services.v3.PropertyDefinitionServiceV3 import java.util.Optional @@ -69,14 +71,11 @@ class PropertyDefinitionControllerV3 @Autowired()(propertyDefinitionService: Pro super.edit(user, name, version, item, request) } - @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @DeleteMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) @PreAuthorize("@authConstants.hasAdminRole(authentication)") - override def disable(@PathVariable name: String, - @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { - - super.disable(name, version) + override def disable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { + super.disable(name) } - } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index bb535cf84..55a8ca071 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -117,22 +117,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1) val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("a new version attempt")) - val response = sendPostByAdmin[Dataset, EntityDisabledException](apiUrl, bodyOpt = Some(dataset2)) + val response = sendPost[Dataset, EntityDisabledException](apiUrl, bodyOpt = Some(dataset2)) assertBadRequest(response) response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") } } - "return 403" when { - s"admin auth is not used for POST" in { - val dataset = DatasetFactory.getDummyDataset("dummyDs") - - val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } - } s"GET $apiUrl/{name}" should { @@ -340,7 +331,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1) val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("ds update")) - val response = sendPutByAdmin[Dataset, EntityDisabledException](s"$apiUrl/dummyDs/1", bodyOpt = Some(dataset2)) + val response = sendPut[Dataset, EntityDisabledException](s"$apiUrl/dummyDs/1", bodyOpt = Some(dataset2)) assertBadRequest(response) response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") @@ -925,7 +916,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1, disabled = true) datasetFixture.add(dsA1, dsA2, dsB) - val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/dsA") + val response = sendPut[String, DisabledPayload](s"$apiUrl/dsA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = false) @@ -951,7 +942,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) datasetFixture.add(dsA1, dsA2) - val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/dsA") + val response = sendPut[String, DisabledPayload](s"$apiUrl/dsA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = false) @@ -970,22 +961,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 404" when { "no Dataset with the given name exists" should { "enable nothing" in { - val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/aDataset") + val response = sendPut[String, DisabledPayload](s"$apiUrl/aDataset") assertNotFound(response) } } } - - "return 403" when { - s"admin auth is not used for DELETE" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(datasetV1) - - val response = sendDelete[Validation](s"$apiUrl/datasetA") - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } s"DELETE $apiUrl/{name}" can { @@ -997,7 +977,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1) datasetFixture.add(dsA1, dsA2, dsB) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/dsA") + val response = sendDelete[DisabledPayload](s"$apiUrl/dsA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -1023,7 +1003,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) datasetFixture.add(dsA1, dsA2) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/dsA") + val response = sendDelete[DisabledPayload](s"$apiUrl/dsA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -1042,21 +1022,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "return 404" when { "no Dataset with the given name exists" should { "disable nothing" in { - val response = sendDeleteByAdmin[String](s"$apiUrl/aDataset") + val response = sendDelete[String](s"$apiUrl/aDataset") assertNotFound(response) } } } - - "return 403" when { - s"admin auth is not used for DELETE" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) - datasetFixture.add(datasetV1) - - val response = sendDelete[Validation](s"$apiUrl/datasetA") - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index 380fae84f..6a2b9c2b1 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -328,7 +328,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mtB = MappingTableFactory.getDummyMappingTable(name = "mtB", version = 1) mappingTableFixture.add(mtA1, mtA2, mtB) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/mtA") + val response = sendDelete[DisabledPayload](s"$apiUrl/mtA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -354,7 +354,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mtA2 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 2, disabled = false) mappingTableFixture.add(mtA1, mtA2) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/mtA") + val response = sendDelete[DisabledPayload](s"$apiUrl/mtA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -376,7 +376,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) mappingTableFixture.add(mappingTable1, mappingTable2) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/mappingTable") + val response = sendDelete[DisabledPayload](s"$apiUrl/mappingTable") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -406,7 +406,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", conformance = List(mcr("mappingTable", 2)), disabled = true) datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) - val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/mappingTable") + val response = sendDelete[UsedIn](s"$apiUrl/mappingTable") assertBadRequest(response) response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) @@ -417,21 +417,11 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be "return 404" when { "no MappingTable with the given name exists" should { "disable nothing" in { - val response = sendDeleteByAdmin[String](s"$apiUrl/aMappingTable") + val response = sendDelete[String](s"$apiUrl/aMappingTable") assertNotFound(response) } } } - - "return 403" when { - s"admin auth is not used for DELETE" in { - val mappingTableV1 = MappingTableFactory.getDummyMappingTable(name = "mappingTableA", version = 1) - mappingTableFixture.add(mappingTableV1) - - val response = sendDelete[Validation](s"$apiUrl/mappingTableA") - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index 56cf49b69..ba7c8d5f7 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -795,7 +795,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schB = SchemaFactory.getDummySchema(name = "schB", version = 1, disabled = true) schemaFixture.add(schA1, schA2, schB) - val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/schA") + val response = sendPut[String, DisabledPayload](s"$apiUrl/schA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = false) @@ -821,7 +821,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) schemaFixture.add(schA1, schA2) - val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/schA") + val response = sendPut[String, DisabledPayload](s"$apiUrl/schA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = false) @@ -840,21 +840,12 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "return 404" when { "no Schema with the given name exists" should { "enable nothing" in { - val response = sendPutByAdmin[String, DisabledPayload](s"$apiUrl/aSchema") + val response = sendPut[String, DisabledPayload](s"$apiUrl/aSchema") assertNotFound(response) } } } - "return 403" when { - s"admin auth is not used for DELETE" in { - val schemaV1 = SchemaFactory.getDummySchema(name = "schemaA", version = 1) - schemaFixture.add(schemaV1) - - val response = sendDelete[Validation](s"$apiUrl/schemaA") - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } s"DELETE $apiUrl/{name}" can { @@ -866,7 +857,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schB = SchemaFactory.getDummySchema(name = "schB", version = 1) schemaFixture.add(schA1, schA2, schB) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schA") + val response = sendDelete[DisabledPayload](s"$apiUrl/schA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -892,7 +883,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) schemaFixture.add(schA1, schA2) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schA") + val response = sendDelete[DisabledPayload](s"$apiUrl/schA") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -914,7 +905,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schema") + val response = sendDelete[DisabledPayload](s"$apiUrl/schema") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -937,7 +928,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/schema") + val response = sendDelete[DisabledPayload](s"$apiUrl/schema") assertOk(response) response.getBody shouldBe DisabledPayload(disabled = true) @@ -967,7 +958,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", schemaName = "schema", schemaVersion = 2, disabled = true) datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) - val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") + val response = sendDelete[UsedIn](s"$apiUrl/schema") assertBadRequest(response) response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) @@ -983,7 +974,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mapping2", schemaName = "schema", schemaVersion = 2, disabled = false) mappingTableFixture.add(mappingTable1, mappingTable2) - val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") + val response = sendDelete[UsedIn](s"$apiUrl/schema") assertBadRequest(response) response.getBody shouldBe UsedIn(None, Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) @@ -1001,7 +992,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", schemaName = "schema", schemaVersion = 2) datasetFixture.add(dataset2) - val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/schema") + val response = sendDelete[UsedIn](s"$apiUrl/schema") assertBadRequest(response) response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset2", 1))), Some(Seq(MenasReference(None, "mapping1", 1)))) @@ -1012,21 +1003,11 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn "return 404" when { "no Schema with the given name exists" should { "disable nothing" in { - val response = sendDeleteByAdmin[String](s"$apiUrl/aSchema") + val response = sendDelete[String](s"$apiUrl/aSchema") assertNotFound(response) } } } - - "return 403" when { - s"admin auth is not used for DELETE" in { - val schemaV1 = SchemaFactory.getDummySchema(name = "schemaA", version = 1) - schemaFixture.add(schemaV1) - - val response = sendDelete[Validation](s"$apiUrl/schemaA") - response.getStatusCode shouldBe HttpStatus.FORBIDDEN - } - } } } From a80d9ac6bc86214d611dfed403fb0d6fe21e48bc Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Fri, 6 May 2022 14:13:31 +0200 Subject: [PATCH 57/67] #1693 Swagger differs in paths based on `spring.profiles.active` being `dev` or not. --- .../enceladus/rest_api/SpringFoxConfig.scala | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala index d57f4b30e..4373dadad 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala @@ -17,7 +17,7 @@ package za.co.absa.enceladus.rest_api import com.google.common.base.Predicate import com.google.common.base.Predicates.or -import org.springframework.context.annotation.{Bean, Configuration} +import org.springframework.context.annotation.{Bean, Configuration, Primary, Profile} import springfox.documentation.builders.PathSelectors.regex import springfox.documentation.builders.{ApiInfoBuilder, RequestHandlerSelectors} import springfox.documentation.spi.DocumentationType @@ -28,34 +28,55 @@ import za.co.absa.enceladus.utils.general.ProjectMetadata @Configuration @EnableSwagger2 class SpringFoxConfig extends ProjectMetadata { + + import org.springframework.beans.factory.annotation.Value + + @Value("${spring.profiles.active:}") + private val activeProfiles: String = null + @Bean def api(): Docket = { + val isDev = activeProfiles.split(",").map(_.toLowerCase).contains("dev") + new Docket(DocumentationType.SWAGGER_2) - .apiInfo(apiInfo) + .apiInfo(apiInfo(isDev)) .select .apis(RequestHandlerSelectors.any) - .paths(filteredPaths) + .paths(filteredPaths(isDev)) .build } - private def filteredPaths: Predicate[String] = - or[String]( - // api v2 + private def filteredPaths(isDev: Boolean): Predicate[String] = { + val v2devPaths = Seq( regex("/api/dataset.*"), regex("/api/schema.*"), regex("/api/mappingTable.*"), regex("/api/properties.*"), regex("/api/monitoring.*"), regex("/api/runs.*"), regex("/api/user.*"), regex("/api/spark.*"), - regex("/api/configuration.*"), - - // api v3 + regex("/api/configuration.*") + ) + val v2prodPaths = Seq( + regex("/api/.*/importItem"), + regex("/api/.*/exportItem/.*") + ) + val v3paths = Seq( regex("/api-v3/datasets.*"), regex("/api-v3/schemas.*"), regex("/api-v3/mapping-tables.*"), regex("/api-v3/property-definitions.*") + ) + val paths: Seq[Predicate[String]] = if (isDev) { + v2devPaths ++ v3paths + } else { + v2prodPaths ++ v3paths + } + + or[String]( + paths: _* ) + } - private def apiInfo = + private def apiInfo(isDev: Boolean) = new ApiInfoBuilder() - .title("Menas API") + .title(s"Menas API${ if (isDev) " - DEV " else ""}") .description("Menas API reference for developers") .license("Apache 2.0 License") .version(projectVersion) // api or project? From 562da606dc77a93c36dfb5c38198de90464ed439 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 9 May 2022 09:21:47 +0200 Subject: [PATCH 58/67] #1693 Swagger API: dev-profile: full v2+v3 API, non-dev: full v3 API --- .../co/absa/enceladus/rest_api/SpringFoxConfig.scala | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala index 4373dadad..4f2010588 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala @@ -47,26 +47,23 @@ class SpringFoxConfig extends ProjectMetadata { } private def filteredPaths(isDev: Boolean): Predicate[String] = { - val v2devPaths = Seq( + val v2Paths = Seq( regex("/api/dataset.*"), regex("/api/schema.*"), regex("/api/mappingTable.*"), regex("/api/properties.*"), regex("/api/monitoring.*"), regex("/api/runs.*"), regex("/api/user.*"), regex("/api/spark.*"), regex("/api/configuration.*") ) - val v2prodPaths = Seq( - regex("/api/.*/importItem"), - regex("/api/.*/exportItem/.*") - ) + val v3paths = Seq( regex("/api-v3/datasets.*"), regex("/api-v3/schemas.*"), regex("/api-v3/mapping-tables.*"), regex("/api-v3/property-definitions.*") ) val paths: Seq[Predicate[String]] = if (isDev) { - v2devPaths ++ v3paths + v2Paths ++ v3paths } else { - v2prodPaths ++ v3paths + v3paths } or[String]( From eac77dd01d9463890a4f1b8b2998a8b4b03b606a Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 9 May 2022 09:41:16 +0200 Subject: [PATCH 59/67] #1693 unused import cleanup --- .../controllers/v3/PropertyDefinitionControllerV3.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala index 8b1c0d6f1..1b3063349 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala @@ -15,7 +15,6 @@ package za.co.absa.enceladus.rest_api.controllers.v3 -import com.mongodb.client.result.UpdateResult import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.access.prepost.PreAuthorize @@ -24,11 +23,9 @@ import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.{ExportableObject, Validation} -import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload import za.co.absa.enceladus.rest_api.services.v3.PropertyDefinitionServiceV3 -import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest From db1227b4de9163281bfad25c65147cbfd6f62b27 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 9 May 2022 10:37:06 +0200 Subject: [PATCH 60/67] #1693 `NamedLatestVersion` generalized into a multipurpose `NamedVersion`. Small updates, thanks @benedeki --- .../{NamedLatestVersion.scala => NamedVersion.scala} | 2 +- .../model/versionedModel/VersionedSummary.scala | 2 +- .../controllers/v3/VersionedModelControllerV3.scala | 10 +++++----- .../v3/DatasetControllerV3IntegrationSuite.scala | 6 +++--- ...ropertyDefinitionControllerV3IntegrationSuite.scala | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) rename data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/{NamedLatestVersion.scala => NamedVersion.scala} (91%) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala similarity index 91% rename from data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala rename to data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala index 8c44eff11..74d61f2b9 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala @@ -15,4 +15,4 @@ package za.co.absa.enceladus.model.versionedModel -case class NamedLatestVersion(name: String, version: Int) +case class NamedVersion(name: String, version: Int) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index 909a193dc..b07e95aa3 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -16,7 +16,7 @@ package za.co.absa.enceladus.model.versionedModel case class VersionedSummary(_id: String, latestVersion: Int) { - def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion) + def toNamedVersion: NamedVersion = NamedVersion(_id, latestVersion) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 692d8fef7..469c3e4f9 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -38,7 +38,7 @@ import scala.concurrent.Future import scala.util.{Failure, Success, Try} object VersionedModelControllerV3 { - val LatestVersionKey = "latest" + final val LatestVersionKey = "latest" } abstract class VersionedModelControllerV3[C <: VersionedModel with Product @@ -51,16 +51,16 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product // todo maybe offset/limit? @GetMapping(Array("")) @ResponseStatus(HttpStatus.OK) - def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedLatestVersion]] = { + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedVersion]] = { versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) - .map(_.map(_.toNamedLatestVersion)) + .map(_.map(_.toNamedVersion)) } @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedLatestVersion] = { + def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedVersion] = { versionedModelService.getLatestVersionSummary(name) map { - case Some(entity) => entity.toNamedLatestVersion + case Some(entity) => entity.toNamedVersion case None => throw notFound() } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 55a8ca071..c0e49cc59 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion +import za.co.absa.enceladus.model.versionedModel.NamedVersion import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.exceptions.EntityDisabledException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} @@ -136,9 +136,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA parent = Some(DatasetFactory.toParent(datasetV1))) datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[NamedLatestVersion](s"$apiUrl/datasetA") + val response = sendGet[NamedVersion](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == NamedLatestVersion("datasetA", 2)) + assert(response.getBody == NamedVersion("datasetA", 2)) } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index c2bd792a9..7bfec1ff0 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -28,7 +28,7 @@ import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} -import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion +import za.co.absa.enceladus.model.versionedModel.NamedVersion import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @@ -142,9 +142,9 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w version = 2, parent = Some(PropertyDefinitionFactory.toParent(pdV1))) propertyDefinitionFixture.add(pdV1, pdV2) - val response = sendGet[NamedLatestVersion](s"$apiUrl/pdA") + val response = sendGet[NamedVersion](s"$apiUrl/pdA") assertOk(response) - assert(response.getBody == NamedLatestVersion("pdA", 2)) + assert(response.getBody == NamedVersion("pdA", 2)) } } From 56ba9ed05263612b773170309b2f350d93f03fc1 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 9 May 2022 12:24:07 +0200 Subject: [PATCH 61/67] #1693 PRReview `/{name}/{version}/defaults` processing generalization with `MappingTableControllerV3.withMappingTableToResponse` --- .../v3/MappingTableControllerV3.scala | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala index efb83c268..3e7a4f030 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala @@ -26,6 +26,7 @@ import za.co.absa.enceladus.rest_api.services.v3.MappingTableServiceV3 import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest +import scala.concurrent.Future @RestController @RequestMapping(Array("/api-v3/mapping-tables")) @@ -54,15 +55,10 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @PathVariable version: String, @RequestBody newDefaults: Array[DefaultValue], request: HttpServletRequest - ): CompletableFuture[ResponseEntity[Validation]] = { - for { - existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion) - existingMt = existingMtOpt.getOrElse(throw notFound()) - updatedMtAndValidationOpt <- mappingTableService.updateDefaults(user.getUsername, name, existingMt.version, newDefaults.toList) - (updatedMt, validation) = updatedMtAndValidationOpt.getOrElse(throw notFound()) - response = createdWithNameVersionLocationBuilder(name, updatedMt.version, request, - stripLastSegments = 3, suffix = s"/defaults").body(validation) // stripping: /{name}/{version}/defaults - } yield response + ): CompletableFuture[ResponseEntity[Validation]] = { + withMappingTableToResponse(name, version, user, request) { existingMt => + mappingTableService.updateDefaults(user.getUsername, name, existingMt.version, newDefaults.toList) + } } @PostMapping(path = Array("/{name}/{version}/defaults")) @@ -73,14 +69,22 @@ class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableSer @RequestBody newDefault: DefaultValue, request: HttpServletRequest ): CompletableFuture[ResponseEntity[Validation]] = { - // request processing as above in PUT except for: mappingTableService.{updateDefaults -> addDefault} being used + withMappingTableToResponse(name, version, user, request) { existingMt => + mappingTableService.addDefault(user.getUsername, name, existingMt.version, newDefault) + } + } + + private def withMappingTableToResponse(name: String, version: String, user: UserDetails, request: HttpServletRequest, + stripLastSegments: Int = 3, suffix: String = s"/defaults") + (updateExistingMtFn: MappingTable => Future[Option[(MappingTable, Validation)]]): + Future[ResponseEntity[Validation]] = { for { existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion) existingMt = existingMtOpt.getOrElse(throw notFound()) - updatedMtAndValidationOpt <- mappingTableService.addDefault(user.getUsername, name, existingMt.version, newDefault) + updatedMtAndValidationOpt <- updateExistingMtFn(existingMt) (updatedMt, validation) = updatedMtAndValidationOpt.getOrElse(throw notFound()) response = createdWithNameVersionLocationBuilder(name, updatedMt.version, request, - stripLastSegments = 3, suffix = s"/defaults").body(validation) // stripping: /{name}/{version}/defaults + stripLastSegments, suffix).body(validation) // for .../defaults: stripping /{name}/{version}/defaults } yield response } From 0ff68513ee09e9024426f697ce53f5928154a50a Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 12 May 2022 13:51:06 +0200 Subject: [PATCH 62/67] #1692 Review update: API v3 summary (NamedVersion) now contains `disabled` information - mainly on GET ...{/name} --- .../model/test/factories/DatasetFactory.scala | 7 +++--- .../model/versionedModel/NamedVersion.scala | 5 +++- .../versionedModel/VersionedSummary.scala | 14 +++++++++-- .../controllers/DatasetController.scala | 6 ++--- .../VersionedModelController.scala | 5 ++-- .../VersionedMongoRepository.scala | 14 +++++++---- .../DatasetControllerV3IntegrationSuite.scala | 24 ++++++++++++++++++- ...finitionControllerV3IntegrationSuite.scala | 2 +- .../DatasetRepositoryIntegrationSuite.scala | 14 +++++------ 9 files changed, 66 insertions(+), 25 deletions(-) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala index f81f73248..63b55cf0f 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala @@ -16,11 +16,10 @@ package za.co.absa.enceladus.model.test.factories import java.time.ZonedDateTime - import za.co.absa.enceladus.model.Dataset import za.co.absa.enceladus.model.conformanceRule._ import za.co.absa.enceladus.model.menas.MenasReference -import za.co.absa.enceladus.model.versionedModel.VersionedSummary +import za.co.absa.enceladus.model.versionedModel.VersionedSummaryV2 object DatasetFactory extends EntityFactory[Dataset] { @@ -127,8 +126,8 @@ object DatasetFactory extends EntityFactory[Dataset] { MenasReference(collection, name, version) } - def toSummary(dataset: Dataset): VersionedSummary = { - VersionedSummary(dataset.name, dataset.version) + def toSummaryV2(dataset: Dataset): VersionedSummaryV2 = { + VersionedSummaryV2(dataset.name, dataset.version) } } diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala index 74d61f2b9..d7348e475 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala @@ -15,4 +15,7 @@ package za.co.absa.enceladus.model.versionedModel -case class NamedVersion(name: String, version: Int) +/** + * V3 Wrapper for [[za.co.absa.enceladus.model.versionedModel.VersionedSummary]] + */ +case class NamedVersion(name: String, version: Int, disabled: Boolean) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index b07e95aa3..95d395f23 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -15,8 +15,18 @@ package za.co.absa.enceladus.model.versionedModel -case class VersionedSummary(_id: String, latestVersion: Int) { - def toNamedVersion: NamedVersion = NamedVersion(_id, latestVersion) +/** + * V2 Representation of `VersionedSummary` - V2 does not carry disabled information + */ +case class VersionedSummaryV2(_id: String, latestVersion: Int) + +case class VersionedSummary(_id: String, latestVersion: Int, disabledSet: Set[Boolean]) { + def toV2: VersionedSummaryV2 = VersionedSummaryV2(_id, latestVersion) + + def toNamedVersion: NamedVersion = { + val disabled = disabledSet.contains(true) // legacy mixed state reported as disabled for V3 summary + NamedVersion(_id, latestVersion, disabled) + } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index fcdd69e91..29c063474 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -30,7 +30,7 @@ import za.co.absa.enceladus.rest_api.services.DatasetService import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.model.properties.PropertyDefinition -import za.co.absa.enceladus.model.versionedModel.VersionedSummary +import za.co.absa.enceladus.model.versionedModel.VersionedSummaryV2 import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel.Constants.DefaultValidationLevelName @@ -49,9 +49,9 @@ class DatasetController @Autowired()(datasetService: DatasetService) @GetMapping(Array("/latest")) @ResponseStatus(HttpStatus.OK) def getLatestVersions(@RequestParam(value = "missing_property", required = false) - missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummaryV2]] = { datasetService.getLatestVersions(missingProperty.toScalaOption) - .map(datasets => datasets.map(dataset => VersionedSummary(dataset.name, dataset.version))) + .map(datasets => datasets.map(dataset => VersionedSummaryV2(dataset.name, dataset.version))) } @PostMapping(Array("/{datasetName}/rule/create")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index e129a8b82..af0c844f0 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -23,7 +23,7 @@ import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn} -import za.co.absa.enceladus.model.versionedModel._ +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummaryV2} import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService import za.co.absa.enceladus.model.menas.audit._ @@ -38,8 +38,9 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @GetMapping(Array("/list", "/list/{searchQuery}")) @ResponseStatus(HttpStatus.OK) - def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummaryV2]] = { versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) + .map(_.map(_.toV2)) } @GetMapping(Array("/searchSuggestions")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 22972bfc7..149caa718 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -27,7 +27,7 @@ import org.mongodb.scala.model.Updates._ import org.mongodb.scala.model._ import org.mongodb.scala.result.UpdateResult import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionedSummaryV2} import scala.concurrent.Future import scala.reflect.ClassTag @@ -68,10 +68,13 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab } val pipeline = Seq( filter(Filters.and(searchFilter, getNotDisabledFilter)), - Aggregates.group("$name", Accumulators.max("latestVersion", "$version")), + Aggregates.group("$name", + Accumulators.max("latestVersion", "$version") + ), sort(Sorts.ascending("_id")) ) - collection.aggregate[VersionedSummary](pipeline).toFuture() + collection.aggregate[VersionedSummaryV2](pipeline).toFuture() + .map(_.map(summaryV2 => VersionedSummary(summaryV2._id, summaryV2.latestVersion, Set(true)))) // because of the notDisabled filter } def getLatestVersions(missingProperty: Option[String]): Future[Seq[C]] = { @@ -90,7 +93,10 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { val pipeline = Seq( filter(getNameFilter(name)), - Aggregates.group("$name", Accumulators.max("latestVersion", "$version")) + Aggregates.group("$name", + Accumulators.max("latestVersion", "$version"), + Accumulators.addToSet("disabledSet", "$disabled") + ) ) collection.aggregate[VersionedSummary](pipeline).headOption() } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index c0e49cc59..a036eb426 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -138,7 +138,29 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val response = sendGet[NamedVersion](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == NamedVersion("datasetA", 2)) + assert(response.getBody == NamedVersion("datasetA", 2, disabled = false)) + } + + "a Dataset with the given name exists - all disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1, disabled = true) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, disabled = true) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[NamedVersion](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == NamedVersion("datasetA", 2, disabled = true)) + } + + "a Dataset with with mixed disabled states" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, disabled = true) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[NamedVersion](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == NamedVersion("datasetA", 2, disabled = true)) // mixed state -> disabled } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 7bfec1ff0..6026b4906 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -144,7 +144,7 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w val response = sendGet[NamedVersion](s"$apiUrl/pdA") assertOk(response) - assert(response.getBody == NamedVersion("pdA", 2)) + assert(response.getBody == NamedVersion("pdA", 2, disabled = false)) } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 087381db9..5919c0650 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -500,7 +500,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset2, dataset3, dataset4, dataset5) val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("dataset2"))) - val expected = Seq(dataset3).map(DatasetFactory.toSummary) + val expected = Seq(dataset3).map(DatasetFactory.toSummaryV2) assert(actual == expected) } "search query is a partial match" in { @@ -512,7 +512,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset2, dataset3, dataset4, dataset5) val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("tas"))) - val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummary) + val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummaryV2) assert(actual == expected) } @@ -548,7 +548,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset1ver1, dataset1ver2, dataset2ver1, abc1) val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some(""))) - val expected = Seq(abc1, dataset1ver2, dataset2ver1).map(DatasetFactory.toSummary) + val expected = Seq(abc1, dataset1ver2, dataset2ver1).map(DatasetFactory.toSummaryV2) assert(actual == expected) } } @@ -565,7 +565,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(None)) - val expected = Seq(dataset1ver2, dataset2ver2).map(DatasetFactory.toSummary) + val expected = Seq(dataset1ver2, dataset2ver2).map(DatasetFactory.toSummaryV2) assert(actual == expected) } } @@ -579,11 +579,11 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers } "returns even the disabled dataset" when { "only disabled dataset exists" in { - val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = false) val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) datasetFixture.add(dataset1, dataset2) val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) - actual shouldBe Some(VersionedSummary("datasetA", 2)) // warning: currently, this method reports the disabled, too + actual shouldBe Some(VersionedSummary("datasetA", 2, Set(true, false))) } } @@ -595,7 +595,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset1, dataset2, dataset3) val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) - actual shouldBe Some(VersionedSummary("datasetA", 3)) + actual shouldBe Some(VersionedSummary("datasetA", 3, Set(false))) } } } From 4574b090034c5a08af3ec79114b123f5d3e9825b Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Thu, 12 May 2022 14:11:31 +0200 Subject: [PATCH 63/67] #1692 Review update: UsedIn.{isEmpty|nonEmpty} omitted from serialization. IT updated. --- .../scala/za/co/absa/enceladus/model/UsedIn.scala | 2 ++ .../MappingTableControllerV3IntegrationSuite.scala | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala index decf6c23a..f963f9c37 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala @@ -41,10 +41,12 @@ case class UsedIn(datasets: Option[Seq[MenasReference]] = None, } } + @JsonIgnore val isEmpty: Boolean = { normalized.datasets == None && normalized.mappingTables == None } + @JsonIgnore val nonEmpty: Boolean = !isEmpty } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index a549d3bfb..d58112909 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -270,15 +270,19 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) datasetFixture.add(datasetA, datasetB, datasetC) - val response = sendGet[UsedIn](s"$apiUrl/mappingTable/used-in") + val response = sendGet[String](s"$apiUrl/mappingTable/used-in") assertOk(response) // datasetB is disabled -> not reported // datasetC is reported, because this is a version-less check - response.getBody shouldBe UsedIn( - datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), - mappingTables = None - ) + // String-typed this time to also check isEmpty/nonEmpty serialization presence + response.getBody shouldBe + """ + |{"datasets":[ + |{"collection":null,"name":"datasetA","version":1},{"collection":null,"name":"datasetC","version":1} + |], + |"mappingTables":null} + |""".stripMargin.replaceAll("[\\r\\n]", "") } } } From 006feb0cf0b26bfe548b40f5a92b19980a21a21e Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 17 May 2022 09:13:25 +0200 Subject: [PATCH 64/67] #1693 VersionedSummaryV2 Bson support fix --- .../za/co/absa/enceladus/rest_api/utils/implicits/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala index 897bc0d17..5b0a4afd1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala @@ -60,7 +60,7 @@ package object implicits { classOf[Run], classOf[Schema], classOf[SchemaField], classOf[SplineReference], classOf[RunSummary], classOf[RunDatasetNameGroupedSummary], classOf[RunDatasetVersionGroupedSummary], classOf[RuntimeConfig], classOf[OozieSchedule], classOf[OozieScheduleInstance], classOf[ScheduleTiming], classOf[DataFormat], - classOf[UserInfo], classOf[VersionedSummary], classOf[MenasAttachment], classOf[MenasReference], + classOf[UserInfo], classOf[VersionedSummary], classOf[VersionedSummaryV2], classOf[MenasAttachment], classOf[MenasReference], classOf[PropertyDefinition], classOf[PropertyType], classOf[Essentiality], classOf[LandingPageInformation], classOf[TodaysRunsStatistics], classOf[DataFrameFilter] From 29058928c4488fc17df7e7a752d38ccb83d71025 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 17 May 2022 11:13:41 +0200 Subject: [PATCH 65/67] #1693 VersionedSummaryV2 support fix - tests & impl --- .../enceladus/model/test/factories/DatasetFactory.scala | 9 +++++---- .../rest_api/repositories/VersionedMongoRepository.scala | 2 +- .../repositories/DatasetRepositoryIntegrationSuite.scala | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala index 63b55cf0f..ebe47f5cc 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala @@ -15,11 +15,12 @@ package za.co.absa.enceladus.model.test.factories -import java.time.ZonedDateTime import za.co.absa.enceladus.model.Dataset import za.co.absa.enceladus.model.conformanceRule._ import za.co.absa.enceladus.model.menas.MenasReference -import za.co.absa.enceladus.model.versionedModel.VersionedSummaryV2 +import za.co.absa.enceladus.model.versionedModel.VersionedSummary + +import java.time.ZonedDateTime object DatasetFactory extends EntityFactory[Dataset] { @@ -126,8 +127,8 @@ object DatasetFactory extends EntityFactory[Dataset] { MenasReference(collection, name, version) } - def toSummaryV2(dataset: Dataset): VersionedSummaryV2 = { - VersionedSummaryV2(dataset.name, dataset.version) + def toSummary(dataset: Dataset): VersionedSummary = { + VersionedSummary(dataset.name, dataset.version, Set(dataset.disabled)) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 149caa718..d3fe970f3 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -74,7 +74,7 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab sort(Sorts.ascending("_id")) ) collection.aggregate[VersionedSummaryV2](pipeline).toFuture() - .map(_.map(summaryV2 => VersionedSummary(summaryV2._id, summaryV2.latestVersion, Set(true)))) // because of the notDisabled filter + .map(_.map(summaryV2 => VersionedSummary(summaryV2._id, summaryV2.latestVersion, Set(false)))) // because of the notDisabled filter } def getLatestVersions(missingProperty: Option[String]): Future[Seq[C]] = { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 5919c0650..c2c535c01 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -500,7 +500,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset2, dataset3, dataset4, dataset5) val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("dataset2"))) - val expected = Seq(dataset3).map(DatasetFactory.toSummaryV2) + val expected = Seq(dataset3).map(DatasetFactory.toSummary) assert(actual == expected) } "search query is a partial match" in { @@ -512,7 +512,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset2, dataset3, dataset4, dataset5) val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("tas"))) - val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummaryV2) + val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummary) assert(actual == expected) } @@ -548,7 +548,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset1ver1, dataset1ver2, dataset2ver1, abc1) val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some(""))) - val expected = Seq(abc1, dataset1ver2, dataset2ver1).map(DatasetFactory.toSummaryV2) + val expected = Seq(abc1, dataset1ver2, dataset2ver1).map(DatasetFactory.toSummary) assert(actual == expected) } } @@ -565,7 +565,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(None)) - val expected = Seq(dataset1ver2, dataset2ver2).map(DatasetFactory.toSummaryV2) + val expected = Seq(dataset1ver2, dataset2ver2).map(DatasetFactory.toSummary) assert(actual == expected) } } From e593df11bc158077d1abfac3e2a6662217fb66ce Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Tue, 17 May 2022 14:36:21 +0200 Subject: [PATCH 66/67] #1693 disable fail due to nonEmpty used in now carries a wrapper with an error message (`UsedIn` wrapped in `EntityInUseException`) --- .../controllers/RestExceptionHandler.scala | 4 +-- .../exceptions/EntityInUseException.scala | 3 +- .../services/VersionedModelService.scala | 3 +- .../SchemaApiFeaturesIntegrationSuite.scala | 28 ++++++++++++------- ...ingTableControllerV3IntegrationSuite.scala | 10 ++++--- ...finitionControllerV3IntegrationSuite.scala | 7 +++-- .../SchemaControllerV3IntegrationSuite.scala | 19 +++++++++---- 7 files changed, 48 insertions(+), 26 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala index b1fc91db7..115c18469 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala @@ -116,8 +116,8 @@ class RestExceptionHandler { } @ExceptionHandler(value = Array(classOf[EntityInUseException])) - def handleValidationException(exception: EntityInUseException): ResponseEntity[UsedIn] = { - ResponseEntity.badRequest().body(exception.usedIn) + def handleValidationException(exception: EntityInUseException): ResponseEntity[EntityInUseException] = { + ResponseEntity.badRequest().body(exception) } @ExceptionHandler(value = Array(classOf[MethodArgumentTypeMismatchException])) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala index 9661f8767..2235654e0 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala @@ -17,4 +17,5 @@ package za.co.absa.enceladus.rest_api.exceptions import za.co.absa.enceladus.model.UsedIn -case class EntityInUseException(usedIn: UsedIn) extends Exception() +case class EntityInUseException(message: String = "There are dependencies present preventing the action", + usedIn: UsedIn) extends Exception() diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index 94e941e24..de26f6e36 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -289,7 +289,8 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit private def disableVersion(name: String, version: Option[Int], usedIn: UsedIn, principal: UserDetails): Future[UpdateResult] = { if (usedIn.nonEmpty) { - throw EntityInUseException(usedIn) + val entityVersionStr = s"""entity "$name"${ version.map(" v" + _).getOrElse("")}""" // either "entity MyName" or "entity MyName v23" + throw EntityInUseException(s"""Cannot disable $entityVersionStr, because it is used in the following entities""", usedIn) } else { versionedMongoRepository.disableVersion(name, version, principal.getUsername) } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala index 18935a7fb..a0a95a40d 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala @@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.integration.controllers import java.io.File import java.nio.file.{Files, Path} - import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder import com.github.tomakehurst.wiremock.core.WireMockConfiguration @@ -40,6 +39,7 @@ import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException import za.co.absa.enceladus.restapi.TestResourcePath import scala.collection.immutable.HashMap @@ -303,16 +303,18 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[UsedIn](s"$apiUrl/disable/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq(MenasReference(None, "dataset", 1))), Some(Seq())) + val expected = EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset", 1))), Some(Seq())) + ) assert(actual == expected) } } - "some version of the Schema is used by a enabled MappingTable" should { + "some version of the Schema is used by an enabled MappingTable" should { "return a list of the entities the Schema is used in" in { val mappingTable = MappingTableFactory.getDummyMappingTable(name = "mapping", schemaName = "schema", schemaVersion = 1, disabled = false) mappingTableFixture.add(mappingTable) @@ -320,12 +322,14 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[UsedIn](s"$apiUrl/disable/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping", 1)))) + val expected = EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping", 1)))) + ) assert(actual == expected) } } @@ -455,12 +459,14 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[UsedIn](s"$apiUrl/disable/schema/1") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema/1") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq(MenasReference(None, "dataset1", 1))), Some(Seq())) + val expected = EntityInUseException("""Cannot disable entity "schema" v1, because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1))), Some(Seq())) + ) assert(actual == expected) } } @@ -473,12 +479,14 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[UsedIn](s"$apiUrl/disable/schema/1") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema/1") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping1", 1)))) + val expected = EntityInUseException("""Cannot disable entity "schema" v1, because it is used in the following entities""", + UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping1", 1)))) + ) assert(actual == expected) } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index d58112909..4f975dcc0 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -26,12 +26,12 @@ import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.menas.MenasReference -import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, SchemaFactory, PropertyDefinitionFactory} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.{DefaultValue, MappingTable, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload -import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException @RunWith(classOf[SpringRunner]) @@ -412,10 +412,12 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", conformance = List(mcr("mappingTable", 2)), disabled = true) datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) - val response = sendDelete[UsedIn](s"$apiUrl/mappingTable") + val response = sendDelete[EntityInUseException](s"$apiUrl/mappingTable") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "mappingTable", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + ) } } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 6026b4906..2c325405c 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -29,6 +29,7 @@ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} import za.co.absa.enceladus.model.versionedModel.NamedVersion +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @@ -514,10 +515,12 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", properties = Some(Map("keyA" -> "x")), disabled = true) datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) - val response = sendDeleteByAdmin[UsedIn](s"$apiUrl/keyA") + val response = sendDeleteByAdmin[EntityInUseException](s"$apiUrl/keyA") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "keyA", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + ) } } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index e9c32fb3d..d03194ac5 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -30,6 +30,7 @@ import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} import za.co.absa.enceladus.model.{Schema, SchemaField, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.{DisabledPayload, RestResponse} @@ -957,10 +958,12 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", schemaName = "schema", schemaVersion = 2, disabled = true) datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) - val response = sendDelete[UsedIn](s"$apiUrl/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/schema") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + ) } } "the Schema is used by a enabled MappingTable" should { @@ -973,10 +976,12 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mapping2", schemaName = "schema", schemaVersion = 2, disabled = false) mappingTableFixture.add(mappingTable1, mappingTable2) - val response = sendDelete[UsedIn](s"$apiUrl/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/schema") assertBadRequest(response) - response.getBody shouldBe UsedIn(None, Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(None, Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) + ) } } "the Schema is used by combination of MT and DS" should { @@ -991,10 +996,12 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", schemaName = "schema", schemaVersion = 2) datasetFixture.add(dataset2) - val response = sendDelete[UsedIn](s"$apiUrl/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/schema") assertBadRequest(response) - response.getBody shouldBe UsedIn(Some(Seq(MenasReference(None, "dataset2", 1))), Some(Seq(MenasReference(None, "mapping1", 1)))) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset2", 1))), Some(Seq(MenasReference(None, "mapping1", 1)))) + ) } } } From 34292c48187388a9e0a50fde7f35eddc87cf2e3d Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Wed, 18 May 2022 11:30:17 +0200 Subject: [PATCH 67/67] #1693 `Future {throw x}` replaced with `Future.failed(x)` in rest_api, cleanup --- .../rest_api/controllers/DatasetController.scala | 8 ++++---- .../enceladus/rest_api/controllers/HDFSController.scala | 5 +++-- .../rest_api/controllers/v3/DatasetControllerV3.scala | 7 +++---- .../controllers/v3/VersionedModelControllerV3.scala | 4 ++-- .../rest_api/exceptions/EndpointDisabledException.scala | 1 - 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index 29c063474..bc0ec4f1c 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -67,7 +67,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) case Some((ds, validation)) => ds // v2 disregarding validation case _ => throw notFound() } - case _ => throw notFound() + case _ => Future.failed(notFound()) } } yield res } @@ -104,7 +104,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) } else { datasetService.filterProperties(dsProperties, DatasetController.paramsToPropertyDefinitionFilter(scalaFilterMap)) } - case None => throw notFound() + case None => Future.failed(notFound()) } } @@ -126,7 +126,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def getPropertiesValidation(@PathVariable datasetName: String, @PathVariable datasetVersion: Int): CompletableFuture[Validation] = { datasetService.getVersion(datasetName, datasetVersion).flatMap { case Some(entity) => datasetService.validateProperties(entity.propertiesAsMap, forRun = false) - case None => throw notFound() + case None => Future.failed(notFound()) } } @@ -135,7 +135,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def getPropertiesValidationForRun(@PathVariable datasetName: String, @PathVariable datasetVersion: Int): CompletableFuture[Validation] = { datasetService.getVersion(datasetName, datasetVersion).flatMap { case Some(entity) => datasetService.validateProperties(entity.propertiesAsMap, forRun = true) - case None => throw notFound() + case None => Future.failed(notFound()) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala index d5ed48dae..5953c8570 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala @@ -16,7 +16,6 @@ package za.co.absa.enceladus.rest_api.controllers import java.util.concurrent.CompletableFuture - import org.apache.hadoop.fs.Path import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.HttpStatus @@ -24,6 +23,8 @@ import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.menas.HDFSFolder import za.co.absa.enceladus.rest_api.services.HDFSService +import scala.concurrent.Future + @RestController @RequestMapping(Array("/api/hdfs")) class HDFSController @Autowired() (hdfsService: HDFSService) extends BaseController { @@ -41,7 +42,7 @@ class HDFSController @Autowired() (hdfsService: HDFSService) extends BaseControl if (exists) { hdfsService.getFolder(path) } else { - throw notFound() + Future.failed(notFound()) } } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index b56cf937a..6f69ef820 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -28,6 +28,7 @@ import za.co.absa.enceladus.rest_api.utils.implicits._ import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future @RestController @RequestMapping(path = Array("/api-v3/datasets")) @@ -57,14 +58,12 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) case Some((entity, validation)) => // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties") - .body(validation) // todo include in tests + .body(validation) case None => throw notFound() } } } - // todo putIntoInfoFile switch needed? - @GetMapping(Array("/{name}/{version}/rules")) @ResponseStatus(HttpStatus.OK) def getConformanceRules(@PathVariable name: String, @@ -90,7 +89,7 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) suffix = s"/rules/$addedRuleOrder").body(validation) case _ => throw notFound() } - case None => throw notFound() + case None => Future.failed(notFound()) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 469c3e4f9..911604609 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -48,7 +48,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product import scala.concurrent.ExecutionContext.Implicits.global - // todo maybe offset/limit? + // todo maybe offset/limit -> Issue #2060 @GetMapping(Array("")) @ResponseStatus(HttpStatus.OK) def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedVersion]] = { @@ -59,7 +59,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedVersion] = { - versionedModelService.getLatestVersionSummary(name) map { + versionedModelService.getLatestVersionSummary(name).map { case Some(entity) => entity.toNamedVersion case None => throw notFound() } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala index 62f11853a..76b919e1a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala @@ -15,6 +15,5 @@ package za.co.absa.enceladus.rest_api.exceptions -// todo, there is no usage, remove? case class EndpointDisabledException(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause)