From a3fdde406a93f4b3c54b0fbb34699c772e70ad19 Mon Sep 17 00:00:00 2001 From: Daniel K Date: Mon, 9 May 2022 10:49:39 +0200 Subject: [PATCH] Feature/1693 api v3 versioned model - Dataset v3 implementation (#2046) * #1693 API v3: VersionedModel v3 - VersionedModelControllerV3 , DatasetControllerV3 - login allowed at /api/login (common) - v2/v3 BaseRestApiTest distinquished - location header for put/post and post-import - /{name}/{version}/used-in - supports latest for as version-expression, impl for datasets improved by actual existence checking + IT test cases for non-existing/non-latest queries - /{name}/{version}/validation` impl added - conformance rule mgmt GET+POST datasets/dsName/version/rules, GET datasets/dsName/version/rules/# + IT - Swagger API: dev-profile: full v2+v3 API, non-dev: full v3 API - IT testcases --- .../versionedModel/NamedLatestVersion.scala | 18 + .../versionedModel/VersionedSummary.scala | 6 +- .../enceladus/rest_api/SpringFoxConfig.scala | 41 +- .../controllers/DatasetController.scala | 4 +- .../controllers/MappingTableController.scala | 4 +- .../controllers/RestExceptionHandler.scala | 5 + .../controllers/SchemaController.scala | 2 +- .../VersionedModelController.scala | 10 +- .../controllers/v3/DatasetControllerV3.scala | 111 +++ .../v3/VersionedModelControllerV3.scala | 213 ++++ .../VersionedMongoRepository.scala | 21 +- .../rest_api/services/DatasetService.scala | 50 +- .../services/MappingTableService.scala | 10 +- .../services/PropertyDefinitionService.scala | 8 +- .../rest_api/services/SchemaService.scala | 12 +- .../services/VersionedModelService.scala | 114 ++- .../services/v3/DatasetServiceV3.scala | 96 ++ .../services/v3/HavingSchemaService.scala | 34 + .../AuthenticationIntegrationSuite.scala | 2 +- .../controllers/BaseRestApiTest.scala | 13 +- .../DatasetApiIntegrationSuite.scala | 2 +- ...ropertyDefinitionApiIntegrationSuite.scala | 2 +- .../controllers/RunApiIntegrationSuite.scala | 2 +- .../SchemaApiFeaturesIntegrationSuite.scala | 2 +- .../DatasetControllerV3IntegrationSuite.scala | 927 ++++++++++++++++++ .../DatasetRepositoryIntegrationSuite.scala | 78 +- .../services/DatasetServiceTest.scala | 2 +- .../services/VersionedModelServiceTest.scala | 34 +- 28 files changed, 1709 insertions(+), 114 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala create mode 100644 rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala new file mode 100644 index 000000000..8c44eff11 --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.versionedModel + +case class NamedLatestVersion(name: String, version: Int) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index 87a0d365d..909a193dc 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -15,4 +15,8 @@ package za.co.absa.enceladus.model.versionedModel -case class VersionedSummary(_id: String, latestVersion: Int) +case class VersionedSummary(_id: String, latestVersion: Int) { + def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion) +} + + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala index 6121ad047..4f2010588 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala @@ -17,7 +17,7 @@ package za.co.absa.enceladus.rest_api import com.google.common.base.Predicate import com.google.common.base.Predicates.or -import org.springframework.context.annotation.{Bean, Configuration} +import org.springframework.context.annotation.{Bean, Configuration, Primary, Profile} import springfox.documentation.builders.PathSelectors.regex import springfox.documentation.builders.{ApiInfoBuilder, RequestHandlerSelectors} import springfox.documentation.spi.DocumentationType @@ -28,27 +28,52 @@ import za.co.absa.enceladus.utils.general.ProjectMetadata @Configuration @EnableSwagger2 class SpringFoxConfig extends ProjectMetadata { + + import org.springframework.beans.factory.annotation.Value + + @Value("${spring.profiles.active:}") + private val activeProfiles: String = null + @Bean def api(): Docket = { + val isDev = activeProfiles.split(",").map(_.toLowerCase).contains("dev") + new Docket(DocumentationType.SWAGGER_2) - .apiInfo(apiInfo) + .apiInfo(apiInfo(isDev)) .select .apis(RequestHandlerSelectors.any) - .paths(filteredPaths) + .paths(filteredPaths(isDev)) .build } - private def filteredPaths: Predicate[String] = - or[String](regex("/api/dataset.*"), regex("/api/schema.*"), + private def filteredPaths(isDev: Boolean): Predicate[String] = { + val v2Paths = Seq( + regex("/api/dataset.*"), regex("/api/schema.*"), regex("/api/mappingTable.*"), regex("/api/properties.*"), - regex("/api/monitoring.*"),regex("/api/runs.*"), + regex("/api/monitoring.*"), regex("/api/runs.*"), regex("/api/user.*"), regex("/api/spark.*"), regex("/api/configuration.*") ) - private def apiInfo = + val v3paths = Seq( + regex("/api-v3/datasets.*"), regex("/api-v3/schemas.*"), + regex("/api-v3/mapping-tables.*"), regex("/api-v3/property-definitions.*") + ) + + val paths: Seq[Predicate[String]] = if (isDev) { + v2Paths ++ v3paths + } else { + v3paths + } + + or[String]( + paths: _* + ) + } + + private def apiInfo(isDev: Boolean) = new ApiInfoBuilder() - .title("Menas API") + .title(s"Menas API${ if (isDev) " - DEV " else ""}") .description("Menas API reference for developers") .license("Apache 2.0 License") .version(projectVersion) // api or project? diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index 0f33bf7d1..fcdd69e91 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -64,7 +64,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) latestVersion <- datasetService.getLatestVersionValue(datasetName) res <- latestVersion match { case Some(version) => datasetService.addConformanceRule(user.getUsername, datasetName, version, rule).map { - case Some(ds) => ds + case Some((ds, validation)) => ds // v2 disregarding validation case _ => throw notFound() } case _ => throw notFound() @@ -113,7 +113,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def replaceProperties(@AuthenticationPrincipal principal: UserDetails, @PathVariable datasetName: String, @RequestBody newProperties: Optional[Map[String, String]]): CompletableFuture[ResponseEntity[Option[Dataset]]] = { - datasetService.replaceProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map { + datasetService.updateProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map { case None => throw notFound() case Some(dataset) => val location: URI = new URI(s"/api/dataset/${dataset.name}/${dataset.version}") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala index 57eb11a9b..3709ead7d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala @@ -40,7 +40,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ @RequestBody upd: MenasObject[Array[DefaultValue]]): CompletableFuture[MappingTable] = { mappingTableService.updateDefaults(user.getUsername, upd.id.name, upd.id.version, upd.value.toList).map { - case Some(entity) => entity + case Some(entity) => entity._1 // v2 disregarding validation case None => throw notFound() } } @@ -51,7 +51,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ @RequestBody newDefault: MenasObject[DefaultValue]): CompletableFuture[MappingTable] = { mappingTableService.addDefault(user.getUsername, newDefault.id.name, newDefault.id.version, newDefault.value).map { - case Some(entity) => entity + case Some(entity) => entity._1 // v2 disregarding validation case None => throw notFound() } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala index 5fa14a03b..5ef13e650 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala @@ -44,6 +44,11 @@ class RestExceptionHandler { private val logger = LoggerFactory.getLogger(this.getClass) + @ExceptionHandler(value = Array(classOf[IllegalArgumentException])) + def handleIllegalArgumentException(exception: IllegalArgumentException): ResponseEntity[Any] = { + ResponseEntity.badRequest().body(exception.getMessage) + } + @ExceptionHandler(value = Array(classOf[AsyncRequestTimeoutException])) def handleAsyncRequestTimeoutException(exception: AsyncRequestTimeoutException): ResponseEntity[Any] = { val message = Option(exception.getMessage).getOrElse("Request timeout expired.") diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala index a29070477..9438a3d01 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala @@ -148,7 +148,7 @@ class SchemaController @Autowired()( // the parsing of sparkStruct can fail, therefore we try to save it first before saving the attachment update <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct) _ <- attachmentService.uploadAttachment(menasAttachment) - } yield update + } yield update.map(_._1) // v2 disregarding the validation } catch { case e: SchemaParsingException => throw e.copy(schemaType = schemaType) //adding schema type } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index dbe997478..e129a8b82 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers import java.util.Optional import java.util.concurrent.CompletableFuture - import com.mongodb.client.result.UpdateResult import org.springframework.http.HttpStatus import org.springframework.security.core.annotation.AuthenticationPrincipal @@ -29,6 +28,7 @@ import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService import za.co.absa.enceladus.model.menas.audit._ + abstract class VersionedModelController[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { @@ -39,7 +39,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @GetMapping(Array("/list", "/list/{searchQuery}")) @ResponseStatus(HttpStatus.OK) def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { - versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption) + versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) } @GetMapping(Array("/searchSuggestions")) @@ -114,7 +114,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @ResponseStatus(HttpStatus.CREATED) def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = { - versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { + versionedModelService.importSingleItemV2(importObject.item, principal.getUsername, importObject.metadata).map { case Some(entity) => entity case None => throw notFound() } @@ -130,7 +130,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au versionedModelService.create(item, principal.getUsername) } }.map { - case Some(entity) => entity + case Some((entity, validation)) => entity // v2 does not support validation-warnings on create case None => throw notFound() } } @@ -140,7 +140,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au def edit(@AuthenticationPrincipal user: UserDetails, @RequestBody item: C): CompletableFuture[C] = { versionedModelService.update(user.getUsername, item).map { - case Some(entity) => entity + case Some((entity, validation)) => entity // v2 disregarding validation on edit case None => throw notFound() } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala new file mode 100644 index 000000000..b56cf937a --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -0,0 +1,111 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.model.Validation +import za.co.absa.enceladus.model.conformanceRule.ConformanceRule +import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3 +import za.co.absa.enceladus.rest_api.utils.implicits._ + +import java.util.concurrent.CompletableFuture +import javax.servlet.http.HttpServletRequest +import scala.concurrent.ExecutionContext.Implicits.global + +@RestController +@RequestMapping(path = Array("/api-v3/datasets")) +class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) + extends VersionedModelControllerV3(datasetService) { + + @GetMapping(Array("/{name}/{version}/properties")) + @ResponseStatus(HttpStatus.OK) + def getAllPropertiesForVersion(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Map[String, String]] = { + forVersionExpression(name, version)(datasetService.getVersion).map { + case Some(entity) => entity.propertiesAsMap + case None => throw notFound() + } + } + + @PutMapping(Array("/{name}/{version}/properties")) + @ResponseStatus(HttpStatus.OK) + def updateProperties(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody newProperties: java.util.Map[String, String], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + forVersionExpression(name, version) { case (dsName, dsVersion) => + datasetService.updateProperties(principal.getUsername, dsName, dsVersion, newProperties.toScalaMap).map { + + case Some((entity, validation)) => + // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties + createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties") + .body(validation) // todo include in tests + case None => throw notFound() + } + } + } + + // todo putIntoInfoFile switch needed? + + @GetMapping(Array("/{name}/{version}/rules")) + @ResponseStatus(HttpStatus.OK) + def getConformanceRules(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Seq[ConformanceRule]] = { + forVersionExpression(name, version)(datasetService.getVersion).map { + case Some(entity) => entity.conformance + case None => throw notFound() + } + } + + @PostMapping(Array("/{name}/{version}/rules")) + @ResponseStatus(HttpStatus.CREATED) + def addConformanceRule(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: String, + @RequestBody rule: ConformanceRule, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + forVersionExpression(name, version)(datasetService.getVersion).flatMap { + case Some(entity) => datasetService.addConformanceRule(user.getUsername, name, entity.version, rule).map { + case Some((updatedDs, validation)) => + val addedRuleOrder = updatedDs.conformance.last.order + createdWithNameVersionLocationBuilder(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules + suffix = s"/rules/$addedRuleOrder").body(validation) + case _ => throw notFound() + } + case None => throw notFound() + } + } + + @GetMapping(Array("/{name}/{version}/rules/{order}")) + @ResponseStatus(HttpStatus.OK) + def getConformanceRuleByOrder(@PathVariable name: String, + @PathVariable version: String, + @PathVariable order: Int): CompletableFuture[ConformanceRule] = { + for { + optDs <- forVersionExpression(name, version)(datasetService.getVersion) + ds = optDs.getOrElse(throw notFound()) + rule = ds.conformance.find(_.order == order).getOrElse(throw notFound()) + } yield rule + } + +} + + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala new file mode 100644 index 000000000..41aea585e --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -0,0 +1,213 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import com.mongodb.client.result.UpdateResult +import org.springframework.http.{HttpStatus, ResponseEntity} +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails +import org.springframework.web.bind.annotation._ +import org.springframework.web.servlet.support.ServletUriComponentsBuilder +import za.co.absa.enceladus.model.menas.audit._ +import za.co.absa.enceladus.model.versionedModel._ +import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.controllers.BaseController +import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey +import za.co.absa.enceladus.rest_api.services.VersionedModelService + +import java.net.URI +import java.util +import java.util.Optional +import java.util.concurrent.CompletableFuture +import javax.servlet.http.HttpServletRequest +import scala.concurrent.Future +import scala.util.{Failure, Success, Try} + +object VersionedModelControllerV3 { + val LatestVersionKey = "latest" +} + +abstract class VersionedModelControllerV3[C <: VersionedModel with Product + with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { + + import za.co.absa.enceladus.rest_api.utils.implicits._ + + import scala.concurrent.ExecutionContext.Implicits.global + + // todo maybe offset/limit? + @GetMapping(Array("")) + @ResponseStatus(HttpStatus.OK) + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedLatestVersion]] = { + versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) + .map(_.map(_.toNamedLatestVersion)) + } + + @GetMapping(Array("/{name}")) + @ResponseStatus(HttpStatus.OK) + def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedLatestVersion] = { + versionedModelService.getLatestVersionSummary(name) map { + case Some(entity) => entity.toNamedLatestVersion + case None => throw notFound() + } + } + + @GetMapping(Array("/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + def getVersionDetail(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[C] = { + forVersionExpression(name, version)(versionedModelService.getVersion).map { + case Some(entity) => entity + case None => throw notFound() + } + } + + + @GetMapping(Array("/{name}/audit-trail")) + @ResponseStatus(HttpStatus.OK) + def getAuditTrail(@PathVariable name: String): CompletableFuture[AuditTrail] = { + versionedModelService.getAuditTrail(name) + } + + @GetMapping(Array("/{name}/{version}/used-in")) + @ResponseStatus(HttpStatus.OK) + def usedIn(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[UsedIn] = { + forVersionExpression(name, version) { case (name, versionInt) => versionedModelService.getUsedIn(name, Some(versionInt)) } + } + + @GetMapping(Array("/{name}/{version}/export")) + @ResponseStatus(HttpStatus.OK) + def exportSingleEntity(@PathVariable name: String, @PathVariable version: String): CompletableFuture[String] = { + forVersionExpression(name, version)(versionedModelService.exportSingleItem) + } + + @PostMapping(Array("/{name}/import")) + @ResponseStatus(HttpStatus.CREATED) + def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @RequestBody importObject: ExportableObject[C], + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + if (name != importObject.item.name) { + Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${importObject.item.name}'")) + } else { + versionedModelService.importSingleItemV3(importObject.item, principal.getUsername, importObject.metadata).map { + case Some((entity, validation)) => + // stripping two last segments, instead of /api-v3/dastasets/dsName/import + /dsName/dsVersion we want /api-v3/dastasets + /dsName/dsVersion + createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 2).body(validation) + case None => throw notFound() + } + } + } + + @GetMapping(Array("/{name}/{version}/validation")) + @ResponseStatus(HttpStatus.OK) + def validation(@PathVariable name: String, + @PathVariable version: String): CompletableFuture[Validation] = { + forVersionExpression(name, version)(versionedModelService.validate) + } + + @PostMapping(Array("")) + @ResponseStatus(HttpStatus.CREATED) + def create(@AuthenticationPrincipal principal: UserDetails, + @RequestBody item: C, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + versionedModelService.isDisabled(item.name).flatMap { isDisabled => + if (isDisabled) { + versionedModelService.recreate(principal.getUsername, item) + } else { + versionedModelService.create(item, principal.getUsername) + } + }.map { + case Some((entity, validation)) => createdWithNameVersionLocationBuilder(entity.name, entity.version, request).body(validation) + case None => throw notFound() + } + } + + @PutMapping(Array("/{name}/{version}")) + @ResponseStatus(HttpStatus.NO_CONTENT) + def edit(@AuthenticationPrincipal user: UserDetails, + @PathVariable name: String, + @PathVariable version: Int, + @RequestBody item: C, + request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { + + if (name != item.name) { + Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${item.name}'")) + } else if (version != item.version) { + Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) + } else { + versionedModelService.update(user.getUsername, item).map { + case Some((updatedEntity, validation)) => + createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) + case None => throw notFound() + } + } + } + + @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + def disable(@PathVariable name: String, + @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { + val v = if (version.isPresent) { + // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException + Some(version.get.toInt) + } else { + None + } + versionedModelService.disableVersion(name, v) + } + + /** + * For entity's name and version expression (either a number or 'latest'), the forVersionFn is called. + * + * @param name + * @param versionStr + * @param forVersionFn + * @return + */ + protected def forVersionExpression[T](name: String, versionStr: String) + (forVersionFn: (String, Int) => Future[T]): Future[T] = { + versionStr.toLowerCase match { + case LatestVersionKey => + versionedModelService.getLatestVersionValue(name).flatMap { + case None => Future.failed(notFound()) + case Some(actualLatestVersion) => forVersionFn(name, actualLatestVersion) + } + case nonLatestVersionString => Try(nonLatestVersionString.toInt) match { + case Success(actualVersion) => forVersionFn(name, actualVersion) + case Failure(exception) => + Future.failed(new IllegalArgumentException(s"Cannot convert '$versionStr' to a valid version expression. " + + s"Either use 'latest' or an actual version number. Underlying problem: ${exception.getMessage}")) + } + } + } + + protected def createdWithNameVersionLocationBuilder(name: String, version: Int, request: HttpServletRequest, + stripLastSegments: Int = 0, suffix: String = ""): ResponseEntity.BodyBuilder = { + val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString + + val location: URI = ServletUriComponentsBuilder.fromRequest(request) + .path(s"$strippingPrefix/{name}/{version}$suffix") + .buildAndExpand(name, version.toString) + .normalize // will normalize `/one/two/../three` into `/one/tree` + .toUri // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1 + + // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918 + + ResponseEntity.created(location) + } + +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index d59d7d6ad..90f9dd6e8 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -16,7 +16,6 @@ package za.co.absa.enceladus.rest_api.repositories import java.time.ZonedDateTime - import org.mongodb.scala._ import org.mongodb.scala.bson._ import org.mongodb.scala.bson.collection.immutable.Document @@ -62,7 +61,7 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.distinct[String]("name", getNotDisabledFilter).toFuture().map(_.sorted) } - def getLatestVersionsSummary(searchQuery: Option[String] = None): Future[Seq[VersionedSummary]] = { + def getLatestVersionsSummarySearch(searchQuery: Option[String] = None): Future[Seq[VersionedSummary]] = { val searchFilter = searchQuery match { case Some(search) => Filters.regex("name", search, "i") case None => Filters.expr(true) @@ -85,12 +84,22 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.find(getNameVersionFilter(name, Some(version))).headOption() } - def getLatestVersionValue(name: String): Future[Option[Int]] = { + /** + * Beware that this method ignores the disabled flag of the entities + */ + def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { val pipeline = Seq( filter(getNameFilter(name)), Aggregates.group("$name", Accumulators.max("latestVersion", "$version")) ) - collection.aggregate[VersionedSummary](pipeline).headOption().map(_.map(_.latestVersion)) + collection.aggregate[VersionedSummary](pipeline).headOption() + } + + /** + * Beware that this method ignores the disabled flag of the entities + */ + def getLatestVersionValue(name: String): Future[Option[Int]] = { + getLatestVersionSummary(name).map(_.map(_.latestVersion)) } def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { @@ -162,8 +171,8 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab val pipeline = Seq( filter(Filters.notEqual("disabled", true)), Aggregates.group("$name", - Accumulators.max("latestVersion", "$version"), - Accumulators.last("doc","$$ROOT")), + Accumulators.max("latestVersion", "$version"), + Accumulators.last("doc", "$$ROOT")), Aggregates.replaceRoot("$doc")) ++ postAggFilter.map(Aggregates.filter) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 368ffe6bb..e707c98e1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -28,6 +28,7 @@ import za.co.absa.enceladus.model.properties.essentiality.Mandatory import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel import DatasetService._ +import za.co.absa.enceladus.rest_api.exceptions.{NotFoundException, ValidationException} import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import scala.concurrent.Future @@ -43,7 +44,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository import scala.concurrent.ExecutionContext.Implicits.global - override def update(username: String, dataset: Dataset): Future[Option[Dataset]] = { + override def update(username: String, dataset: Dataset): Future[Option[(Dataset, Validation)]] = { super.updateFuture(username, dataset.name, dataset.version) { latest => updateSchedule(dataset, latest).map({ withSchedule => withSchedule @@ -52,7 +53,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository .setHDFSPath(dataset.hdfsPath) .setHDFSPublishPath(dataset.hdfsPublishPath) .setConformance(dataset.conformance) - .setProperties(removeBlankProperties(dataset.properties)) + .setProperties(removeBlankPropertiesOpt(dataset.properties)) .setDescription(dataset.description).asInstanceOf[Dataset] }) } @@ -93,10 +94,18 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { - Future.successful(UsedIn()) + val existingEntity = version match { + case Some(version) => getVersion(name, version) + case None => getLatestVersion(name) + } + + existingEntity.flatMap { + case Some(_) => Future.successful(UsedIn()) // empty usedIn for existing datasets + case None => Future.failed(NotFoundException(s"Dataset '$name' in version ${version.getOrElse("any")}' not found")) + } } - override def create(newDataset: Dataset, username: String): Future[Option[Dataset]] = { + override def create(newDataset: Dataset, username: String): Future[Option[(Dataset, Validation)]] = { val dataset = Dataset(name = newDataset.name, description = newDataset.description, hdfsPath = newDataset.hdfsPath, @@ -104,24 +113,26 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository schemaName = newDataset.schemaName, schemaVersion = newDataset.schemaVersion, conformance = List(), - properties = removeBlankProperties(newDataset.properties)) + properties = removeBlankPropertiesOpt(newDataset.properties)) super.create(dataset, username) } - def addConformanceRule(username: String, datasetName: String, datasetVersion: Int, rule: ConformanceRule): Future[Option[Dataset]] = { + def addConformanceRule(username: String, datasetName: String, datasetVersion: Int, + rule: ConformanceRule): Future[Option[(Dataset, Validation)]] = { update(username, datasetName, datasetVersion) { dataset => dataset.copy(conformance = dataset.conformance :+ rule) } } - def replaceProperties(username: String, datasetName: String, - updatedProperties: Option[Map[String, String]]): Future[Option[Dataset]] = { + // kept for API v2 usage only + def updateProperties(username: String, datasetName: String, + updatedProperties: Option[Map[String, String]]): Future[Option[Dataset]] = { for { latestVersion <- getLatestVersionNumber(datasetName) update <- update(username, datasetName, latestVersion) { latest => - latest.copy(properties = removeBlankProperties(updatedProperties)) + latest.copy(properties = removeBlankPropertiesOpt(updatedProperties)) } - } yield update + } yield update.map(_._1) // v2 does not expect validation on update } private def validateExistingProperty(key: String, value: String, @@ -196,7 +207,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } - def validateProperties(properties: Map[String, String], forRun: Boolean): Future[Validation] = { + def validateProperties(properties: Map[String, String], forRun: Boolean = false): Future[Validation] = { datasetPropertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => val propDefsMap = Map(propDefs.map { propDef => (propDef.name, propDef) }: _*) // map(key, propDef) @@ -219,7 +230,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository def getLatestVersions(missingProperty: Option[String]): Future[Seq[Dataset]] = datasetMongoRepository.getLatestVersions(missingProperty) - override def importItem(item: Dataset, username: String): Future[Option[Dataset]] = { + override def importItem(item: Dataset, username: String): Future[Option[(Dataset, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) @@ -260,6 +271,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } + // CR-related methods: private def validateConformanceRules(conformanceRules: List[ConformanceRule], maybeSchema: Future[Option[Schema]]): Future[Validation] = { @@ -426,12 +438,22 @@ object DatasetService { * @param properties original properties * @return properties without empty-string value entries */ - def removeBlankProperties(properties: Option[Map[String, String]]): Option[Map[String, String]] = { + private[services] def removeBlankPropertiesOpt(properties: Option[Map[String, String]]): Option[Map[String, String]] = { properties.map { - _.filter { case (_, propValue) => propValue.nonEmpty } + removeBlankProperties } } + /** + * Removes properties having empty-string value. Effectively mapping such properties' values from Some("") to None. + * This is Backend-implementation related to DatasetService.replaceBlankProperties(dataset) on Frontend + * @param properties original properties + * @return properties without empty-string value entries + */ + private[services] def removeBlankProperties(properties: Map[String, String]): Map[String, String] = { + properties.filter { case (_, propValue) => propValue.nonEmpty } + } + private[services] def replacePrefixIfFound(fieldName: String, replacement: String, lookFor: String): Option[String] = { fieldName match { case `lookFor` => Some(replacement) // exact match diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala index f5c085b1a..9bcecfe86 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala @@ -38,7 +38,7 @@ class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTabl used.map(refs => UsedIn(Some(refs), None)) } - override def create(mt: MappingTable, username: String): Future[Option[MappingTable]] = { + override def create(mt: MappingTable, username: String): Future[Option[(MappingTable, Validation)]] = { val mappingTable = MappingTable(name = mt.name, description = mt.description, schemaName = mt.schemaName, @@ -48,19 +48,19 @@ class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTabl super.create(mappingTable, username) } - def updateDefaults(username: String, mtName: String, mtVersion: Int, defaultValues: List[DefaultValue]): Future[Option[MappingTable]] = { + def updateDefaults(username: String, mtName: String, mtVersion: Int, defaultValues: List[DefaultValue]): Future[Option[(MappingTable, Validation)]] = { super.update(username, mtName, mtVersion) { latest => latest.setDefaultMappingValue(defaultValues) } } - def addDefault(username: String, mtName: String, mtVersion: Int, defaultValue: DefaultValue): Future[Option[MappingTable]] = { + def addDefault(username: String, mtName: String, mtVersion: Int, defaultValue: DefaultValue): Future[Option[(MappingTable, Validation)]] = { super.update(username, mtName, mtVersion) { latest => latest.setDefaultMappingValue(latest.defaultMappingValue :+ defaultValue) } } - override def update(username: String, mt: MappingTable): Future[Option[MappingTable]] = { + override def update(username: String, mt: MappingTable): Future[Option[(MappingTable, Validation)]] = { super.update(username, mt.name, mt.version) { latest => latest .setHDFSPath(mt.hdfsPath) @@ -71,7 +71,7 @@ class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTabl } } - override def importItem(item: MappingTable, username: String): Future[Option[MappingTable]] = { + override def importItem(item: MappingTable, username: String): Future[Option[(MappingTable, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala index ffbac0e04..7b4e5a039 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala @@ -18,7 +18,7 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import za.co.absa.enceladus.rest_api.repositories.PropertyDefinitionMongoRepository -import za.co.absa.enceladus.model.UsedIn +import za.co.absa.enceladus.model.{UsedIn, Validation} import za.co.absa.enceladus.model.properties.PropertyDefinition import scala.concurrent.Future @@ -31,7 +31,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = Future.successful(UsedIn()) - override def update(username: String, propertyDef: PropertyDefinition): Future[Option[PropertyDefinition]] = { + override def update(username: String, propertyDef: PropertyDefinition): Future[Option[(PropertyDefinition, Validation)]] = { super.update(username, propertyDef.name, propertyDef.version) { latest => latest .setPropertyType(propertyDef.propertyType) @@ -45,7 +45,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert propertyDefMongoRepository.distinctCount() } - override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[PropertyDefinition]] = { + override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[(PropertyDefinition, Validation)]] = { val propertyDefBase = PropertyDefinition( name = newPropertyDef.name, description = newPropertyDef.description, @@ -63,7 +63,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert super.create(propertyDefinition, username) } - override private[services] def importItem(item: PropertyDefinition, username: String): Future[Option[PropertyDefinition]] = { + override private[services] def importItem(item: PropertyDefinition, username: String): Future[Option[(PropertyDefinition, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index 4eb64a691..684f8907b 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -17,7 +17,7 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.model.{Schema, UsedIn} +import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} import scala.concurrent.Future @@ -39,13 +39,13 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, } yield UsedIn(Some(usedInD), Some(usedInM)) } - def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[Option[Schema]] = { + def schemaUpload(username: String, schemaName: String, schemaVersion: Int, fields: StructType): Future[Option[(Schema, Validation)]] = { super.update(username, schemaName, schemaVersion)({ oldSchema => oldSchema.copy(fields = sparkMenasConvertor.convertSparkToMenasFields(fields.fields).toList) }) } - override def recreate(username: String, schema: Schema): Future[Option[Schema]] = { + override def recreate(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { for { latestVersion <- getLatestVersionNumber(schema.name) update <- super.update(username, schema.name, latestVersion) { latest => @@ -56,19 +56,19 @@ class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, } yield update } - override def update(username: String, schema: Schema): Future[Option[Schema]] = { + override def update(username: String, schema: Schema): Future[Option[(Schema, Validation)]] = { super.update(username, schema.name, schema.version) { latest => latest.setDescription(schema.description).asInstanceOf[Schema] } } - override def create(newSchema: Schema, username: String): Future[Option[Schema]] = { + override def create(newSchema: Schema, username: String): Future[Option[(Schema, Validation)]] = { val schema = Schema(name = newSchema.name, description = newSchema.description) super.create(schema, username) } - override def importItem(item: Schema, username: String): Future[Option[Schema]] = { + override def importItem(item: Schema, username: String): Future[Option[(Schema, Validation)]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) case None => super.create(item.copy(version = 1), username) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index b11fae9d0..eb6ef28cc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -28,16 +28,18 @@ import za.co.absa.enceladus.model.menas.audit._ import scala.concurrent.Future import com.mongodb.MongoWriteException +import VersionedModelService._ +// scalastyle:off number.of.methods abstract class VersionedModelService[C <: VersionedModel with Product with Auditable[C]] - (versionedMongoRepository: VersionedMongoRepository[C]) extends ModelService(versionedMongoRepository) { +(versionedMongoRepository: VersionedMongoRepository[C]) extends ModelService(versionedMongoRepository) { import scala.concurrent.ExecutionContext.Implicits.global private[services] val logger = LoggerFactory.getLogger(this.getClass) - def getLatestVersionsSummary(searchQuery: Option[String]): Future[Seq[VersionedSummary]] = { - versionedMongoRepository.getLatestVersionsSummary(searchQuery) + def getLatestVersionsSummarySearch(searchQuery: Option[String]): Future[Seq[VersionedSummary]] = { + versionedMongoRepository.getLatestVersionsSummarySearch(searchQuery) } def getLatestVersions(): Future[Seq[C]] = { @@ -74,6 +76,10 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getLatestVersionValue(name) } + def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { + versionedMongoRepository.getLatestVersionSummary(name) + } + def exportSingleItem(name: String, version: Int): Future[String] = { getVersion(name, version).flatMap({ case Some(item) => Future(item.exportItem()) @@ -88,7 +94,8 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit }) } - def importSingleItem(item: C, username: String, metadata: Map[String, String]): Future[Option[C]] = { + // v2 has external validate validation applied only to imports (not create/edits) via validateSingleImport + def importSingleItemV2(item: C, username: String, metadata: Map[String, String]): Future[Option[C]] = { for { validation <- validateSingleImport(item, metadata) result <- { @@ -98,7 +105,12 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit throw ValidationException(validation) } } - } yield result + } yield result.map(_._1) // v disregards internal common update-based validation + } + + // v3 has internal validation on importItem (because it is based on update + def importSingleItemV3(item: C, username: String, metadata: Map[String, String]): Future[Option[(C, Validation)]] = { + importItem(item, username) } private[services] def validateSingleImport(item: C, metadata: Map[String, String]): Future[Validation] = { @@ -122,7 +134,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit ) } - private[services] def importItem(item: C, username: String): Future[Option[C]] + private[services] def importItem(item: C, username: String): Future[Option[(C, Validation)]] private[services] def validateSchema(schemaName: String, schemaVersion: Int, @@ -168,11 +180,11 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit val allParents = getParents(name) allParents.flatMap({ parents => - val msgs = if(parents.size < 2) Seq() else { + val msgs = if (parents.size < 2) Seq() else { val pairs = parents.sliding(2) pairs.map(p => p.head.getAuditMessages(p(1))).toSeq } - if(parents.isEmpty) { + if (parents.isEmpty) { this.getLatestVersion(name).map({ case Some(entity) => AuditTrail(msgs.reverse :+ entity.createdMessage) case None => throw NotFoundException() @@ -189,9 +201,13 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit MenasReference(Some(versionedMongoRepository.collectionBaseName), item.name, item.version) } - private[rest_api] def create(item: C, username: String): Future[Option[C]] = { + private[rest_api] def create(item: C, username: String): Future[Option[(C, Validation)]] = { + // individual validations are deliberately not run in parallel - the latter may not be needed if the former fails for { - validation <- validate(item) + validation <- for { + generalValidation <- validate(item) + creationValidation <- validateForCreation(item) + } yield generalValidation.merge(creationValidation) _ <- if (validation.isValid) { versionedMongoRepository.create(item, username) .recover { @@ -202,39 +218,45 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit throw ValidationException(validation) } detail <- getLatestVersion(item.name) - } yield detail + } yield detail.map(d => (d, validation)) // valid validation may contain warnings } - def recreate(username: String, item: C): Future[Option[C]] = { + def recreate(username: String, item: C): Future[Option[(C, Validation)]] = { for { latestVersion <- getLatestVersionNumber(item.name) update <- update(username, item.setVersion(latestVersion).asInstanceOf[C]) } yield update } - def update(username: String, item: C): Future[Option[C]] + def update(username: String, item: C): Future[Option[(C, Validation)]] - private[services] def updateFuture(username: String, itemName: String, itemVersion: Int)(transform: C => Future[C]): Future[Option[C]] = { + private[services] def updateFuture(username: String, itemName: String, itemVersion: Int)(transform: C => Future[C]): Future[Option[(C, Validation)]] = { for { versionToUpdate <- getLatestVersion(itemName) - transformed <- if (versionToUpdate.isEmpty) { + (transformed, transformedValidation) <- if (versionToUpdate.isEmpty) { Future.failed(NotFoundException(s"Version $itemVersion of $itemName not found")) } else if (versionToUpdate.get.version != itemVersion) { Future.failed(ValidationException(Validation().withError("version", s"Version $itemVersion of $itemName is not the latest version, therefore cannot be edited"))) - } - else { - transform(versionToUpdate.get) + } else { + for { + updatedEntity <- transform(versionToUpdate.get) + validation <- validate(updatedEntity) + } yield if (validation.isValid) { + (updatedEntity, validation) // successful outcome, validation may still hold warnings + } else { + throw ValidationException(validation) + } } update <- versionedMongoRepository.update(username, transformed) .recover { case e: MongoWriteException => throw ValidationException(Validation().withError("version", s"entity '$itemName' with this version already exists: ${itemVersion + 1}")) } - } yield Some(update) + } yield Some((update, transformedValidation)) } - private[services] def update(username: String, itemName: String, itemVersion: Int)(transform: C => C): Future[Option[C]] = { - this.updateFuture(username, itemName, itemVersion){ item: C => + private[services] def update(username: String, itemName: String, itemVersion: Int)(transform: C => C): Future[Option[(C, Validation)]] = { + this.updateFuture(username, itemName, itemVersion) { item: C => Future { transform(item) } @@ -266,30 +288,60 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.isDisabled(name) } + /** + * Retrieves model@version and calls + * [[za.co.absa.enceladus.rest_api.services.VersionedModelService#validate(java.lang.Object)]] + * + * In order to extend this behavior, override the mentioned method instead. (that's why this is `final`) + * + * @param name + * @param version + * @return + */ + final def validate(name: String, version: Int): Future[Validation] = { + getVersion(name, version).flatMap({ + case Some(entity) => validate(entity) + case _ => Future.failed(NotFoundException(s"Entity by name=$name, version=$version is not found!")) + }) + } + + /** + * Provides common validation (currently entity name validation). Override to extend for further specific validations. + * + * @param item + * @return + */ def validate(item: C): Future[Validation] = { validateName(item.name) } - protected[services] def validateName(name: String): Future[Validation] = { - val validation = Validation() + def validateForCreation(item: C): Future[Validation] = { + isUniqueName(item.name).map { isUnique => + if (isUnique) { + Validation.empty + } else { + Validation.empty.withError("name", s"entity with name already exists: '${item.name}'") + } + } + } + protected[services] def validateName(name: String): Future[Validation] = { if (hasWhitespace(name)) { - Future.successful(validation.withError("name", s"name contains whitespace: '$name'")) + Future.successful(Validation.empty.withError("name", s"name contains whitespace: '$name'")) } else { - isUniqueName(name).map { isUnique => - if (isUnique) { - validation - } else { - validation.withError("name", s"entity with name already exists: '$name'") - } - } + Future.successful(Validation.empty) } } +} + +object VersionedModelService { private[services] def hasWhitespace(name: String): Boolean = Option(name).exists(definedName => !definedName.matches("""\w+""")) + private[services] def hasValidNameChars(name: String): Boolean = Option(name).exists(definedName => definedName.matches("""[a-zA-Z0-9._-]+""")) + private[services] def hasValidApiVersion(version: Option[String]): Boolean = version.contains(ModelVersion.toString) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala new file mode 100644 index 000000000..f5dff3127 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} +import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.rest_api.exceptions.ValidationException +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} +import za.co.absa.enceladus.rest_api.services.DatasetService._ +import za.co.absa.enceladus.rest_api.services.{DatasetService, MappingTableService, PropertyDefinitionService, SchemaService} + +import scala.concurrent.Future + +// this DatasetService is a V3 difference wrapper - once V2 is removed, implementations can/should be merged +@Service +class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoRepository, + oozieRepository: OozieRepository, + datasetPropertyDefinitionService: PropertyDefinitionService, + mappingTableService: MappingTableService, + val schemaService: SchemaService) + extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) + with HavingSchemaService { + + import scala.concurrent.ExecutionContext.Implicits.global + + def validateRules(item: Dataset): Future[Validation] = { + val validationsFutList: Seq[Future[Validation]] = item.conformance.map { + case r: MappingConformanceRule => + mappingTableService.getVersion(r.mappingTable, r.mappingTableVersion).map { + case Some(_) => Validation.empty //MT exists + case None => Validation.empty.withError("mapping-table", s"Mapping table ${r.mappingTable} v${r.mappingTableVersion} not found!") + } + case _ => Future.successful(Validation.empty) // no other validations besides mapping CRs + } + + Future.sequence(validationsFutList).map { listOfVals => + listOfVals.foldLeft(Validation.empty)(_ merge _) + } + } + + // general entity validation is extendable for V3 - here with properties validation + override def validate(item: Dataset): Future[Validation] = { + // individual validations are deliberately not run in parallel - the latter may not be needed if the former fails + for { + originalValidation <- super.validate(item) + propertiesValidation <- validateProperties(item.propertiesAsMap) + schemaValidation <- validateSchemaExists(item.schemaName, item.schemaVersion) + rulesValidation <- validateRules(item) + } yield originalValidation.merge(propertiesValidation).merge(schemaValidation).merge(rulesValidation) + } + + override def addConformanceRule(username: String, datasetName: String, + datasetVersion: Int, rule: ConformanceRule): Future[Option[(Dataset, Validation)]] = { + update(username, datasetName, datasetVersion) { dataset => + val existingRuleOrders = dataset.conformance.map(_.order).toSet + if (!existingRuleOrders.contains(rule.order)) { + dataset.copy(conformance = dataset.conformance :+ rule) // adding the rule + } else { + throw new IllegalArgumentException(s"Rule with order ${rule.order} cannot be added, another rule with this order already exists.") + } + } + } + + def updateProperties(username: String, datasetName: String, datasetVersion: Int, + updatedProperties: Map[String, String]): Future[Option[(Dataset, Validation)]] = { + for { + successfulValidation <- validateProperties(updatedProperties).flatMap { + case validation if !validation.isValid => Future.failed(ValidationException(validation)) // warnings are ok for update + case validation => Future.successful(validation) // empty or with warnings + } + + // updateFuture includes latest-check and version increase + update <- updateFuture(username, datasetName, datasetVersion) { latest => + Future.successful(latest.copy(properties = Some(removeBlankProperties(updatedProperties)))) + } + } yield update + } + +} + + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala new file mode 100644 index 000000000..498721d8b --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import za.co.absa.enceladus.model.Validation +import za.co.absa.enceladus.rest_api.services.SchemaService + +import scala.concurrent.{ExecutionContext, Future} + +trait HavingSchemaService { + protected def schemaService: SchemaService + + def validateSchemaExists(schemaName: String, schemaVersion: Int) + (implicit executionContext: ExecutionContext): Future[Validation] = { + schemaService.getVersion(schemaName, schemaVersion).map { + case None => Validation.empty.withError("schema", s"Schema $schemaName v$schemaVersion not found!") + case Some(_) => Validation.empty + } + } + +} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala index 209497733..43fd206b1 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/AuthenticationIntegrationSuite.scala @@ -29,7 +29,7 @@ import scala.concurrent.{Await, Future} @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class AuthenticationIntegrationSuite extends BaseRestApiTest { +class AuthenticationIntegrationSuite extends BaseRestApiTestV2 { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 87fbcb3dc..ab2b8aa1e 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -32,7 +32,10 @@ import za.co.absa.enceladus.rest_api.integration.repositories.BaseRepositoryTest import scala.concurrent.Future import scala.reflect.ClassTag -abstract class BaseRestApiTest extends BaseRepositoryTest { +abstract class BaseRestApiTestV2 extends BaseRestApiTest("/api/login", "/api") +abstract class BaseRestApiTestV3 extends BaseRestApiTest("/api/login", "/api-v3") + +abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseRepositoryTest { import scala.concurrent.ExecutionContext.Implicits.global @@ -50,7 +53,9 @@ abstract class BaseRestApiTest extends BaseRepositoryTest { @Value("${menas.auth.inmemory.admin.password}") val adminPasswd: String = "" - private lazy val baseUrl = s"http://localhost:$port/api" + // expecting apiPath to be /api for v2 and /api-v3 for v3 + private lazy val baseUrl = s"http://localhost:$port$apiPath" + private lazy val loginBaseUrl = s"http://localhost:$port$loginPath" private lazy val authHeaders = getAuthHeaders(user, passwd) private lazy val authHeadersAdmin = getAuthHeaders(adminUser, adminPasswd) @@ -71,7 +76,7 @@ abstract class BaseRestApiTest extends BaseRepositoryTest { } def getAuthHeaders(username: String, password: String): HttpHeaders = { - val loginUrl = s"$baseUrl/login?username=$username&password=$password&submit=Login" + val loginUrl = s"$loginBaseUrl?username=$username&password=$password&submit=Login" val response = restTemplate.postForEntity(loginUrl, HttpEntity.EMPTY, classOf[String]) @@ -249,4 +254,6 @@ abstract class BaseRestApiTest extends BaseRepositoryTest { assert(responseEntity.getStatusCode == HttpStatus.CREATED) } + def stripBaseUrl(fullUrl: String): String = fullUrl.stripPrefix(baseUrl) + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala index 82dd8717d..837d7d1c2 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala @@ -35,7 +35,7 @@ import za.co.absa.enceladus.rest_api.integration.fixtures._ @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetApiIntegrationSuite extends BaseRestApiTest with BeforeAndAfterAll { +class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll { @Autowired private val datasetFixture: DatasetFixtureService = null diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala index 7c194a8d0..c8b9ad592 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala @@ -31,7 +31,7 @@ import za.co.absa.enceladus.model.test.factories.PropertyDefinitionFactory @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTest with BeforeAndAfterAll with Matchers { +class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll with Matchers { @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala index 51e383391..f17e17905 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/RunApiIntegrationSuite.scala @@ -30,7 +30,7 @@ import za.co.absa.enceladus.model.{Run, SplineReference, Validation} @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class RunApiIntegrationSuite extends BaseRestApiTest { +class RunApiIntegrationSuite extends BaseRestApiTestV2 { import za.co.absa.enceladus.rest_api.integration.RunImplicits.RunExtensions import za.co.absa.enceladus.model.Validation._ diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala index 15a510f34..6251e5f71 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala @@ -47,7 +47,7 @@ import scala.collection.immutable.HashMap @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTest with BeforeAndAfterAll { +class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll { private val port = 8877 // same port as in test/resources/application.conf in the `menas.schemaRegistry.baseUrl` key private val wireMockServer = new WireMockServer(WireMockConfiguration.wireMockConfig().port(port)) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala new file mode 100644 index 000000000..e0f523000 --- /dev/null +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -0,0 +1,927 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.integration.controllers.v3 + +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.HttpStatus +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, LiteralConformanceRule, MappingConformanceRule} +import za.co.absa.enceladus.model.dataFrameFilter._ +import za.co.absa.enceladus.model.properties.essentiality.Essentiality +import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} +import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion +import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} +import za.co.absa.enceladus.rest_api.integration.fixtures._ + +import scala.collection.JavaConverters._ + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAndAfterAll with Matchers { + + @Autowired + private val datasetFixture: DatasetFixtureService = null + + @Autowired + private val schemaFixture: SchemaFixtureService = null + + @Autowired + private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + + @Autowired + private val mappingTableFixture: MappingTableFixtureService = null + + private val apiUrl = "/datasets" + + // fixtures are cleared after each test + override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture, mappingTableFixture) + + + s"POST $apiUrl" should { + "return 201" when { + "a Dataset is created" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset = DatasetFactory.getDummyDataset("dummyDs", + properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> ""))) + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) + ) + + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + assertCreated(response) + response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/dummyDs/1") + + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(dataset, actual).copy(properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))) // keyC stripped + + assert(actual == expected) + } + "create a new version of Dataset" when { + "the dataset is disabled (i.e. all version are disabled)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) + datasetFixture.add(dataset1, dataset2) + + val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create + val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) + assertCreated(response) + val locationHeaders = response.getHeaders.get("location").asScala + locationHeaders should have size 1 + val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. + + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) + + assert(actual == expected) + } + } + } + + "return 400" when { + "dataset schema does not exits" in { + val dataset = DatasetFactory.getDummyDataset("dummyDs") + // there are schemas defined + + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("schema" -> List("Schema dummySchema v1 not found!"))) + } + + "datasets properties are not backed by propDefs (undefined properties)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset = DatasetFactory.getDummyDataset("dummyDs", properties = Some(Map("undefinedProperty1" -> "value1"))) + // propdefs are empty + + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) + } + } + // todo what to do if "the last dataset version is disabled"? + } + + s"GET $apiUrl/{name}" should { + "return 200" when { + "a Dataset with the given name exists - so it gives versions" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", + version = 2, + parent = Some(DatasetFactory.toParent(datasetV1))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[NamedLatestVersion](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == NamedLatestVersion("datasetA", 2)) + } + } + + "return 404" when { + "a Dataset with the given name does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName") + assertNotFound(response) + } + } + } + + s"GET $apiUrl/{name}/latest" should { + "return 200" when { + "a Dataset with the given name exists - gives latest version entity" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", + version = 2, parent = Some(DatasetFactory.toParent(datasetV1))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[Dataset](s"$apiUrl/datasetA/latest") + assertOk(response) + + val actual = response.getBody + val expected = toExpected(datasetV2, actual) + + assert(actual == expected) + } + } + + "return 404" when { + "a Dataset with the given name does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName/latest") + assertNotFound(response) + } + } + } + + s"GET $apiUrl/{name}/{version}" should { + "return 200" when { + "a Dataset with the given name and version exists - gives specified version of entity" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, description = Some("second")) + val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3, description = Some("third")) + datasetFixture.add(datasetV1, datasetV2, datasetV3) + + val response = sendGet[Dataset](s"$apiUrl/datasetA/2") + assertOk(response) + + val actual = response.getBody + val expected = toExpected(datasetV2, actual) + + assert(actual == expected) + } + } + + "return 404" when { + "a Dataset with the given name/version does not exist" in { + val dataset = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(dataset) + + val response = sendGet[String](s"$apiUrl/anotherDatasetName/1") + assertNotFound(response) + + val response2 = sendGet[String](s"$apiUrl/datasetA/7") + assertNotFound(response2) + } + } + } + + private val exampleMappingCr = MappingConformanceRule(0, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable", + mappingTableVersion = 9, //scalastyle:ignore magic.number + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + + s"PUT $apiUrl/{name}/{version}" can { + "return 200" when { + "a Dataset with the given name and version is the latest that exists" should { + "update the dataset (with empty properties stripped)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + + val datasetA1 = DatasetFactory.getDummyDataset("datasetA", + description = Some("init version"), properties = Some(Map("keyA" -> "valA"))) + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + description = Some("second version"), properties = Some(Map("keyA" -> "valA")), version = 2) + datasetFixture.add(datasetA1, datasetA2) + + Seq("keyA", "keyB", "keyC").foreach {propName => propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) + )} + // this will cause missing property 'keyD' to issue a warning if not present + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) + ) + + mappingTableFixture.add(MappingTableFactory.getDummyMappingTable("CurrencyMappingTable", version = 9)) //scalastyle:ignore magic.number + + val datasetA3 = DatasetFactory.getDummyDataset("datasetA", + description = Some("updated"), + properties = Some(Map("keyA" -> "valA", "keyB" -> "valB", "keyC" -> "")), + conformance = List(exampleMappingCr), + version = 2 // update references the last version + ) + + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/2", bodyOpt = Some(datasetA3)) + assertCreated(response) + response.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetA/3") + + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expected = toExpected(datasetA3.copy(version = 3, parent = Some(DatasetFactory.toParent(datasetA2)), properties = Some(Map("keyA" -> "valA", "keyB" -> "valB"))), actual) // blank property stripped + + assert(actual == expected) + } + } + } + + "return 400" when { + "when properties are not backed by propDefs (undefined properties) and schema does not exist" in { + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetA1) + // propdefs are empty, schemas not defined + + val datasetA2 = DatasetFactory.getDummyDataset("datasetA", + description = Some("second version"), properties = Some(Map("keyA" -> "valA"))) // version in payload is irrelevant + + val response = sendPut[Dataset, Validation](s"$apiUrl/datasetA/1", bodyOpt = Some(datasetA2)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe + Validation(Map( + "schema" -> List("Schema dummySchema v1 not found!"), + "keyA" -> List("There is no property definition for key 'keyA'.") + )) + } + + "a Dataset with the given name and version" should { + "fail when version/name in the URL and payload is mismatched" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetA1 = DatasetFactory.getDummyDataset("datasetA", description = Some("init version")) + datasetFixture.add(datasetA1) + + val response = sendPut[Dataset, String](s"$apiUrl/datasetA/7", + bodyOpt = Some(DatasetFactory.getDummyDataset("datasetA", version = 5))) + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody should include("version mismatch: 7 != 5") + + val response2 = sendPut[Dataset, String](s"$apiUrl/datasetABC/4", + bodyOpt = Some(DatasetFactory.getDummyDataset("datasetXYZ", version = 4))) + response2.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response2.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") + } + } + } + } + + s"GET $apiUrl/{name}/audit-trail" should { + "return 404" when { + "when the name does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/audit-trail") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct Dataset" should { + "return an audit trail for the dataset" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA") + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), + parent = Some(DatasetFactory.toParent(dataset1)) + ) + + datasetFixture.add(dataset1, dataset2) + val response = sendGet[String](s"$apiUrl/datasetA/audit-trail") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"entries":[{ + |"menasRef":{"collection":null,"name":"datasetA","version":2}, + |"updatedBy":"dummyUser","updated":"2017-12-04T16:19:17Z", + |"changes":[{"field":"conformance","oldValue":null,"newValue":"LiteralConformanceRule(0,outputCol1,false,litValue1)","message":"Conformance rule added."}] + |},{ + |"menasRef":{"collection":null,"name":"datasetA","version":1}, + |"updatedBy":"dummyUser","updated":"2017-12-04T16:19:17Z", + |"changes":[{"field":"","oldValue":null,"newValue":null,"message":"Dataset datasetA created."}] + |}]}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + } + + s"POST $apiUrl/{name}/import" should { + val importableDs = + """{"metadata":{"exportVersion":1},"item":{ + |"name":"datasetXYZ", + |"description":"Hi, I am the import", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "") + + "return 400" when { + "a Dataset with the given name" should { + "fail when name in the URL and payload is mismatched" in { + val response = sendPost[String, String](s"$apiUrl/datasetABC/import", + bodyOpt = Some(importableDs)) + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") + } + } + } + + "return 400" when { + "imported Dataset fails validation" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing + + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody shouldBe Validation.empty.withError("key2", "There is no property definition for key 'key2'.") + } + } + + "return 201" when { + "there is a existing Dataset" should { + "a +1 version of dataset is added" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("init version")) + datasetFixture.add(dataset1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key3", essentiality = Essentiality.Recommended) + ) + + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetXYZ/2") + response.getBody shouldBe Validation.empty.withWarning("key3", "Property 'key3' is recommended to be present, but was not found!") + + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = DatasetFactory.getDummyDataset(name = "datasetXYZ", version = 2, description = Some("Hi, I am the import"), + properties = Some(Map("key1" -> "val1", "key2" -> "val2")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")), + parent = Some(DatasetFactory.toParent(dataset1)) + ) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + + "there is no such Dataset, yet" should { + "a the version of dataset created" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) + + val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + assertCreated(response) + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version + + val relativeLocation = stripBaseUrl(locationHeader) // because locationHeader contains domain, port, etc. + val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("Hi, I am the import"), + properties = Some(Map("key1" -> "val1", "key2" -> "val2")), + conformance = List(LiteralConformanceRule(0, "outputCol1", controlCheckpoint = false, "litValue1")) + ) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + } + + } + + s"GET $apiUrl/{name}/{version}/export" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/2/export") + assertNotFound(response) + } + } + + "return 200" when { + "there is a correct Dataset version" should { + "return the exported Dataset representation" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2, + properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset", version = 3, description = Some("showing non-latest export")) + datasetFixture.add(dataset2, dataset3) + val response = sendGet[String](s"$apiUrl/dataset/2/export") + + assertOk(response) + + val body = response.getBody + assert(body == + """{"metadata":{"exportVersion":1},"item":{ + |"name":"dataset", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "")) + } + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 404" when { + "when the dataset of latest version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/latest/used-in") + assertNotFound(response) + } + } + + "return 404" when { + "when the dataset of name/version does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetA) + + val response = sendGet[String](s"$apiUrl/notFoundDataset/1/used-in") + assertNotFound(response) + + val response2 = sendGet[String](s"$apiUrl/datasetA/7/used-in") + assertNotFound(response2) + } + } + + "return 200" when { + "any exiting latest dataset" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetA) + val response = sendGet[UsedIn](s"$apiUrl/datasetA/latest/used-in") + assertOk(response) + + response.getBody shouldBe UsedIn(None, None) + } + } + + "return 200" when { + "for existing name+version for dataset" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2) + datasetFixture.add(dataset2) + val response = sendGet[UsedIn](s"$apiUrl/dataset/2/used-in") + + assertOk(response) + response.getBody shouldBe UsedIn(None, None) + } + } + } + + s"GET $apiUrl/{name}/{version}/properties" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/properties") + assertNotFound(response) + } + } + + "return 200" when { + "there is a specific Dataset version" should { + Seq( + ("empty1", Some(Map.empty[String, String])), + ("empty2", None), + ("non-empty", Some(Map("key1" -> "val1", "key2" -> "val2"))) + ).foreach { case (propertiesCaseName, propertiesData) => + s"return dataset properties ($propertiesCaseName)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, properties = propertiesData) + val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3, properties = Some(Map("other" -> "prop"))) + datasetFixture.add(datasetV1, datasetV2, datasetV3) + + val response = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") + assertOk(response) + + val expectedProperties = propertiesData.getOrElse(Map.empty[String, String]) + val body = response.getBody + assert(body == expectedProperties) + } + } + } + } + + "return 200" when { + "there is a latest Dataset version" should { + s"return dataset properties" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[Map[String, String]](s"$apiUrl/datasetA/latest/properties") + assertOk(response) + + val body = response.getBody + assert(body == Map("key1" -> "val1", "key2" -> "val2")) + } + } + } + } + + + s"PUT $apiUrl/{name}/{version}/properties" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendPut[Map[String, String], String](s"$apiUrl/notFoundDataset/456/properties", bodyOpt = Some(Map.empty)) + assertNotFound(response) + } + } + + "return 400" when { + "when version is not the latest (only last version can be updated)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val datasetV3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(datasetV1, datasetV2, datasetV3) + + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/2/properties", bodyOpt = Some(Map.empty)) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("version" -> + List("Version 2 of datasetA is not the latest version, therefore cannot be edited") + )) + } + + "when properties are not backed by propDefs (undefined properties)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + // propdefs are empty + + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("undefinedProperty1" -> "someValue"))) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) + } + + "when properties are not valid (based on propDefs)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("mandatoryA", essentiality = Essentiality.Mandatory), + PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) + ) + + val response1 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("AorB" -> "a"))) // this is ok, but mandatoryA is missing + + assertBadRequest(response1) + response1.getBody shouldBe Validation(Map("mandatoryA" -> List("Dataset property 'mandatoryA' is mandatory, but does not exist!"))) + + val response2 = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("mandatoryA" -> "valueA", "AorB" -> "c"))) // mandatoryA is ok, but AorB has invalid value + + assertBadRequest(response2) + response2.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) + } + } + + "201 Created with location" when { + Seq( + ("non-empty properties map", """{"keyA":"valA","keyB":"valB","keyC":""}""", Some(Map("keyA" -> "valA", "keyB" -> "valB"))), // empty string property would get removed (defined "" => undefined) + ("empty properties map", "{}", Some(Map.empty)) + ).foreach { case (testCaseName, payload, expectedPropertiesSet) => + s"properties are replaced with a new version ($testCaseName)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("keyA"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyB"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyC"), + PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) + ) + + val response1 = sendPut[String, Validation](s"$apiUrl/datasetA/1/properties", bodyOpt = Some(payload)) + assertCreated(response1) + response1.getBody shouldBe Validation.empty.withWarning("keyD", "Property 'keyD' is recommended to be present, but was not found!") + val headers1 = response1.getHeaders + assert(headers1.getFirst("Location").endsWith("/api-v3/datasets/datasetA/2/properties")) + + + val response2 = sendGet[Map[String, String]](s"$apiUrl/datasetA/2/properties") + assertOk(response2) + val responseBody = response2.getBody + responseBody shouldBe expectedPropertiesSet.getOrElse(Map.empty) + } + } + } + } + + // similar to put-properties validation + s"GET $apiUrl/{name}/{version}/validation" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/validation") + assertNotFound(response) + } + } + + // todo name validation - common for versioned entities + + "return 200" when { + "when properties are not backed by propDefs (undefined properties) and schema is missing" in { + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("undefinedProperty1" -> "someValue"))) + datasetFixture.add(datasetV1) + // propdefs are empty, schemas not defined + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + + assertOk(response) + response.getBody shouldBe + Validation(Map( + "undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."), + "schema" -> List("Schema dummySchema v1 not found!") + )) + } + + "when properties are not valid (based on propDefs) - mandatoriness check" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = None) // prop 'mandatoryA' not present + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("mandatoryA", essentiality = Essentiality.Mandatory) + ) + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + assertOk(response) + response.getBody shouldBe Validation(Map("mandatoryA" -> List("Dataset property 'mandatoryA' is mandatory, but does not exist!"))) + } + + "when properties are not valid (based on propDefs) - property conformance" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("AorB" -> "c"))) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) + ) + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + assertOk(response) + response.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) + } + } + } + + private val exampleMcrRule0 = MappingConformanceRule(0, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable", + mappingTableVersion = 9, //scalastyle:ignore magic.number + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + + private val exampleLitRule1 = LiteralConformanceRule(order = 1, controlCheckpoint = true, outputColumn = "something", value = "1.01") + private val dsWithRules1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( + exampleMcrRule0, exampleLitRule1 + )) + + s"GET $apiUrl/{name}/{version}/rules" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/rules") + assertNotFound(response) + } + } + + "return 200" when { + "when there are no conformance rules" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) + + val response = sendGet[Array[ConformanceRule]](s"$apiUrl/datasetA/1/rules") + + assertOk(response) + response.getBody shouldBe Seq() + } + + "when there are some conformance rules" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + datasetFixture.add(dsWithRules1) + + val response = sendGet[Array[ConformanceRule]](s"$apiUrl/datasetA/1/rules") + assertOk(response) + response.getBody shouldBe dsWithRules1.conformance.toArray + } + } + } + + s"POST $apiUrl/{name}/{version}/rules" should { + "return 404" when { + "when the name+version does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", + bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + assertNotFound(response) + } + } + + "return 400" when { + "when the there is a conflicting conf rule #" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( + LiteralConformanceRule(order = 0,"column1", true, "ABC") + )) + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", + bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + assertBadRequest(response) + + response.getBody should include("Rule with order 0 cannot be added, another rule with this order already exists.") + } + } + + "return 400" when { + "when rule is not valid (missing MT)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA") + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", + bodyOpt = Some(exampleMcrRule0)) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("mapping-table", "Mapping table CurrencyMappingTable v9 not found!") + } + } + + "return 201" when { + "when conf rule is added" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( + LiteralConformanceRule(order = 0,"column1", true, "ABC")) + ) + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) + assertCreated(response) + // if, in the future, there can be a rule update resulting in a warning, let's reflect that here + response.getBody shouldBe Validation.empty + + val locationHeader = response.getHeaders.getFirst("location") + locationHeader should endWith("/api-v3/datasets/datasetA/2/rules/1") // increased version in the url and added rule #1 + + val response2 = sendGet[Dataset](s"$apiUrl/datasetA/2") + assertOk(response2) + + val actual = response2.getBody + val expectedDsBase = datasetV1.copy(version = 2, parent = Some(DatasetFactory.toParent(datasetV1)), + conformance = List(datasetV1.conformance.head, exampleLitRule1)) + val expected = toExpected(expectedDsBase, actual) + + assert(actual == expected) + } + } + } + + s"GET $apiUrl/{name}/{version}/rules/{index}" should { + "return 404" when { + "when the name+version does not exist" in { + val response = sendGet[String](s"$apiUrl/notFoundDataset/456/rules/1") + assertNotFound(response) + } + + "when the rule with # does not exist" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + datasetFixture.add(dsWithRules1) + + val response = sendGet[String](s"$apiUrl/datasetA/1/rules/345") + assertNotFound(response) + } + } + + "return 200" when { + "when there is a conformance rule with the order#" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + datasetFixture.add(dsWithRules1) + + val response = sendGet[ConformanceRule](s"$apiUrl/datasetA/1/rules/1") + assertOk(response) + response.getBody shouldBe LiteralConformanceRule(order = 1, controlCheckpoint = true, outputColumn = "something", value = "1.01") + } + } + } + +} diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 4dc11ee16..087381db9 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.integration.repositories import com.mongodb.MongoWriteException import org.junit.runner.RunWith +import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -32,11 +33,12 @@ import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance import za.co.absa.enceladus.model.menas.scheduler.ScheduleTiming import za.co.absa.enceladus.model.menas.scheduler.RuntimeConfig import za.co.absa.enceladus.model.menas.scheduler.dataFormats.ParquetDataFormat +import za.co.absa.enceladus.model.versionedModel.VersionedSummary @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { +class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers { @Autowired private val datasetFixture: DatasetFixtureService = null @@ -474,17 +476,17 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { } } - "DatasetMongoRepository::getLatestVersions" should { + "DatasetMongoRepository::getLatestVersionsSummarySearch" should { "return an empty Seq" when { "no datasets exist and search query is provided" in { - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some("abc"))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("abc"))) assert(actual.isEmpty) } "only disabled dataset exists" in { val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", version = 1, disabled = true, dateDisabled = Option(DatasetFactory.dummyZonedDateTime), userDisabled = Option("user")) datasetFixture.add(dataset1) - assert(await(datasetMongoRepository.getLatestVersionsSummary(Some("dataset1"))).isEmpty) + assert(await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("dataset1"))).isEmpty) } } @@ -496,7 +498,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val dataset5 = DatasetFactory.getDummyDataset(name = "abc", version = 1) datasetFixture.add(dataset2, dataset3, dataset4, dataset5) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some("dataset2"))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("dataset2"))) val expected = Seq(dataset3).map(DatasetFactory.toSummary) assert(actual == expected) @@ -508,7 +510,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val dataset5 = DatasetFactory.getDummyDataset(name = "abc", version = 1) datasetFixture.add(dataset2, dataset3, dataset4, dataset5) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some("tas"))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some("tas"))) val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummary) assert(actual == expected) @@ -544,7 +546,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val abc1 = DatasetFactory.getDummyDataset(name = "abc", version = 1) datasetFixture.add(dataset1ver1, dataset1ver2, dataset2ver1, abc1) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(Some(""))) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(Some(""))) val expected = Seq(abc1, dataset1ver2, dataset2ver1).map(DatasetFactory.toSummary) assert(actual == expected) @@ -561,13 +563,73 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val dataset1ver2 = DatasetFactory.getDummyDataset(name = "dataset1", version = 2) datasetFixture.add(dataset1ver2) - val actual = await(datasetMongoRepository.getLatestVersionsSummary(None)) + val actual = await(datasetMongoRepository.getLatestVersionsSummarySearch(None)) val expected = Seq(dataset1ver2, dataset2ver2).map(DatasetFactory.toSummary) assert(actual == expected) } } + "DatasetMongoRepository::getLatestVersionSummary" should { + "returns no summary" when { + "no datasets exist by the name" in { + val actual = await(datasetMongoRepository.getLatestVersionSummary("notExistingName")) + actual shouldBe None + } + } + "returns even the disabled dataset" when { + "only disabled dataset exists" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) + datasetFixture.add(dataset1, dataset2) + val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) + actual shouldBe Some(VersionedSummary("datasetA", 2)) // warning: currently, this method reports the disabled, too + } + } + + "return give correct version summary" when { + "dataset versions exist" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(dataset1, dataset2, dataset3) + + val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) + actual shouldBe Some(VersionedSummary("datasetA", 3)) + } + } + } + + "DatasetMongoRepository::getLatestVersionValue" should { + "returns no latest version" when { + "no datasets exist by the name" in { + val actual = await(datasetMongoRepository.getLatestVersionValue("notExistingName")) + actual shouldBe None + } + } + "returns even the disabled dataset version" when { + "only disabled dataset exists" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) + datasetFixture.add(dataset1, dataset2) + val actual = await(datasetMongoRepository.getLatestVersionValue("datasetA")) + actual shouldBe Some(2) // warning: currently, this method reports the disabled, too + } + } + + "return gives correct latest version" when { + "dataset versions exist" in { + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "datasetA", version = 3) + datasetFixture.add(dataset1, dataset2, dataset3) + + val actual = await(datasetMongoRepository.getLatestVersionValue("datasetA")) + actual shouldBe Some(3) + } + } + } + "DatasetMongoRepository::distinctCount" should { "return 0" when { "no datasets exists" in { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala index 48914328b..7afcb53c6 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala @@ -305,7 +305,7 @@ class DatasetServiceTest extends VersionedModelServiceTest[Dataset] with Matcher ) val dataset = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(properties)) - DatasetService.removeBlankProperties(dataset.properties) shouldBe Some(Map("propKey1" -> "someValue")) + DatasetService.removeBlankPropertiesOpt(dataset.properties) shouldBe Some(Map("propKey1" -> "someValue")) } test("DatasetService.replacePrefixIfFound replaces field prefixes") { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala index c555e6d0b..34032f344 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/VersionedModelServiceTest.scala @@ -29,23 +29,13 @@ abstract class VersionedModelServiceTest[C <: VersionedModel with Product with A private val validName = "validName" - test("Validate dataset with valid, unique name") { - Mockito.when(modelRepository.isUniqueName(validName)).thenReturn(Future.successful(true)) - + test("Validate a valid model name") { val result = Await.result(service.validateName(validName), shortTimeout) assert(result.isValid) assert(result == Validation()) } - test("Validate dataset with valid, taken name") { - Mockito.when(modelRepository.isUniqueName(validName)).thenReturn(Future.successful(false)) - - val result = Await.result(service.validateName(validName), shortTimeout) - assert(!result.isValid) - assert(result == Validation(Map("name" -> List(s"entity with name already exists: '$validName'")))) - } - - test("Validate dataset with invalid name") { + test("Validate an invalid model name") { assertHasWhitespace(" InvalidName") assertHasWhitespace("InvalidName\t") assertHasWhitespace("Invalid\nName") @@ -59,4 +49,24 @@ abstract class VersionedModelServiceTest[C <: VersionedModel with Product with A assert(result == Validation(Map("name" -> List(s"name contains whitespace: '$name'")))) } + protected val validModels: Seq[C] = Seq.empty // expecting to override to check models + + test("Validate valid models") { + val results = validModels.map{model => (model, Await.result(service.validate(model), shortTimeout))} + + results.foreach { case (model, validation) => + assert(validation.isValid, s"Expected $model to be valid, but $validation found") + } + } + + protected val invalidModels: Seq[C] = Seq.empty // expecting to override to check models + + test("Validate invalid models") { + val results = invalidModels.map{model => (model, Await.result(service.validate(model), shortTimeout))} + + results.foreach { case (model, validation) => + assert(!validation.isValid, s"Expected $model to be invalid, but $validation found") + } + } + }