diff --git a/apps/rule-manager/app/controllers/RulesController.scala b/apps/rule-manager/app/controllers/RulesController.scala index 58f3998fb..78f3a4e90 100644 --- a/apps/rule-manager/app/controllers/RulesController.scala +++ b/apps/rule-manager/app/controllers/RulesController.scala @@ -279,4 +279,23 @@ class RulesController( case Left(error) => BadRequest(s"Invalid request: $error") } } + + def csvImport() = APIAuthAction { implicit request => + val rules = for { + formData <- request.body.asMultipartFormData.toRight("No form data found in request") + file <- formData.file("file").toRight("No file found in request") + tag = formData.dataParts.get("tag").flatMap(_.headOption) + category = formData.dataParts.get("category").flatMap(_.headOption) + } yield RuleManager.csvImport( + file.ref.path.toFile, + tag, + category, + bucketRuleResource + ) + + rules match { + case Right(noOfRulesAdded) => Ok(Json.toJson(noOfRulesAdded)) + case Left(message) => BadRequest(message) + } + } } diff --git a/apps/rule-manager/app/service/RuleManager.scala b/apps/rule-manager/app/service/RuleManager.scala index f74a74316..9e7363d1d 100644 --- a/apps/rule-manager/app/service/RuleManager.scala +++ b/apps/rule-manager/app/service/RuleManager.scala @@ -18,6 +18,8 @@ import model.{DictionaryForm, LTRuleCoreForm, LTRuleXMLForm, PaginatedResponse, import play.api.data.FormError import play.api.libs.json.{Json, OWrites} import scalikejdbc.DBSession +import com.github.tototoshi.csv.CSVReader +import java.io.File object AllRuleData { implicit val writes: OWrites[AllRuleData] = Json.writes[AllRuleData] @@ -32,6 +34,61 @@ case class AllRuleData( ) object RuleManager extends Loggable { + def csvImport( + toFile: File, + maybeTagName: Option[String], + category: Option[String], + bucketRuleResource: BucketRuleResource + ) = { + val reader = CSVReader.open(toFile) + val rules = reader.all() + reader.close() + + val initialRuleOrder = DbRuleDraft.getLatestRuleOrder() + 1 + + val draftRules = rules.zipWithIndex.map { case (rule, index) => + val pattern = rule(0) + val replacement = rule(1) + val description = rule(2) + + DbRuleDraft.withUser( + id = None, + ruleType = RuleType.regex, + pattern = Some(pattern), + category = category, + description = Some(description), + ignore = false, + replacement = Some(replacement), + user = "CSV Import", + ruleOrder = initialRuleOrder + index + ) + } + val ruleIds = DbRuleDraft.batchInsert(draftRules, true) + + for { + tagName <- maybeTagName + tag <- Tags.findAll().find(_.name == tagName).orElse { + log.error(s"Tag $tagName not found") + None + } + tagId <- tag.id.orElse { + log.error(s"Tag $tagName has no ID") + None + } + } yield { + RuleTagDraft.batchInsert(ruleIds.map(RuleTagDraft(_, tagId))) + } + + val rulesWithIds = DbRuleDraft.findRules(ruleIds) + val liveRulesWithIds = rulesWithIds.map(_.toLive("Imported from CSV", true)) + + DbRuleLive.batchInsert(liveRulesWithIds) + + publishLiveRules(bucketRuleResource) + + liveRulesWithIds.size + } + object RuleType { val regex = "regex" val languageToolXML = "languageToolXML" diff --git a/apps/rule-manager/conf/routes b/apps/rule-manager/conf/routes index 4a8206c08..545d23d8c 100644 --- a/apps/rule-manager/conf/routes +++ b/apps/rule-manager/conf/routes @@ -21,6 +21,8 @@ GET /api/rules/batch/:ids controllers.RulesController.getRules(ids +nocsrf POST /api/rules/batch controllers.RulesController.batchUpdate() +nocsrf +POST /api/rules/csv-import controllers.RulesController.csvImport() ++nocsrf GET /api/rules/:id controllers.RulesController.get(id: Int) +nocsrf POST /api/rules/:id controllers.RulesController.update(id: Int) diff --git a/apps/rule-manager/test/db/RuleManagerSpec.scala b/apps/rule-manager/test/db/RuleManagerSpec.scala index 114fd134c..1776a7e53 100644 --- a/apps/rule-manager/test/db/RuleManagerSpec.scala +++ b/apps/rule-manager/test/db/RuleManagerSpec.scala @@ -11,7 +11,7 @@ import com.gu.typerighter.model.{ TextSuggestion } import com.gu.typerighter.rules.BucketRuleResource -import db.{DBTest, DbRuleDraft, DbRuleLive} +import db.{DBTest, DbRuleDraft, DbRuleLive, Tags} import org.scalatest.flatspec.FixtureAnyFlatSpec import org.scalatest.matchers.should.Matchers import scalikejdbc.scalatest.AutoRollback @@ -20,6 +20,8 @@ import com.softwaremill.diffx.scalatest.DiffShouldMatcher._ import fixtures.RuleFixtures import play.api.data.FormError import utils.LocalStack + +import java.io.File import java.time.OffsetDateTime class RuleManagerSpec extends FixtureAnyFlatSpec with Matchers with AutoRollback with DBTest { @@ -558,4 +560,44 @@ class RuleManagerSpec extends FixtureAnyFlatSpec with Matchers with AutoRollback .copy(updatedAt = mockUpdatedAt) } } + + "csvImport" should "create draft and live rules based on the contents of the csv file passed in, ensuring the appropriate tag and category are set" in { + () => + val tagToApply = Tags.create(name = "testTag") + + val file = new File(getClass.getResource("/csv/mps.csv").toURI) + + RuleManager.csvImport( + file, + Some("testTag"), + Some("Style guide and names"), + bucketRuleResource + ) + + val draftRules = DbRuleDraft.findAll() + + draftRules.length shouldBe 3 + + val draftRule1 = draftRules(0) + draftRule1.pattern shouldBe Some("Dami(e|a)n Egan") + draftRule1.replacement shouldBe Some("Damien Egan") + draftRule1.description shouldBe Some("MP last elected in 2024: Labour, Bristol North East") + draftRule1.category shouldBe Some("Style guide and names") + + draftRule1.tags.length shouldBe 1 + draftRule1.tags(0) shouldBe tagToApply.get.id.get + + val liveRules = DbRuleLive.findAll() + + liveRules.length shouldBe 3 + + val liveRule1 = liveRules(0) + liveRule1.pattern shouldBe Some("Dami(e|a)n Egan") + liveRule1.replacement shouldBe Some("Damien Egan") + liveRule1.description shouldBe Some("MP last elected in 2024: Labour, Bristol North East") + liveRule1.category shouldBe Some("Style guide and names") + + liveRule1.tags.length shouldBe 1 + liveRule1.tags(0) shouldBe tagToApply.get.id.get + } } diff --git a/apps/rule-manager/test/resources/csv/mps.csv b/apps/rule-manager/test/resources/csv/mps.csv new file mode 100644 index 000000000..4370e87ad --- /dev/null +++ b/apps/rule-manager/test/resources/csv/mps.csv @@ -0,0 +1,3 @@ +Dami(e|a)n Egan,Damien Egan,"MP last elected in 2024: Labour, Bristol North East" +Clive Efford,Clive Efford,"MP last elected in 2024: Labour, Eltham and Chislehurst" +Maya Ellis,Maya Ellis,"MP last elected in 2024: Labour, Ribble Valley" \ No newline at end of file diff --git a/build.sbt b/build.sbt index 988cdc3d1..4446b6c62 100644 --- a/build.sbt +++ b/build.sbt @@ -172,6 +172,7 @@ val ruleManager = playProject( "org.scalikejdbc" %% "scalikejdbc-test" % scalikejdbcVersion % Test, "org.scalikejdbc" %% "scalikejdbc-syntax-support-macro" % scalikejdbcVersion, "com.gu" %% "editorial-permissions-client" % "2.14", + "com.github.tototoshi" %% "scala-csv" % "2.0.0", ), libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always )