Merge pull request #477 from guardian/csv-import

Introduce`/api/rules/csv-import` endpoint

apps/rule-manager/app/controllers/RulesController.scala

@@ -279,4 +279,23 @@ class RulesController(
       case Left(error) => BadRequest(s"Invalid request: $error")
     }
   }
+
+  def csvImport() = APIAuthAction { implicit request =>
+    val rules = for {
+      formData <- request.body.asMultipartFormData.toRight("No form data found in request")
+      file <- formData.file("file").toRight("No file found in request")
+      tag = formData.dataParts.get("tag").flatMap(_.headOption)
+      category = formData.dataParts.get("category").flatMap(_.headOption)
+    } yield RuleManager.csvImport(
+      file.ref.path.toFile,
+      tag,
+      category,
+      bucketRuleResource
+    )
+
+    rules match {
+      case Right(noOfRulesAdded) => Ok(Json.toJson(noOfRulesAdded))
+      case Left(message)         => BadRequest(message)
+    }
+  }
 }

apps/rule-manager/app/service/RuleManager.scala

@@ -18,6 +18,8 @@ import model.{DictionaryForm, LTRuleCoreForm, LTRuleXMLForm, PaginatedResponse,
 import play.api.data.FormError
 import play.api.libs.json.{Json, OWrites}
 import scalikejdbc.DBSession
+import com.github.tototoshi.csv.CSVReader
+import java.io.File
 
 object AllRuleData {
   implicit val writes: OWrites[AllRuleData] = Json.writes[AllRuleData]
@@ -32,6 +34,61 @@ case class AllRuleData(
 )
 
 object RuleManager extends Loggable {
+  def csvImport(
+      toFile: File,
+      maybeTagName: Option[String],
+      category: Option[String],
+      bucketRuleResource: BucketRuleResource
+  ) = {
+    val reader = CSVReader.open(toFile)
+    val rules = reader.all()
+    reader.close()
+
+    val initialRuleOrder = DbRuleDraft.getLatestRuleOrder() + 1
+
+    val draftRules = rules.zipWithIndex.map { case (rule, index) =>
+      val pattern = rule(0)
+      val replacement = rule(1)
+      val description = rule(2)
+
+      DbRuleDraft.withUser(
+        id = None,
+        ruleType = RuleType.regex,
+        pattern = Some(pattern),
+        category = category,
+        description = Some(description),
+        ignore = false,
+        replacement = Some(replacement),
+        user = "CSV Import",
+        ruleOrder = initialRuleOrder + index
+      )
+    }
+    val ruleIds = DbRuleDraft.batchInsert(draftRules, true)
+
+    for {
+      tagName <- maybeTagName
+      tag <- Tags.findAll().find(_.name == tagName).orElse {
+        log.error(s"Tag $tagName not found")
+        None
+      }
+      tagId <- tag.id.orElse {
+        log.error(s"Tag $tagName has no ID")
+        None
+      }
+    } yield {
+      RuleTagDraft.batchInsert(ruleIds.map(RuleTagDraft(_, tagId)))
+    }
+
+    val rulesWithIds = DbRuleDraft.findRules(ruleIds)
+    val liveRulesWithIds = rulesWithIds.map(_.toLive("Imported from CSV", true))
+
+    DbRuleLive.batchInsert(liveRulesWithIds)
+
+    publishLiveRules(bucketRuleResource)
+
+    liveRulesWithIds.size
+  }
+
   object RuleType {
     val regex = "regex"
     val languageToolXML = "languageToolXML"

apps/rule-manager/conf/routes

@@ -21,6 +21,8 @@ GET     /api/rules/batch/:ids           controllers.RulesController.getRules(ids
 +nocsrf
 POST    /api/rules/batch                controllers.RulesController.batchUpdate()
 +nocsrf
+POST    /api/rules/csv-import           controllers.RulesController.csvImport()
++nocsrf
 GET     /api/rules/:id                  controllers.RulesController.get(id: Int)
 +nocsrf
 POST    /api/rules/:id                  controllers.RulesController.update(id: Int)

apps/rule-manager/test/db/RuleManagerSpec.scala

@@ -11,7 +11,7 @@ import com.gu.typerighter.model.{
   TextSuggestion
 }
 import com.gu.typerighter.rules.BucketRuleResource
-import db.{DBTest, DbRuleDraft, DbRuleLive}
+import db.{DBTest, DbRuleDraft, DbRuleLive, Tags}
 import org.scalatest.flatspec.FixtureAnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import scalikejdbc.scalatest.AutoRollback
@@ -20,6 +20,8 @@ import com.softwaremill.diffx.scalatest.DiffShouldMatcher._
 import fixtures.RuleFixtures
 import play.api.data.FormError
 import utils.LocalStack
+
+import java.io.File
 import java.time.OffsetDateTime
 
 class RuleManagerSpec extends FixtureAnyFlatSpec with Matchers with AutoRollback with DBTest {
@@ -558,4 +560,44 @@ class RuleManagerSpec extends FixtureAnyFlatSpec with Matchers with AutoRollback
             .copy(updatedAt = mockUpdatedAt)
       }
   }
+
+  "csvImport" should "create draft and live rules based on the contents of the csv file passed in, ensuring the appropriate tag and category are set" in {
+    () =>
+      val tagToApply = Tags.create(name = "testTag")
+
+      val file = new File(getClass.getResource("/csv/mps.csv").toURI)
+
+      RuleManager.csvImport(
+        file,
+        Some("testTag"),
+        Some("Style guide and names"),
+        bucketRuleResource
+      )
+
+      val draftRules = DbRuleDraft.findAll()
+
+      draftRules.length shouldBe 3
+
+      val draftRule1 = draftRules(0)
+      draftRule1.pattern shouldBe Some("Dami(e|a)n Egan")
+      draftRule1.replacement shouldBe Some("Damien Egan")
+      draftRule1.description shouldBe Some("MP last elected in 2024: Labour, Bristol North East")
+      draftRule1.category shouldBe Some("Style guide and names")
+
+      draftRule1.tags.length shouldBe 1
+      draftRule1.tags(0) shouldBe tagToApply.get.id.get
+
+      val liveRules = DbRuleLive.findAll()
+
+      liveRules.length shouldBe 3
+
+      val liveRule1 = liveRules(0)
+      liveRule1.pattern shouldBe Some("Dami(e|a)n Egan")
+      liveRule1.replacement shouldBe Some("Damien Egan")
+      liveRule1.description shouldBe Some("MP last elected in 2024: Labour, Bristol North East")
+      liveRule1.category shouldBe Some("Style guide and names")
+
+      liveRule1.tags.length shouldBe 1
+      liveRule1.tags(0) shouldBe tagToApply.get.id.get
+  }
 }

apps/rule-manager/test/resources/csv/mps.csv

@@ -0,0 +1,3 @@
+Dami(e|a)n Egan,Damien Egan,"MP last elected in 2024: Labour, Bristol North East"
+Clive Efford,Clive Efford,"MP last elected in 2024: Labour, Eltham and Chislehurst"
+Maya Ellis,Maya Ellis,"MP last elected in 2024: Labour, Ribble Valley"

build.sbt

@@ -172,6 +172,7 @@ val ruleManager = playProject(
       "org.scalikejdbc" %% "scalikejdbc-test" % scalikejdbcVersion % Test,
       "org.scalikejdbc" %% "scalikejdbc-syntax-support-macro" % scalikejdbcVersion,
       "com.gu" %% "editorial-permissions-client" % "2.14",
+      "com.github.tototoshi" %% "scala-csv" % "2.0.0",
     ),
     libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
   )