Merge pull request #107 from guardian/ld/upgrade-11-after-dynamo-upgrade

Upgrade to Java 11
guardian · Apr 8, 2024 · e4e9f76 · e4e9f76
2 parents 0a8aab6 + 2fcee3f
commit e4e9f76
Show file tree

Hide file tree

Showing 18 changed files with 300 additions and 63 deletions.
diff --git a/.github/workflows/ci-mobile-save-for-later-user-deletion.yaml b/.github/workflows/ci-mobile-save-for-later-user-deletion.yaml
@@ -8,20 +8,20 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - uses: actions/checkout@v3
-      - uses: aws-actions/configure-aws-credentials@v1
+      - uses: actions/checkout@v4
+      - uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }}
           aws-region: eu-west-1
-      - name: Set up JDK 8
+      - name: Set up JDK
         uses: actions/setup-java@v3
         with:
-          java-version: 8
-          distribution: temurin
+          java-version: 11
+          distribution: corretto
           cache: sbt
 
-      - name: Create artifacts
-        run: sbt "project mobile-save-for-later-user-deletion" assembly
+      - name: Run Tests & Create Artifacts
+        run: sbt "project mobile-save-for-later-user-deletion" test assembly
 
       - name: RiffRaff upload artifacts and upload cfn
         uses: guardian/actions-riff-raff@v2

diff --git a/.github/workflows/ci-mobile-save-for-later.yaml b/.github/workflows/ci-mobile-save-for-later.yaml
@@ -8,20 +8,20 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - uses: actions/checkout@v3
-      - uses: aws-actions/configure-aws-credentials@v1
+      - uses: actions/checkout@v4
+      - uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }}
           aws-region: eu-west-1
-      - name: Set up JDK 8
-        uses: actions/setup-java@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v4
         with:
-          java-version: 8
-          distribution: temurin
+          java-version: 11
+          distribution: corretto
           cache: sbt
 
-      - name: Create artifacts
-        run: sbt "project mobile-save-for-later" assembly
+      - name: Run Tests and Create Artifacts
+        run: sbt "project mobile-save-for-later" test assembly
 
       - name: Build CDK
         working-directory: ./cdk
@@ -41,4 +41,4 @@ jobs:
               - cdk/cdk.out/MobileSaveForLater-CODE.template.json
               - cdk/cdk.out/MobileSaveForLater-PROD.template.json
             mobile-save-for-later:
-             - mobile-save-for-later/target/scala-2.12/mobile-save-for-later.jar
+             - mobile-save-for-later/target/scala-2.12/mobile-save-for-later.jar
diff --git a/README.md b/README.md
@@ -74,6 +74,51 @@ Try(mapper.readValue[List[SavedArticle]](json)) match {
 
 *NB: I found that when a user record has 350+ articles the resultant string in too long for idea to handle. 
 
+## Running apps locally
+
+### Pre-requisites
+
+- Mobile credentials from [Janus](https://janus.gutools.co.uk/login)
+- Install [AWS Sam](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html)
+- Install [Docker](https://docs.docker.com/engine/install/)
+- Make sure you run the app with Java 11
+
+### Mobile Save For Later
+
+Build a jar of the project by running:
+- `sbt "project mobile-save-for-later" riffRaffPackageType`
+  Navigate to the project
+- `cd mobile-save-for-later`
+
+Run the lambda service
+- `DOCKER_HOST=unix://$HOME/.docker/run/docker.sock sam local start-lambda --debug`
+
+Execute a function locally
+- `DOCKER_HOST=unix://$HOME/.docker/run/docker.sock sam local invoke`
+
+Execute a function using AWS cli
+- `aws lambda invoke --function-name "mobilesaveforlaterFETCHcdkCODE" --endpoint-url "http://127.0.0.1:3001" --no-verify-ssl out.txt --profile mobile --region eu-west-1`
+
+### Mobile Save For Later User Deletion
+
+Build a jar of the project by running:
+- `sbt "project mobile-save-for-later-user-deletion" riffRaffPackageType`
+  Navigate to the project
+- `cd mobile-save-for-later-user-deletion`
+
+Run the lambda service
+- `DOCKER_HOST=unix://$HOME/.docker/run/docker.sock sam local start-lambda --debug`
+
+Execute a function locally
+- `DOCKER_HOST=unix://$HOME/.docker/run/docker.sock sam local invoke`
+
+Execute a function using AWS cli
+- `aws lambda invoke --function-name "mobilesaveforlateruserdeletionCODE" --endpoint-url "http://127.0.0.1:3001" --no-verify-ssl out.txt --profile mobile --region eu-west-1`
+
+
 ## Testing the Apps on CODE
 
 [Save For Later App](docs/testing/save-for-later.md)
+
+[User Deletion App](docs/testing/user-deletion.md)
+
diff --git a/build.sbt b/build.sbt
@@ -20,6 +20,7 @@ def projectMaker(projectName: String) = Project(projectName, file(projectName))
   .dependsOn(common % "compile->compile")
   .aggregate(common)
 
+ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
 def commonAssemblySettings(module: String): immutable.Seq[Def.Setting[_]] =
   commonSettings ++ List(
     assemblyJarName := s"${name.value}.jar",
@@ -38,7 +39,6 @@ val commonSettings: immutable.Seq[Def.Setting[_]] = List(
     awsJavaSdk,
     jackson,
     jacksonDataFormat,
-    jacksonJdk8DataType,
     jacksonJsrDataType,
     log4j,
     commonsIo,
@@ -49,10 +49,9 @@ val commonSettings: immutable.Seq[Def.Setting[_]] = List(
     specsScalaCheck,
     specsMock
   ),
-  assembly / assemblyMergeStrategy := {
+  ThisBuild / assemblyMergeStrategy := {
     case "META-INF/MANIFEST.MF" => MergeStrategy.discard
-    case "META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat" =>
-      new MergeFilesStrategy
+    case PathList(ps @ _*) if ps.last equalsIgnoreCase "Log4j2Plugins.dat" => sbtassembly.Log4j2MergeStrategy.plugincache
     case _ => MergeStrategy.first
   },
   dependencyOverrides ++= Seq(
@@ -68,7 +67,7 @@ val commonSettings: immutable.Seq[Def.Setting[_]] = List(
     "-deprecation",
     "-encoding",
     "UTF-8",
-    "-target:jvm-1.8",
+    "-release:11",
     "-Ypartial-unification",
     "-Ywarn-dead-code"
   )

diff --git a/cdk/lib/__snapshots__/mobile-save-for-later.test.ts.snap b/cdk/lib/__snapshots__/mobile-save-for-later.test.ts.snap
@@ -750,7 +750,7 @@ Object {
             "Arn",
           ],
         },
-        "Runtime": "java8",
+        "Runtime": "java11",
         "Tags": Array [
           Object {
             "Key": "App",
@@ -980,7 +980,7 @@ Object {
             "Arn",
           ],
         },
-        "Runtime": "java8",
+        "Runtime": "java11",
         "Tags": Array [
           Object {
             "Key": "App",
@@ -2002,7 +2002,7 @@ Object {
             "Arn",
           ],
         },
-        "Runtime": "java8",
+        "Runtime": "java11",
         "Tags": Array [
           Object {
             "Key": "App",
@@ -2232,7 +2232,7 @@ Object {
             "Arn",
           ],
         },
-        "Runtime": "java8",
+        "Runtime": "java11",
         "Tags": Array [
           Object {
             "Key": "App",

diff --git a/cdk/lib/mobile-save-for-later.ts b/cdk/lib/mobile-save-for-later.ts
@@ -41,7 +41,7 @@ export class MobileSaveForLater extends GuStack {
     const app = "mobile-save-for-later";
 
     const commonLambdaProps = {
-      runtime: Runtime.JAVA_8, // We should upgrade to Java 11 in a future PR
+      runtime: Runtime.JAVA_11,
       app,
       fileName: `${app}.jar`,
     };

diff --git a/common/src/main/scala/com/gu/sfl/lib/Jackson.scala b/common/src/main/scala/com/gu/sfl/lib/Jackson.scala
@@ -3,7 +3,6 @@ package com.gu.sfl.lib
 import java.text.SimpleDateFormat
 
 import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
-import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
@@ -17,6 +16,5 @@ object Jackson {
   mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
   mapper.setDateFormat(formatter)
   mapper.registerModule(DefaultScalaModule)
-  mapper.registerModule(new Jdk8Module())
   mapper.registerModule(new JavaTimeModule())
 }
diff --git a/docs/testing/user-deletion.md b/docs/testing/user-deletion.md
@@ -0,0 +1,33 @@
+### Testing User Deletion on CODE
+
+### Testing via the website
+* Sign in to the website on [CODE](https://code.dev-theguardian.com/uk)
+* Under My Account -> Settings scroll down and hit Delete Account
+* Check the cloudwatch output in the [user delete lambda](https://eu-west-1.console.aws.amazon.com/lambda/home?region=eu-west-1#/functions/mobile-save-for-later-user-deletion-CODE?tab=monitoring)
+
+### Testing the lambda directly
+
+* Go to the [user delete lambda](https://eu-west-1.console.aws.amazon.com/lambda/home?region=eu-west-1#/functions/mobile-save-for-later-user-deletion-CODE?tab=monitoring)
+* Run the test tab with this message structure:
+```agsl
+{
+  "Records": [
+    {
+      "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
+      "receiptHandle": "MessageReceiptHandle",
+      "body": "{\"Message\": { \"userId\": \"1\"}}",
+      "attributes": {
+        "ApproximateReceiveCount": "1",
+        "SentTimestamp": "1523232000000",
+        "SenderId": "123456789012",
+        "ApproximateFirstReceiveTimestamp": "1523232000001"
+      },
+      "messageAttributes": {},
+      "md5OfBody": "{{{md5_of_body}}}",
+      "eventSource": "aws:sqs",
+      "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
+      "awsRegion": "us-east-1"
+    }
+  ]
+}
+```
diff --git a/mobile-save-for-later-user-deletion/conf/cfn.yaml b/mobile-save-for-later-user-deletion/conf/cfn.yaml
@@ -149,7 +149,7 @@ Resources:
       Handler: com.gu.sfl.userdeletion.UserDeletionLambda::handler
       MemorySize: 512
       Role: !GetAtt UserDeletionRole.Arn
-      Runtime: java8
+      Runtime: java11
       Timeout: 300
   UserDeletionEventSource:
     Type: AWS::Lambda::EventSourceMapping

diff --git a/...save-for-later-user-deletion/src/main/scala/coml/gu/sfl/userdeletion/db/SflDynamoDb.scala b/...save-for-later-user-deletion/src/main/scala/coml/gu/sfl/userdeletion/db/SflDynamoDb.scala
@@ -5,7 +5,7 @@ import org.scanamo.syntax._
 import com.gu.sfl.Logging
 import com.gu.sfl.persistence.{DynamoSavedArticles, PersistenceConfig}
 import com.gu.sfl.userdeletion.model.UserDeleteMessage
-import org.scanamo.DeleteReturn.Nothing
+import org.scanamo.DeleteReturn.OldValue
 import org.scanamo.generic.auto.genericDerivedFormat
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient
 
@@ -17,7 +17,7 @@ class SflDynamoDb(persistanceConfig: PersistenceConfig) extends Logging {
   private val scanamo = Scanamo(client)
   def deleteSavedArticleasForUser(user: UserDeleteMessage) = {
     logger.info(s"Deleting record for user id: ${user.userId}")
-    val dbResponse = scanamo.exec(table.deleteAndReturn(Nothing)("userId" === user.userId))
+    val dbResponse = scanamo.exec(table.deleteAndReturn(OldValue)("userId" === user.userId))
       .fold(s"Unable to delete record for user ${user.userId}")((_) => s"Deleted record for ${user.userId}")
     logger.info(dbResponse)
   }

diff --git a/mobile-save-for-later-user-deletion/template.yml b/mobile-save-for-later-user-deletion/template.yml
@@ -0,0 +1,84 @@
+# This AWS SAM template has been generated from your function's configuration. If
+# your function has one or more triggers, note that the AWS resources associated
+# with these triggers aren't fully specified in this template and include
+# placeholder values. Open this template in AWS Application Composer or your
+# favorite IDE and modify it to specify a serverless application with other AWS
+# resources.
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: An AWS Serverless Application Model template describing your function.
+Resources:
+  mobilesaveforlateruserdeletionCODE:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: 'target/scala-2.12/mobile-save-for-later-user-deletion.jar'
+      Description: Lamba that deletes saved for later data for deleted users
+      MemorySize: 384
+      Timeout: 300
+      Handler: com.gu.sfl.userdeletion.UserDeletionLambda::handler
+      Runtime: java11
+      Architectures:
+        - x86_64
+      EphemeralStorage:
+        Size: 512
+      Environment:
+        Variables:
+          App: mobile-save-for-later-user-deletion
+          SaveForLaterApp: mobile-save-for-later
+          Stack: mobile
+          Stage: CODE
+      EventInvokeConfig:
+        MaximumEventAgeInSeconds: 21600
+        MaximumRetryAttempts: 2
+      PackageType: Zip
+      Policies:
+        - Statement:
+            - Action:
+                - dynamodb:*
+              Resource:
+                - >-
+                  arn:aws:dynamodb:eu-west-1:201359054765:table/mobile-save-for-later-CODE-articles
+              Effect: Allow
+            - Action:
+                - lamda:InvokeFunction
+              Resource:
+                - >-
+                  arn:aws:lambda:eu-west-1:201359054765:function:mobile-save-for-later-user-deletion-CODE
+              Effect: Allow
+            - Action:
+                - logs:CreateLogGroup
+                - logs:CreateLogStream
+                - logs:PutLogEvents
+              Resource: '*'
+              Effect: Allow
+            - Action:
+                - sqs:ReceiveMessage
+                - sqs:DeleteMessage
+                - sqs:GetQueueAttributes
+              Resource:
+                - arn:aws:sqs:eu-west-1:201359054765:UserIdDeleteQueue-CODE
+              Effect: Allow
+      SnapStart:
+        ApplyOn: None
+      Tags:
+        App: mobile-save-for-later-user-deletion
+        Stack: mobile
+        Stage: CODE
+        gu:build-tool: guardian/actions-riff-raff
+        gu:repo: guardian/mobile-save-for-later
+      Events:
+        SQS1:
+          Type: SQS
+          Properties:
+            Queue:
+              Fn::GetAtt:
+                - SQSQueue1
+                - Arn
+            BatchSize: 5
+      RuntimeManagementConfig:
+        UpdateRuntimeOn: Auto
+  SQSQueue1:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: SQSQueue1
+      SqsManagedSseEnabled: true
diff --git a/mobile-save-for-later/src/main/scala/com/gu/sfl/controller/SaveArticlesController.scala b/mobile-save-for-later/src/main/scala/com/gu/sfl/controller/SaveArticlesController.scala
@@ -45,7 +45,7 @@ class SaveArticlesController(updateSavedArticles: UpdateSavedArticles)(implicit
       maybeUpdatedArticles
     }).map {
       case Right(syncedPrefs) =>
-        logger.debug("Got articles back from db")
+        logger.info("Got articles back from db")
         okSavedArticlesResponse(syncedPrefs)
       case Left(error) =>
         logger.error(s"Error saving articles: ${error.message}")

diff --git a/mobile-save-for-later/src/main/scala/com/gu/sfl/lib/SavedArticlesMerger.scala b/mobile-save-for-later/src/main/scala/com/gu/sfl/lib/SavedArticlesMerger.scala
@@ -1,6 +1,5 @@
 package com.gu.sfl.lib
 
-import com.amazonaws.util.Md5Utils
 import com.gu.sfl.Logging
 import com.gu.sfl.exception.{SaveForLaterError, SavedArticleMergeError}
 import com.gu.sfl.model._