Arc 2643 audit log table (#2559)

* chore: audit log service

* ARC-2643 add api router to fetch audit log

* chore: audit log service

* chore: audit log service

* chore: PR review comments

---------

Co-authored-by: Gary Xue <[email protected]>
Co-authored-by: Gary Xue <[email protected]>

.env.development

@@ -17,6 +17,8 @@ GLOBAL_HASH_SECRET=testsecret
 #Dyamno for deployment status
 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1
 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache
+DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1
+DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log
 
 # The Postgres URL used to connect to the database and secret for encrypting data
 DATABASE_URL=postgres://postgres:[email protected]:5432/jira-dev

.env.e2e

@@ -21,6 +21,8 @@ DEBUG=nock.*
 #Dyamno for deployment status
 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1
 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache
+DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1
+DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log
 
 MICROS_AWS_REGION=us-west-1
 

.env.test

@@ -26,6 +26,8 @@ DEBUG=nock.*
 #Dyamno for deployment status
 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1
 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache
+DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1
+DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log
 
 MICROS_AWS_REGION=us-west-1
 

.localstack/dynamodb.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-
+# DEPLOYMENT_HISTORY_CACHE_TABLE
 echo "===== creating dynamo table ${DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME} ====="
 
 awslocal dynamodb create-table \
@@ -15,8 +15,27 @@ awslocal dynamodb create-table \
         ReadCapacityUnits=10,WriteCapacityUnits=5
 
 echo "===== table ${DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME} created ====="
+
+# AUDIT_LOG_TABLE
+echo "===== creating dynamo table ${DYNAMO_AUDIT_LOG_TABLE_NAME} ====="
+
+awslocal dynamodb create-table \
+    --table-name $DYNAMO_AUDIT_LOG_TABLE_NAME \
+    --key-schema \
+      AttributeName=Id,KeyType=HASH \
+      AttributeName=CreatedAt,KeyType=RANGE \
+    --attribute-definitions \
+      AttributeName=Id,AttributeType=S \
+      AttributeName=CreatedAt,AttributeType=N \
+    --region $DYNAMO_AUDIT_LOG_TABLE_REGION \
+    --provisioned-throughput \
+        ReadCapacityUnits=10,WriteCapacityUnits=5
+
+echo "===== table ${DYNAMO_AUDIT_LOG_TABLE_NAME} created ====="
+
 echo "===== checking now ====="
 
+awslocal dynamodb list-tables --region $DYNAMO_AUDIT_LOG_TABLE_REGION
 awslocal dynamodb list-tables --region $DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION
 
 echo "===== check finished ====="

docker-compose.yml

@@ -31,7 +31,9 @@ services:
     environment:
       - DEFAULT_REGION=us-west-1
       - DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache
+      - DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log
       - DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1
+      - DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1
       - LAMBDA_REMOTE_DOCKER=false
       - LAMBDA_EXECUTOR=local # runs lambda inside temp directory instead of new docker container
       - SQS_ENDPOINT_STRATEGY=off # sets the SQS queue domain/path to the legacy version

github-for-jira.sd.yml

@@ -180,6 +180,17 @@ resources:
       TTLAttributeName: ExpiredAfter
       dataType:
         - UGC/PrimaryIdentifier    # Sha of a commit that point to user's code
+  - name: audit-log
+    type: dynamo-db
+    attributes: &audit-log-table-attributes
+      HashKeyName: Id
+      HashKeyType: "S"
+      RangeKeyName: CreatedAt
+      RangeKeyType: "N"
+      ReadWriteCapacityMode: ON_DEMAND
+      TTLAttributeName: ExpiredAfter
+      dataType:
+        - UGC/PrimaryIdentifier    # Sha of a commit that point to user's code
 
 scaling:
   instance: t2.small
@@ -432,6 +443,10 @@ environmentOverrides:
         type: dynamo-db
         attributes:
           <<: *table-attributes
+      - name: audit-log
+        type: dynamo-db
+        attributes:
+          <<: *audit-log-table-attributes
 
     alarms:
       overrides:
@@ -535,6 +550,10 @@ environmentOverrides:
         type: dynamo-db
         attributes:
           <<: *table-attributes
+      - name: audit-log
+        type: dynamo-db
+        attributes:
+          <<: *audit-log-table-attributes
 
       - type: globaledge
         name: proxy
@@ -636,6 +655,10 @@ environmentOverrides:
         type: dynamo-db
         attributes:
           <<: *table-attributes
+      - name: audit-log
+        type: dynamo-db
+        attributes:
+          <<: *audit-log-table-attributes
 
       - type: globaledge
         name: proxy

src/config/env.ts

@@ -141,6 +141,8 @@ export interface EnvVars {
 	//DyamoDB for deployment status history
 	DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION: string;
 	DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME: string;
+	DYNAMO_AUDIT_LOG_TABLE_NAME: string;
+	DYNAMO_AUDIT_LOG_TABLE_REGION: string;
 
 	// Micros Lifecycle Env Vars
 	SNS_NOTIFICATION_LIFECYCLE_QUEUE_URL?: string;

src/routes/api/api-router.ts

@@ -30,6 +30,7 @@ import { ApiReplyFailedEntitiesFromDataDepotPost } from "./api-replay-failed-ent
 import { RepoSyncState } from "models/reposyncstate";
 import { ApiResyncFailedTasksPost } from "./api-resync-failed-tasks";
 import { GHESVerificationRouter } from "./ghes-app-verification/ghes-app-verification-router";
+import { AuditLogApiRouter } from "./audit-log/audit-log-api-router";
 
 export const ApiRouter = Router();
 
@@ -226,6 +227,7 @@ ApiRouter.post("/reset-failed-pending-deployment-cursor", ResetFailedAndPendingD
 ApiRouter.post("/replay-rejected-entities-from-data-depot", ApiReplyFailedEntitiesFromDataDepotPost);
 ApiRouter.post("/resync-failed-tasks",ApiResyncFailedTasksPost);
 ApiRouter.use("/verify/githubapp/:gitHubAppId", GHESVerificationRouter);
+ApiRouter.use("/audit-log", AuditLogApiRouter);
 
 ApiRouter.use("/jira", ApiJiraRouter);
 ApiRouter.use("/:installationId", param("installationId").isInt(), returnOnValidationError, ApiInstallationRouter);

src/routes/api/audit-log/audit-log-api-router.ts

@@ -0,0 +1,15 @@
+import { Router } from "express";
+import { ApiAuditLogGetBySubscriptionId } from "./audit-log-get-by-sub-id";
+import { param, query } from "express-validator";
+import { returnOnValidationError } from "../api-utils";
+
+export const AuditLogApiRouter = Router({ mergeParams: true });
+
+AuditLogApiRouter.get("/subscription/:subscriptionId",
+	param("subscriptionId").isInt(),
+	query("entityType").isString(),
+	query("entityId").isString(),
+	query("issueKey").isString(),
+	returnOnValidationError,
+	ApiAuditLogGetBySubscriptionId
+);

src/routes/api/audit-log/audit-log-get-by-sub-id.test.ts

@@ -0,0 +1,62 @@
+import supertest from "supertest";
+import { when } from "jest-when";
+import { omit } from "lodash";
+import { getFrontendApp } from "../../../app";
+import { DatabaseStateCreator, CreatorResult } from "test/utils/database-state-creator";
+import { findLog } from "services/audit-log-service";
+
+jest.mock("services/audit-log-service");
+
+describe("AuditLogApiGetBySubscriptionId", () => {
+
+	const makeApiCall = (subscriptionId: string | number, params: Record<string, unknown>) => {
+		return supertest(getFrontendApp())
+			.get(`/api/audit-log/subscription/${subscriptionId}`)
+			.query(params)
+			.set("X-Slauth-Mechanism", "test")
+			.send();
+	};
+
+	let db: CreatorResult;
+	let params;
+
+	beforeEach(async () => {
+		db = await new DatabaseStateCreator().forServer().create();
+		params = {
+			issueKey: "ABC-123",
+			entityType: "commit",
+			entityId: "abcd-efgh-ijkl"
+		};
+	});
+
+	describe.each(["issueKey", "entityType", "entityId"])("param validation", (paramKey) => {
+		it(`should return 422 on missing param ${paramKey}`, async () => {
+			await makeApiCall(db.subscription.id, omit(params, paramKey))
+				.expect(422);
+		});
+	});
+
+	it(`should return error on missing subscription`, async () => {
+		await makeApiCall(db.subscription.id + 1, params)
+			.expect(500);
+	});
+
+
+	it("should return audit log data successfully", async () => {
+
+		when(findLog).calledWith({
+			subscriptionId: db.subscription.id,
+			issueKey: "ABC-123",
+			entityType: "commit",
+			entityId: "abcd-efgh-ijkl"
+		}, expect.anything()).mockResolvedValue({
+			name: "hello"
+		} as any);
+
+		const result = await makeApiCall(db.subscription.id, params).expect(200);
+
+		expect(result.body).toEqual({
+			name: "hello"
+		});
+	});
+});

src/routes/api/audit-log/audit-log-get-by-sub-id.ts

@@ -0,0 +1,27 @@
+import { Request, Response } from "express";
+import { Subscription } from "models/subscription";
+import { findLog, AuditInfoPK } from "services/audit-log-service";
+
+export const ApiAuditLogGetBySubscriptionId = async (req: Request, res: Response): Promise<void> => {
+
+	const { subscriptionId } = req.params;
+	const { issueKey, entityType, entityId } = req.query;
+
+	const subscription = await Subscription.findByPk(subscriptionId);
+
+	if (subscription === null) {
+		throw new Error("Cannot find subscription by id " + subscriptionId);
+	}
+
+	const auditInfo: AuditInfoPK = {
+		subscriptionId: Number(subscriptionId),
+		issueKey: String(issueKey),
+		entityType: String(entityType),
+		entityId: String(entityId)
+	};
+
+	const result = await findLog(auditInfo, req.log);
+
+	res.status(200).json(result);
+
+};

src/services/audit-log-service.test.ts

@@ -0,0 +1,113 @@
+import { getLogger } from "config/logger";
+import { envVars } from "config/env";
+import { dynamodb as ddb } from "config/dynamodb";
+import { createHashWithoutSharedSecret } from "utils/encryption";
+import { auditLog, findLog } from "./audit-log-service";
+
+const logger = getLogger("test");
+const ONE_DAY_IN_MILLISECONDS = 24 * 60 * 60 * 1000;
+
+describe("audit log service", () => {
+	describe("auditLog", () => {
+		it("should successfully save DD api call audit info to dynamo db", async () => {
+			const createdAt = new Date();
+			const subscriptionId = 241412;
+			const entityId = "25e1008";
+			const entityAction = "pushed";
+			const entityType = "commit";
+			const source = "backfill";
+			const issueKey = "ARC-2605";
+			const ID = `subID_${subscriptionId}_typ_${entityType}_id_${entityId}_issKey_${issueKey}`;
+			await auditLog(
+				{
+					source,
+					entityType,
+					entityAction,
+					entityId,
+					subscriptionId,
+					issueKey,
+					createdAt
+				},
+				logger
+			);
+			const result = await ddb
+				.getItem({
+					TableName: envVars.DYNAMO_AUDIT_LOG_TABLE_NAME,
+					Key: {
+						Id: { S: createHashWithoutSharedSecret(ID) },
+						CreatedAt: { N: String(createdAt.getTime()) }
+					},
+					AttributesToGet: [
+						"Id",
+						"CreatedAt",
+						"ExpiredAfter",
+						"source",
+						"entityType",
+						"entityAction",
+						"entityId",
+						"subscriptionId",
+						"issueKey"
+					]
+				})
+				.promise();
+			expect(result.$response.error).toBeNull();
+			expect(result.Item).toEqual({
+				Id: { S: createHashWithoutSharedSecret(ID) },
+				CreatedAt: { N: String(createdAt.getTime()) },
+				ExpiredAfter: {
+					N: String(
+						Math.floor((createdAt.getTime() + ONE_DAY_IN_MILLISECONDS) / 1000)
+					)
+				},
+				source: { S: source },
+				entityAction: { S: entityAction },
+				entityId: { S: entityId },
+				entityType: { S: entityType },
+				issueKey: { S: issueKey },
+				subscriptionId: { N: String(subscriptionId) }
+			});
+		});
+
+		describe("auditLog", () => {
+			it("should successfully save DD api call audit info to dynamo db", async () => {
+				const createdAt = new Date();
+				const subscriptionId = 241412;
+				const entityId = "25e1008";
+				const entityAction = "pushed";
+				const entityType = "commit";
+				const source = "backfill";
+				const issueKey = "ARC-2605";
+				await auditLog(
+					{
+						source,
+						entityType,
+						entityAction,
+						entityId,
+						subscriptionId,
+						issueKey,
+						createdAt
+					},
+					logger
+				);
+				const result = await findLog(
+					{
+						entityType,
+						entityId,
+						subscriptionId,
+						issueKey
+					},
+					logger
+				);
+				expect(result).toEqual([{
+					entityAction,
+					entityId,
+					entityType,
+					issueKey,
+					source,
+					subscriptionId,
+					createdAt
+				}]);
+			});
+		});
+	});
+});