diff --git a/.env.development b/.env.development index 4c71ab24bc..592343c523 100644 --- a/.env.development +++ b/.env.development @@ -17,6 +17,8 @@ GLOBAL_HASH_SECRET=testsecret #Dyamno for deployment status DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache +DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1 +DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log # The Postgres URL used to connect to the database and secret for encrypting data DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/jira-dev diff --git a/.env.e2e b/.env.e2e index 63d750e0b6..91a296c9ed 100644 --- a/.env.e2e +++ b/.env.e2e @@ -21,6 +21,8 @@ DEBUG=nock.* #Dyamno for deployment status DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache +DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1 +DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log MICROS_AWS_REGION=us-west-1 diff --git a/.env.test b/.env.test index d8b673a155..0f3a4bef4d 100644 --- a/.env.test +++ b/.env.test @@ -26,6 +26,8 @@ DEBUG=nock.* #Dyamno for deployment status DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1 DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache +DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1 +DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log MICROS_AWS_REGION=us-west-1 diff --git a/.localstack/dynamodb.sh b/.localstack/dynamodb.sh index 14bfc3a75c..2fb73a2ab0 100755 --- a/.localstack/dynamodb.sh +++ b/.localstack/dynamodb.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# DEPLOYMENT_HISTORY_CACHE_TABLE echo "===== creating dynamo table ${DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME} =====" awslocal dynamodb create-table \ @@ -15,8 +15,27 @@ awslocal dynamodb create-table \ ReadCapacityUnits=10,WriteCapacityUnits=5 echo "===== table ${DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME} created =====" + +# AUDIT_LOG_TABLE +echo "===== creating dynamo table ${DYNAMO_AUDIT_LOG_TABLE_NAME} =====" + +awslocal dynamodb create-table \ + --table-name $DYNAMO_AUDIT_LOG_TABLE_NAME \ + --key-schema \ + AttributeName=Id,KeyType=HASH \ + AttributeName=CreatedAt,KeyType=RANGE \ + --attribute-definitions \ + AttributeName=Id,AttributeType=S \ + AttributeName=CreatedAt,AttributeType=N \ + --region $DYNAMO_AUDIT_LOG_TABLE_REGION \ + --provisioned-throughput \ + ReadCapacityUnits=10,WriteCapacityUnits=5 + +echo "===== table ${DYNAMO_AUDIT_LOG_TABLE_NAME} created =====" + echo "===== checking now =====" +awslocal dynamodb list-tables --region $DYNAMO_AUDIT_LOG_TABLE_REGION awslocal dynamodb list-tables --region $DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION echo "===== check finished =====" diff --git a/docker-compose.yml b/docker-compose.yml index f89324fb97..c4013a3aba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,9 @@ services: environment: - DEFAULT_REGION=us-west-1 - DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME=deployment-history-cache + - DYNAMO_AUDIT_LOG_TABLE_NAME=audit-log - DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION=us-west-1 + - DYNAMO_AUDIT_LOG_TABLE_REGION=us-west-1 - LAMBDA_REMOTE_DOCKER=false - LAMBDA_EXECUTOR=local # runs lambda inside temp directory instead of new docker container - SQS_ENDPOINT_STRATEGY=off # sets the SQS queue domain/path to the legacy version diff --git a/github-for-jira.sd.yml b/github-for-jira.sd.yml index 51e6bd9082..96457d6e5f 100644 --- a/github-for-jira.sd.yml +++ b/github-for-jira.sd.yml @@ -180,6 +180,17 @@ resources: TTLAttributeName: ExpiredAfter dataType: - UGC/PrimaryIdentifier # Sha of a commit that point to user's code + - name: audit-log + type: dynamo-db + attributes: &audit-log-table-attributes + HashKeyName: Id + HashKeyType: "S" + RangeKeyName: CreatedAt + RangeKeyType: "N" + ReadWriteCapacityMode: ON_DEMAND + TTLAttributeName: ExpiredAfter + dataType: + - UGC/PrimaryIdentifier # Sha of a commit that point to user's code scaling: instance: t2.small @@ -432,6 +443,10 @@ environmentOverrides: type: dynamo-db attributes: <<: *table-attributes + - name: audit-log + type: dynamo-db + attributes: + <<: *audit-log-table-attributes alarms: overrides: @@ -535,6 +550,10 @@ environmentOverrides: type: dynamo-db attributes: <<: *table-attributes + - name: audit-log + type: dynamo-db + attributes: + <<: *audit-log-table-attributes - type: globaledge name: proxy @@ -636,6 +655,10 @@ environmentOverrides: type: dynamo-db attributes: <<: *table-attributes + - name: audit-log + type: dynamo-db + attributes: + <<: *audit-log-table-attributes - type: globaledge name: proxy diff --git a/src/config/env.ts b/src/config/env.ts index 664b97871e..4157be9c1c 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -141,6 +141,8 @@ export interface EnvVars { //DyamoDB for deployment status history DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_REGION: string; DYNAMO_DEPLOYMENT_HISTORY_CACHE_TABLE_NAME: string; + DYNAMO_AUDIT_LOG_TABLE_NAME: string; + DYNAMO_AUDIT_LOG_TABLE_REGION: string; // Micros Lifecycle Env Vars SNS_NOTIFICATION_LIFECYCLE_QUEUE_URL?: string; diff --git a/src/routes/api/api-router.ts b/src/routes/api/api-router.ts index a96fe5702b..10f48cc763 100644 --- a/src/routes/api/api-router.ts +++ b/src/routes/api/api-router.ts @@ -30,6 +30,7 @@ import { ApiReplyFailedEntitiesFromDataDepotPost } from "./api-replay-failed-ent import { RepoSyncState } from "models/reposyncstate"; import { ApiResyncFailedTasksPost } from "./api-resync-failed-tasks"; import { GHESVerificationRouter } from "./ghes-app-verification/ghes-app-verification-router"; +import { AuditLogApiRouter } from "./audit-log/audit-log-api-router"; export const ApiRouter = Router(); @@ -226,6 +227,7 @@ ApiRouter.post("/reset-failed-pending-deployment-cursor", ResetFailedAndPendingD ApiRouter.post("/replay-rejected-entities-from-data-depot", ApiReplyFailedEntitiesFromDataDepotPost); ApiRouter.post("/resync-failed-tasks",ApiResyncFailedTasksPost); ApiRouter.use("/verify/githubapp/:gitHubAppId", GHESVerificationRouter); +ApiRouter.use("/audit-log", AuditLogApiRouter); ApiRouter.use("/jira", ApiJiraRouter); ApiRouter.use("/:installationId", param("installationId").isInt(), returnOnValidationError, ApiInstallationRouter); diff --git a/src/routes/api/audit-log/audit-log-api-router.ts b/src/routes/api/audit-log/audit-log-api-router.ts new file mode 100644 index 0000000000..c9ada0c046 --- /dev/null +++ b/src/routes/api/audit-log/audit-log-api-router.ts @@ -0,0 +1,15 @@ +import { Router } from "express"; +import { ApiAuditLogGetBySubscriptionId } from "./audit-log-get-by-sub-id"; +import { param, query } from "express-validator"; +import { returnOnValidationError } from "../api-utils"; + +export const AuditLogApiRouter = Router({ mergeParams: true }); + +AuditLogApiRouter.get("/subscription/:subscriptionId", + param("subscriptionId").isInt(), + query("entityType").isString(), + query("entityId").isString(), + query("issueKey").isString(), + returnOnValidationError, + ApiAuditLogGetBySubscriptionId +); diff --git a/src/routes/api/audit-log/audit-log-get-by-sub-id.test.ts b/src/routes/api/audit-log/audit-log-get-by-sub-id.test.ts new file mode 100644 index 0000000000..09d5b6384f --- /dev/null +++ b/src/routes/api/audit-log/audit-log-get-by-sub-id.test.ts @@ -0,0 +1,62 @@ +import supertest from "supertest"; +import { when } from "jest-when"; +import { omit } from "lodash"; +import { getFrontendApp } from "../../../app"; +import { DatabaseStateCreator, CreatorResult } from "test/utils/database-state-creator"; +import { findLog } from "services/audit-log-service"; + +jest.mock("services/audit-log-service"); + +describe("AuditLogApiGetBySubscriptionId", () => { + + const makeApiCall = (subscriptionId: string | number, params: Record) => { + return supertest(getFrontendApp()) + .get(`/api/audit-log/subscription/${subscriptionId}`) + .query(params) + .set("X-Slauth-Mechanism", "test") + .send(); + }; + + let db: CreatorResult; + let params; + + beforeEach(async () => { + db = await new DatabaseStateCreator().forServer().create(); + params = { + issueKey: "ABC-123", + entityType: "commit", + entityId: "abcd-efgh-ijkl" + }; + }); + + describe.each(["issueKey", "entityType", "entityId"])("param validation", (paramKey) => { + it(`should return 422 on missing param ${paramKey}`, async () => { + await makeApiCall(db.subscription.id, omit(params, paramKey)) + .expect(422); + }); + }); + + it(`should return error on missing subscription`, async () => { + await makeApiCall(db.subscription.id + 1, params) + .expect(500); + }); + + + it("should return audit log data successfully", async () => { + + when(findLog).calledWith({ + subscriptionId: db.subscription.id, + issueKey: "ABC-123", + entityType: "commit", + entityId: "abcd-efgh-ijkl" + }, expect.anything()).mockResolvedValue({ + name: "hello" + } as any); + + const result = await makeApiCall(db.subscription.id, params).expect(200); + + expect(result.body).toEqual({ + name: "hello" + }); + }); +}); diff --git a/src/routes/api/audit-log/audit-log-get-by-sub-id.ts b/src/routes/api/audit-log/audit-log-get-by-sub-id.ts new file mode 100644 index 0000000000..9e5248acb9 --- /dev/null +++ b/src/routes/api/audit-log/audit-log-get-by-sub-id.ts @@ -0,0 +1,27 @@ +import { Request, Response } from "express"; +import { Subscription } from "models/subscription"; +import { findLog, AuditInfoPK } from "services/audit-log-service"; + +export const ApiAuditLogGetBySubscriptionId = async (req: Request, res: Response): Promise => { + + const { subscriptionId } = req.params; + const { issueKey, entityType, entityId } = req.query; + + const subscription = await Subscription.findByPk(subscriptionId); + + if (subscription === null) { + throw new Error("Cannot find subscription by id " + subscriptionId); + } + + const auditInfo: AuditInfoPK = { + subscriptionId: Number(subscriptionId), + issueKey: String(issueKey), + entityType: String(entityType), + entityId: String(entityId) + }; + + const result = await findLog(auditInfo, req.log); + + res.status(200).json(result); + +}; diff --git a/src/services/audit-log-service.test.ts b/src/services/audit-log-service.test.ts new file mode 100644 index 0000000000..a7866060eb --- /dev/null +++ b/src/services/audit-log-service.test.ts @@ -0,0 +1,113 @@ +import { getLogger } from "config/logger"; +import { envVars } from "config/env"; +import { dynamodb as ddb } from "config/dynamodb"; +import { createHashWithoutSharedSecret } from "utils/encryption"; +import { auditLog, findLog } from "./audit-log-service"; + +const logger = getLogger("test"); +const ONE_DAY_IN_MILLISECONDS = 24 * 60 * 60 * 1000; + +describe("audit log service", () => { + describe("auditLog", () => { + it("should successfully save DD api call audit info to dynamo db", async () => { + const createdAt = new Date(); + const subscriptionId = 241412; + const entityId = "25e1008"; + const entityAction = "pushed"; + const entityType = "commit"; + const source = "backfill"; + const issueKey = "ARC-2605"; + const ID = `subID_${subscriptionId}_typ_${entityType}_id_${entityId}_issKey_${issueKey}`; + await auditLog( + { + source, + entityType, + entityAction, + entityId, + subscriptionId, + issueKey, + createdAt + }, + logger + ); + const result = await ddb + .getItem({ + TableName: envVars.DYNAMO_AUDIT_LOG_TABLE_NAME, + Key: { + Id: { S: createHashWithoutSharedSecret(ID) }, + CreatedAt: { N: String(createdAt.getTime()) } + }, + AttributesToGet: [ + "Id", + "CreatedAt", + "ExpiredAfter", + "source", + "entityType", + "entityAction", + "entityId", + "subscriptionId", + "issueKey" + ] + }) + .promise(); + expect(result.$response.error).toBeNull(); + expect(result.Item).toEqual({ + Id: { S: createHashWithoutSharedSecret(ID) }, + CreatedAt: { N: String(createdAt.getTime()) }, + ExpiredAfter: { + N: String( + Math.floor((createdAt.getTime() + ONE_DAY_IN_MILLISECONDS) / 1000) + ) + }, + source: { S: source }, + entityAction: { S: entityAction }, + entityId: { S: entityId }, + entityType: { S: entityType }, + issueKey: { S: issueKey }, + subscriptionId: { N: String(subscriptionId) } + }); + }); + + describe("auditLog", () => { + it("should successfully save DD api call audit info to dynamo db", async () => { + const createdAt = new Date(); + const subscriptionId = 241412; + const entityId = "25e1008"; + const entityAction = "pushed"; + const entityType = "commit"; + const source = "backfill"; + const issueKey = "ARC-2605"; + await auditLog( + { + source, + entityType, + entityAction, + entityId, + subscriptionId, + issueKey, + createdAt + }, + logger + ); + const result = await findLog( + { + entityType, + entityId, + subscriptionId, + issueKey + }, + logger + ); + expect(result).toEqual([{ + entityAction, + entityId, + entityType, + issueKey, + source, + subscriptionId, + createdAt + }]); + }); + }); + }); +}); diff --git a/src/services/audit-log-service.ts b/src/services/audit-log-service.ts new file mode 100644 index 0000000000..b27be17619 --- /dev/null +++ b/src/services/audit-log-service.ts @@ -0,0 +1,108 @@ +import Logger from "bunyan"; +import { envVars } from "config/env"; +import { getLogger } from "config/logger"; +import { dynamodb as ddb } from "config/dynamodb"; +import { createHashWithoutSharedSecret } from "utils/encryption"; + +const defaultLogger = getLogger("DeploymentDynamoLogger"); + +export type AuditInfoPK = { + entityType: string; + entityId: string; + subscriptionId: number; + issueKey: string; +}; + +export type AuditInfo = AuditInfoPK & { + createdAt: Date; + entityAction: string; + source: string; +}; + +const ONE_DAY_IN_MILLISECONDS = 24 * 60 * 60 * 1000; + +export const auditLog = async (auditInfo: AuditInfo, logger: Logger) => { + logger.debug("Saving auditInfo to db"); + const { + source, + entityAction, + entityId, + entityType, + subscriptionId, + issueKey, + createdAt + } = auditInfo; + const result = await ddb + .putItem({ + TableName: envVars.DYNAMO_AUDIT_LOG_TABLE_NAME, + Item: { + Id: { S: getKey(auditInfo) }, //partition key + CreatedAt: { N: String(createdAt.getTime()) }, //sort key + ExpiredAfter: { + N: String( + Math.floor( + (auditInfo.createdAt.getTime() + ONE_DAY_IN_MILLISECONDS) / 1000 + ) + ) + }, //ttl + source: { S: source }, + entityAction: { S: entityAction }, + entityId: { S: entityId }, + entityType: { S: entityType }, + issueKey: { S: issueKey }, + subscriptionId: { N: String(subscriptionId) } + } + }) + .promise(); + if (result.$response.error) { + throw result.$response.error; + } +}; + +export const findLog = async ( + params: AuditInfoPK, + logger: Logger = defaultLogger +): Promise => { + logger.debug("Finding audit log for DD call"); + const result = await ddb + .query({ + TableName: envVars.DYNAMO_AUDIT_LOG_TABLE_NAME, + KeyConditionExpression: "Id = :id", + ExpressionAttributeValues: { + ":id": { S: getKey(params) } + }, + ScanIndexForward: false, + Limit: 1 + }) + .promise(); + + if (result.$response.error) { + throw result.$response.error; + } + + if (!result.Items?.length) { + return []; + } + + const items = result.Items; + + return items.map((item) => ({ + source: String(item.source.S), + entityType: String(item.entityType.S), + entityAction: String(item.entityAction.S), + entityId: String(item.entityId.S), + subscriptionId: Number(item.subscriptionId.N), + issueKey: String(item.issueKey.S), + createdAt: new Date(Number(item.CreatedAt.N)) + })); +}; + +/* + * The partition key (return of this function) + range key (creation time of the deployment status) will be the unique identifier of the each entry + */ +const getKey = (auditInfo: AuditInfoPK) => { + const { entityId, entityType, subscriptionId, issueKey } = auditInfo; + return createHashWithoutSharedSecret( + `subID_${subscriptionId}_typ_${entityType}_id_${entityId}_issKey_${issueKey}` + ); +}; diff --git a/test/snapshots/app.test.ts.snap b/test/snapshots/app.test.ts.snap index 0653193af7..663fb0d058 100644 --- a/test/snapshots/app.test.ts.snap +++ b/test/snapshots/app.test.ts.snap @@ -97,6 +97,8 @@ exports[`app getFrontendApp please review routes and update snapshot when adding query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,urlencodedParser,jsonParser,LogMiddleware,slauthMiddleware,rateLimit,ApiResyncFailedTasksPost :POST ^/?(?=/|$)^/api/?(?=/|$)^/verify/githubapp/(?:([^/]+?))/?(?=/|$)^/verify-get-apps/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,urlencodedParser,jsonParser,LogMiddleware,slauthMiddleware,rateLimit,GHESVerifyGetApps +:GET ^/?(?=/|$)^/api/?(?=/|$)^/audit-log/?(?=/|$)^/subscription/(?:([^/]+?))/?$ + query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,urlencodedParser,jsonParser,LogMiddleware,slauthMiddleware,rateLimit,middleware,middleware,middleware,middleware,returnOnValidationError,ApiAuditLogGetBySubscriptionId :POST ^/?(?=/|$)^/api/?(?=/|$)^/jira/?(?=/|$)^/(?:([^/]+?))/verify/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,urlencodedParser,jsonParser,LogMiddleware,slauthMiddleware,rateLimit,middleware,returnOnValidationError,ApiJiraVerifyPost :POST ^/?(?=/|$)^/api/?(?=/|$)^/jira/?(?=/|$)^/(?:([^/]+?))/uninstall/?$ diff --git a/test/utils/database-state-creator.ts b/test/utils/database-state-creator.ts index 3bc146e1ed..b2657429c5 100644 --- a/test/utils/database-state-creator.ts +++ b/test/utils/database-state-creator.ts @@ -7,7 +7,7 @@ import path from "path"; import { getHashedKey } from "models/sequelize"; import { v4 } from "uuid"; -interface CreatorResult { +export interface CreatorResult { installation: Installation; subscription: Subscription; repoSyncState: RepoSyncState | undefined;