chore: add new endpoint for backfill

spa/src/api/subscriptions/index.ts

@@ -1,5 +1,7 @@
+import { RestSyncReqBody } from "~/src/rest-interfaces";
 import { axiosRest } from "../axiosInstance";
 
 export default {
-	getSubscriptions: () => axiosRest.get("/rest/subscriptions")
+	getSubscriptions: () => axiosRest.get("/rest/subscriptions"),
+	syncSubscriptions: (data: RestSyncReqBody) => axiosRest.post(`/rest/app/cloud/sync`, data),
 };

src/rest-interfaces/index.ts

@@ -1,3 +1,9 @@
+export type RestSyncReqBody = {
+	installationId: number;
+	syncType: string;
+	source: string;
+	commitsFromDate: string;
+}
 
 export type GetRedirectUrlResponse = {
 	redirectUrl: string;

src/rest/rest-router.ts

@@ -1,5 +1,6 @@
 import { Router } from "express";
 import { JwtHandler } from "./middleware/jwt/jwt-handler";
+import { body } from "express-validator";
 import { OAuthRouter } from "./routes/oauth";
 import { OAuthCallbackHandler, OrgsInstalledHandler, OrgsInstallRequestedHandler } from "./routes/github-callback";
 import { GitHubOrgsRouter } from "./routes/github-orgs";
@@ -10,6 +11,7 @@ import { RestErrorHandler } from "./middleware/error";
 import { JiraAdminEnforceMiddleware } from "./middleware/jira-admin/jira-admin-check";
 import { AnalyticsProxyHandler } from "./routes/analytics-proxy";
 import { SubscriptionsRouter } from "./routes/subscriptions";
+import { SyncRouterHandler } from "./routes/sync";
 import { DeferredRouter } from "./routes/deferred";
 
 export const RestRouter = Router({ mergeParams: true });
@@ -27,6 +29,14 @@ RestRouter.use("/subscriptions", JwtHandler, JiraAdminEnforceMiddleware, Subscri
  */
 RestRouter.use("/app/:cloudOrUUID", subRouter);
 
+subRouter.post(
+	"/sync",
+	body("commitsFromDate").optional().isISO8601(),
+	JwtHandler,
+	JiraAdminEnforceMiddleware,
+	SyncRouterHandler
+);
+
 subRouter.get("/github-callback", OAuthCallbackHandler);
 subRouter.get("/github-installed", OrgsInstalledHandler);
 subRouter.get("/github-requested", OrgsInstallRequestedHandler);

src/rest/routes/sync/index.test.ts

@@ -0,0 +1,155 @@
+import { getFrontendApp } from "~/src/app";
+import { Installation } from "models/installation";
+import { Subscription } from "models/subscription";
+import express, { Express } from "express";
+import { RootRouter } from "routes/router";
+import supertest from "supertest";
+import { encodeSymmetric } from "atlassian-jwt";
+import { GitHubServerApp } from "models/github-server-app";
+import { v4 as newUUID } from "uuid";
+import { sqsQueues } from "~/src/sqs/queues";
+
+jest.mock("~/src/sqs/queues");
+jest.mock("config/feature-flags");
+
+describe("Checking the deferred request parsing route", () => {
+	let app: Express;
+	let installation: Installation;
+	const installationIdForCloud = 1;
+	const installationIdForServer = 2;
+	let gitHubServerApp: GitHubServerApp;
+	// let jwt: string;
+	const testSharedSecret = "test-secret";
+	const clientKey =  "jira-client-key";
+	const getToken = ({
+		secret = testSharedSecret,
+		iss = clientKey,
+		exp = Date.now() / 1000 + 10000,
+		qsh = "context-qsh",
+		sub = "myAccount" } = {}): string => {
+		return encodeSymmetric({
+			qsh,
+			iss,
+			exp,
+			sub
+		}, secret);
+	};
+	beforeEach(async () => {
+		app = getFrontendApp();
+		installation = await Installation.install({
+			host: jiraHost,
+			sharedSecret: testSharedSecret,
+			clientKey: clientKey
+		});
+		await Subscription.install({
+			installationId: installationIdForCloud,
+			host: jiraHost,
+			hashedClientKey: installation.clientKey,
+			gitHubAppId: undefined
+		});
+		gitHubServerApp = await GitHubServerApp.install({
+			uuid: newUUID(),
+			appId: 123,
+			gitHubAppName: "My GitHub Server App",
+			gitHubBaseUrl: gheUrl,
+			gitHubClientId: "lvl.1234",
+			gitHubClientSecret: "myghsecret",
+			webhookSecret: "mywebhooksecret",
+			privateKey: "myprivatekey",
+			installationId: installation.id
+		}, jiraHost);
+		await Subscription.install({
+			installationId: installationIdForServer,
+			host: jiraHost,
+			hashedClientKey: installation.clientKey,
+			gitHubAppId: gitHubServerApp.id
+		});
+		app = express();
+		app.use(RootRouter);
+	});
+
+	describe("cloud", () => {
+		it("should throw 401 error when no github token is passed", async () => {
+			const resp = await supertest(app)
+				.get("/rest/app/cloud/sync");
+
+			expect(resp.status).toEqual(401);
+		});
+
+		it("should return 202 on correct post for /rest/app/cloud/sync one for Cloud app", async () => {
+			return supertest(app)
+				.post("/rest/app/cloud/sync")
+				.set("authorization", `${getToken()}`)
+				.send({
+					installationId: installationIdForCloud,
+					jiraHost
+				})
+				.expect(202)
+				.then(() => {
+					expect(sqsQueues.backfill.sendMessage).toBeCalledWith(expect.objectContaining({
+						installationId: installationIdForCloud,
+						jiraHost,
+						startTime: expect.anything(),
+						gitHubAppConfig: expect.objectContaining({ gitHubAppId: undefined, uuid: undefined })
+					}), expect.anything(), expect.anything());
+				});
+		});
+
+		it("should run incremental sync", async() => {
+			const commitsFromDate = new Date(new Date().getTime() - 2000);
+			const backfillSince = new Date(new Date().getTime() - 1000);
+			const subscription = await Subscription.getSingleInstallation(
+				jiraHost,
+				installationIdForServer,
+				gitHubServerApp.id
+			);
+			await subscription?.update({
+				syncStatus: "COMPLETE",
+				backfillSince
+			});
+			return supertest(app)
+				.post(`/rest/app/${gitHubServerApp.id}/sync`)
+				.set("authorization", `${getToken()}`)
+				.send({
+					installationId: installationIdForServer,
+					jiraHost,
+					commitsFromDate
+				})
+				.expect(202)
+				.then(() => {
+					expect(sqsQueues.backfill.sendMessage).toBeCalledWith(expect.objectContaining({
+						syncType: "partial",
+						installationId: installationIdForServer,
+						jiraHost,
+						commitsFromDate: commitsFromDate.toISOString(),
+						targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert", "codeScanningAlert"],
+						gitHubAppConfig: expect.objectContaining({ gitHubAppId: gitHubServerApp.id, uuid: gitHubServerApp.uuid })
+					}), expect.anything(), expect.anything());
+				});
+		});
+
+		it("should run full sync if explicitly selected by user", async () => {
+			const commitsFromDate = new Date(new Date().getTime() - 2000);
+			return supertest(app)
+				.post(`/rest/app/${gitHubServerApp.id}/sync`)
+				.set("authorization", `${getToken()}`)
+				.send({
+					installationId: installationIdForServer,
+					jiraHost,
+					commitsFromDate,
+					syncType: "full"
+				})
+				.expect(202)
+				.then(() => {
+					expect(sqsQueues.backfill.sendMessage).toBeCalledWith(expect.objectContaining({
+						syncType: "full",
+						installationId: installationIdForServer,
+						jiraHost,
+						commitsFromDate: commitsFromDate.toISOString(),
+						targetTasks: undefined,
+						gitHubAppConfig: expect.objectContaining({ gitHubAppId: gitHubServerApp.id, uuid: gitHubServerApp.uuid })
+					}), expect.anything(), expect.anything());
+				});
+		});
+	});
+});

src/rest/routes/sync/index.ts

@@ -0,0 +1,74 @@
+import { Request, Response } from "express";
+import { ParamsDictionary } from "express-serve-static-core";
+import { errorWrapper } from "../../helper";
+import { Subscription } from "models/subscription";
+import { findOrStartSync } from "~/src/sync/sync-utils";
+import { determineSyncTypeAndTargetTasks } from "~/src/util/github-sync-helper";
+import { BaseLocals } from "..";
+import { RestApiError } from "~/src/config/errors";
+import { RestSyncReqBody } from "~/src/rest-interfaces";
+
+
+const restSyncPost = async (
+	req: Request<ParamsDictionary, unknown, RestSyncReqBody>,
+	res: Response<unknown, BaseLocals>
+) => {
+	//TODO: We are yet to handle enterprise backfill
+	const cloudOrUUID = req.params.cloudOrUUID;
+	const gheUUID = cloudOrUUID === "cloud" ? undefined : req.params.cloudOrUUID;
+	let gitHubAppId : number | undefined = undefined;
+	if (gheUUID) {
+		gitHubAppId = parseFloat(gheUUID);
+	}
+
+	const {
+		installationId: gitHubInstallationId,
+		syncType: syncTypeFromReq,
+		source
+	} = req.body;
+	// A date to start fetching commit history(main and branch) from.
+	const commitsFromDate = req.body.commitsFromDate
+		? new Date(req.body.commitsFromDate)
+		: undefined;
+
+	try {
+		const subscription = await Subscription.getSingleInstallation(
+			res.locals.installation.jiraHost,
+			gitHubInstallationId,
+			gitHubAppId
+		);
+		if (!subscription) {
+			req.log.info(
+				{
+					jiraHost: res.locals.installation.jiraHost,
+					installationId: gitHubInstallationId
+				},
+				"Subscription not found when retrying sync."
+			);
+			throw new RestApiError(400, "INVALID_OR_MISSING_ARG", "Subscription not found, cannot resync.");
+		}
+
+		if (commitsFromDate && commitsFromDate.valueOf() > Date.now()) {
+			throw new RestApiError(400, "INVALID_OR_MISSING_ARG", "Invalid date value, cannot select a future date");
+		}
+
+		const { syncType, targetTasks } = determineSyncTypeAndTargetTasks(
+			syncTypeFromReq,
+			subscription
+		);
+		await findOrStartSync(
+			subscription,
+			req.log,
+			syncType,
+			commitsFromDate || subscription.backfillSince,
+			targetTasks,
+			{ source }
+		);
+
+		res.sendStatus(202);
+	} catch (error: unknown) {
+		throw new RestApiError(500, "UNKNOWN", "Something went wrong");
+	}
+};
+
+export const SyncRouterHandler = errorWrapper("AnalyticsProxyHandler",restSyncPost);

src/routes/jira/sync/jira-sync-post.ts

@@ -1,11 +1,11 @@
-import { Subscription, SyncStatus } from "models/subscription";
+import { Subscription } from "models/subscription";
 import * as Sentry from "@sentry/node";
 import { NextFunction, Request, Response } from "express";
 import { findOrStartSync } from "~/src/sync/sync-utils";
 import { sendAnalytics } from "utils/analytics-client";
 import { AnalyticsEventTypes, AnalyticsTrackEventsEnum, AnalyticsTrackSource } from "interfaces/common";
-import { TaskType, SyncType } from "~/src/sync/sync.types";
 import { booleanFlag, BooleanFlags } from "config/feature-flags";
+import { determineSyncTypeAndTargetTasks, getStartTimeInDaysAgo } from "../../../util/github-sync-helper";
 
 export const JiraSyncPost = async (req: Request, res: Response, next: NextFunction): Promise<void> => {
 	const { installationId: gitHubInstallationId, appId: gitHubAppId, syncType: syncTypeFromReq, source } = req.body;
@@ -35,7 +35,7 @@ export const JiraSyncPost = async (req: Request, res: Response, next: NextFuncti
 			return;
 		}
 
-		const { syncType, targetTasks } = await determineSyncTypeAndTargetTasks(syncTypeFromReq, subscription);
+		const { syncType, targetTasks } = determineSyncTypeAndTargetTasks(syncTypeFromReq, subscription);
 		await findOrStartSync(subscription, req.log, syncType, commitsFromDate || subscription.backfillSince, targetTasks, { source });
 
 		await sendAnalytics(res.locals.jiraHost, AnalyticsEventTypes.TrackEvent, {
@@ -64,26 +64,3 @@ export const JiraSyncPost = async (req: Request, res: Response, next: NextFuncti
 		next(new Error("Unauthorized"));
 	}
 };
-
-const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000;
-const getStartTimeInDaysAgo = (commitsFromDate: Date | undefined) => {
-	if (commitsFromDate === undefined) return undefined;
-	return Math.floor((Date.now() -  commitsFromDate?.getTime()) / MILLISECONDS_IN_ONE_DAY);
-};
-
-type SyncTypeAndTargetTasks = {
-	syncType: SyncType,
-	targetTasks: TaskType[] | undefined,
-};
-
-const determineSyncTypeAndTargetTasks = async (syncTypeFromReq: string, subscription: Subscription): Promise<SyncTypeAndTargetTasks> => {
-	if (syncTypeFromReq === "full") {
-		return { syncType: "full", targetTasks: undefined };
-	}
-
-	if (subscription.syncStatus === SyncStatus.FAILED) {
-		return { syncType: "full", targetTasks: undefined };
-	}
-
-	return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert", "codeScanningAlert"] };
-};

src/util/github-sync-helper.ts

@@ -0,0 +1,26 @@
+import { Subscription, SyncStatus } from "models/subscription";
+import { TaskType, SyncType } from "~/src/sync/sync.types";
+
+
+const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000;
+export const getStartTimeInDaysAgo = (commitsFromDate: Date | undefined) => {
+	if (commitsFromDate === undefined) return undefined;
+	return Math.floor((Date.now() -  commitsFromDate.getTime()) / MILLISECONDS_IN_ONE_DAY);
+};
+
+type SyncTypeAndTargetTasks = {
+	syncType: SyncType,
+	targetTasks: TaskType[] | undefined,
+};
+
+export const determineSyncTypeAndTargetTasks = (syncTypeFromReq: string, subscription: Subscription): SyncTypeAndTargetTasks => {
+	if (syncTypeFromReq === "full") {
+		return { syncType: "full", targetTasks: undefined };
+	}
+
+	if (subscription.syncStatus === SyncStatus.FAILED) {
+		return { syncType: "full", targetTasks: undefined };
+	}
+
+	return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert", "codeScanningAlert"] };
+};

test/snapshots/app.test.ts.snap

@@ -15,6 +15,8 @@ exports[`app getFrontendApp please review routes and update snapshot when adding
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,serveStatic
 :GET ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/?$
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionsGet
+:POST ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/sync/?$
+	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,middleware,JwtHandler,jiraAdminEnforceMiddleware,AnalyticsProxyHandler
 :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/github-callback/?$
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,OAuthCallbackHandler
 :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/github-installed/?$