FAI-13967, FAI-8202: Jira source add source name qualifier to write n…

…on-uid ids for multiple instances (#1867)

destinations/airbyte-faros-destination/resources/source-specific-configs/jira.json

@@ -37,6 +37,15 @@
           "title": "Additional Fields Array Limit",
           "description": "Truncates an additional field's array value to the given length.",
           "default": 50
+        },
+        "source_qualifier": {
+          "order": 37,
+          "type": "string",
+          "title": "Source Qualifier",
+          "description": "The qualifier to append as a suffix to the Jira source name to ensure uniqueness among entities with similar IDs when syncing multiple Jira instances, while preserving their original IDs, e.g. for Boards and Sprints.",
+          "examples": [
+            "Instance2"
+          ]
         }
       }
     }

destinations/airbyte-faros-destination/src/converters/jira/board_issues.ts

@@ -14,7 +14,7 @@ export class BoardIssues extends JiraConverter {
   ): Promise<ReadonlyArray<DestinationRecord>> {
     if (!this.useBoardOwnership(ctx)) return [];
     const issue = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const relation = {
       task: {uid: issue.key, source},
       board: {uid: String(issue.boardId), source},

destinations/airbyte-faros-destination/src/converters/jira/boards.ts

@@ -16,7 +16,7 @@ export class Boards extends JiraConverter {
     if (!this.useBoardOwnership(ctx)) return [];
     const board = record.record.data;
     const uid = board.id.toString();
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     return [
       {
         model: 'tms_TaskBoard',

destinations/airbyte-faros-destination/src/converters/jira/common.ts

@@ -37,9 +37,11 @@ export interface JiraConfig {
   exclude_fields?: string[];
   use_board_ownership?: boolean;
   truncate_limit?: number;
+  source_qualifier?: string;
 }
 
 export abstract class JiraConverter extends Converter {
+  private qualifiedSourceName = false;
   source = 'Jira';
 
   /** All Jira records should have id property */
@@ -51,6 +53,19 @@ export abstract class JiraConverter extends Converter {
     return ctx.config.source_specific_configs?.jira ?? {};
   }
 
+  /**
+   * If the source is not qualified, we need to add the qualifier to the source name
+   */
+  protected initializeSource(ctx: StreamContext): string {
+    if (this.qualifiedSourceName) {
+      return this.source;
+    }
+    const qualifier = this.jiraConfig(ctx).source_qualifier;
+    this.source = qualifier ? `${this.source}_${qualifier}` : this.source;
+    this.qualifiedSourceName = true;
+    return this.source;
+  }
+
   protected additionalFieldsArrayLimit(ctx: StreamContext): number {
     return (
       this.jiraConfig(ctx).additional_fields_array_limit ??

destinations/airbyte-faros-destination/src/converters/jira/epics.ts

@@ -14,7 +14,7 @@ export class Epics extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const epic = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const status = epic.fields.status ?? {};
     let description = null;
     if (typeof epic.fields.description === 'string') {

destinations/airbyte-faros-destination/src/converters/jira/faros_board_issues.ts

@@ -14,6 +14,7 @@ export class FarosBoardIssues extends JiraConverter {
     record: AirbyteRecord,
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
+    this.initializeSource(ctx);
     return this.alias.convert(record, ctx);
   }
 }

destinations/airbyte-faros-destination/src/converters/jira/faros_boards.ts

@@ -20,7 +20,7 @@ export class FarosBoards extends JiraConverter {
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const board = record.record.data;
     const uid = board.uid;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const results: DestinationRecord[] = [
       {
         model: 'tms_TaskBoard',

destinations/airbyte-faros-destination/src/converters/jira/faros_issue_additional_fields.ts

@@ -11,6 +11,7 @@ export class FarosIssueAdditionalFields extends JiraConverter {
     record: AirbyteRecord,
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
+    this.initializeSource(ctx);
     const issue = record.record.data as IssueCompact;
     return [this.convertAdditionalFieldsIssue(issue)];
   }

destinations/airbyte-faros-destination/src/converters/jira/faros_issue_pull_requests.ts

@@ -14,7 +14,7 @@ export class FarosIssuePullRequests extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const pullRequest = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     return [
       {
         model: 'tms_TaskPullRequestAssociation',

destinations/airbyte-faros-destination/src/converters/jira/faros_issues.ts

@@ -51,7 +51,7 @@ export class FarosIssues extends JiraConverter {
     const issue = record.record.data as Issue;
     this.seenIssues.add(issue.key);
 
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const results: DestinationRecord[] = [];
 
     if (issue.updateAdditionalFields) {
@@ -142,7 +142,7 @@ export class FarosIssues extends JiraConverter {
             'status',
             'source',
           ]),
-          project: {uid: issue.project, source: this.source},
+          project: {uid: issue.project, source},
         },
       });
     }
@@ -217,9 +217,7 @@ export class FarosIssues extends JiraConverter {
     return [...results, ...ancestors];
   }
 
-  async onProcessingComplete(
-    ctx: StreamContext
-  ): Promise<ReadonlyArray<DestinationRecord>> {
+  async onProcessingComplete(): Promise<ReadonlyArray<DestinationRecord>> {
     return this.convertDependencies();
   }
 

destinations/airbyte-faros-destination/src/converters/jira/faros_project_version_issues.ts

@@ -13,7 +13,7 @@ export class FarosProjectVersionIssues extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const issue = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     return [
       {
         model: 'tms_TaskReleaseRelationship',

destinations/airbyte-faros-destination/src/converters/jira/faros_project_versions.ts

@@ -14,6 +14,7 @@ export class FarosProjectVersions extends JiraConverter {
     record: AirbyteRecord,
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
+    this.source = this.initializeSource(ctx);
     return this.alias.convert(record, ctx);
   }
 }

destinations/airbyte-faros-destination/src/converters/jira/faros_projects.ts

@@ -12,7 +12,7 @@ export class FarosProjects extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const project = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     return [
       {
         model: 'tms_Project',

destinations/airbyte-faros-destination/src/converters/jira/faros_sprint_reports.ts

@@ -17,7 +17,7 @@ export class FarosSprintReports extends JiraConverter {
     const sprintUid = toString(sprintReport.sprintId);
     const boardUid = toString(sprintReport.boardId);
     const results: DestinationRecord[] = [];
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     for (const issue of sprintReport.issues || []) {
       const status = issue.status
         ? {

destinations/airbyte-faros-destination/src/converters/jira/faros_sprints.ts

@@ -15,7 +15,7 @@ export class FarosSprints extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const sprint = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const uid = toString(sprint.id);
     return [
       {

destinations/airbyte-faros-destination/src/converters/jira/faros_team_memberships.ts

@@ -13,14 +13,15 @@ export class FarosTeamMemberships extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const teamMembership = record.record.data;
+    const source = this.initializeSource(ctx);
     return [
       {
         model: 'tms_TeamMembership',
         record: {
-          team: {uid: teamMembership.teamId, source: this.streamName.source},
+          team: {uid: teamMembership.teamId, source},
           member: {
             uid: teamMembership.memberId,
-            source: this.streamName.source,
+            source,
           },
         },
       },

destinations/airbyte-faros-destination/src/converters/jira/faros_teams.ts

@@ -20,7 +20,7 @@ export class FarosTeams extends JiraConverter {
           name: team.displayName,
           parentTeam: {uid: allTeamsUid},
           teamChain: [team.id, allTeamsUid],
-          source: this.streamName.source,
+          source: this.initializeSource(ctx),
         },
       },
     ];

destinations/airbyte-faros-destination/src/converters/jira/faros_users.ts

@@ -14,6 +14,7 @@ export class FarosUsers extends JiraConverter {
     record: AirbyteRecord,
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
+    this.initializeSource(ctx);
     return this.alias.convert(record, ctx);
   }
 }

destinations/airbyte-faros-destination/src/converters/jira/issues.ts

@@ -430,7 +430,7 @@ export class Issues extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const issue = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const results: DestinationRecord[] = [];
 
     if (!this.fieldIdsByName) {

destinations/airbyte-faros-destination/src/converters/jira/project_versions.ts

@@ -27,7 +27,7 @@ export class ProjectVersions extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const projectVersion = record.record.data as ProjectVersion;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const results: DestinationRecord[] = [
       {
         model: 'tms_Release',

destinations/airbyte-faros-destination/src/converters/jira/projects.ts

@@ -15,7 +15,7 @@ export class Projects extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const project = record.record.data;
-    const source = this.streamName.source;
+    const source = this.initializeSource(ctx);
     const uid = project.key;
     const results: DestinationRecord[] = [];
     results.push({

destinations/airbyte-faros-destination/src/converters/jira/sprints.ts

@@ -82,6 +82,7 @@ export class Sprints extends JiraConverter {
     ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const sprint = record.record.data;
+    const source = this.initializeSource(ctx);
     if (!this.pointsFieldIdsByName) {
       this.pointsFieldIdsByName = Sprints.getFieldIdsByName(ctx);
     }
@@ -107,7 +108,7 @@ export class Sprints extends JiraConverter {
           completedPoints,
           startedAt: Utils.toDate(sprint.startDate),
           endedAt: Utils.toDate(sprint.endDate),
-          source: this.streamName.source,
+          source,
         },
       },
     ];

destinations/airbyte-faros-destination/src/converters/jira/users.ts

@@ -27,7 +27,7 @@ export class Users extends JiraConverter {
           uid,
           name: user.displayName,
           emailAddress: user.emailAddress,
-          source: this.streamName.source,
+          source: this.initializeSource(ctx),
           inactive: user.active != null && !user.active,
         },
       },

destinations/airbyte-faros-destination/src/converters/tromzo/findings.ts

@@ -1,11 +1,11 @@
 import {AirbyteRecord} from 'faros-airbyte-cdk';
 import {Utils} from 'faros-js-client';
+import {toNumber} from 'lodash';
 
 import {CategoryDetail} from '../common/common';
 import {Vulnerability} from '../common/sec';
 import {Converter, DestinationModel, DestinationRecord} from '../converter';
 import {GitHubCommon} from '../github/common';
-import {toNumber} from 'lodash';
 export class Findings extends Converter {
   readonly destinationModels: ReadonlyArray<DestinationModel> = [
     'cicd_ArtifactVulnerability',