[FAI-14747] Support ingesting deployments data from bigquery (#1892)

destinations/airbyte-faros-destination/resources/source-specific-configs/bigquery.json

@@ -0,0 +1,17 @@
+{
+  "title": "BigQuery",
+  "type": "object",
+  "oneOf": [
+    {
+      "type": "object",
+      "title": "Configuration",
+      "properties": {
+        "source_type": {
+          "type": "string",
+          "const": "BigQuery",
+          "order": 0
+        }
+      }
+    }
+  ]
+}

destinations/airbyte-faros-destination/resources/source-specific-configs/spec.json

@@ -33,6 +33,9 @@
         "bamboohr": {
           "$ref": "bamboohr.json"
         },
+        "bigquery": {
+          "$ref": "bigquery.json"
+        },
         "bitbucket": {
           "$ref": "bitbucket.json"
         },

destinations/airbyte-faros-destination/src/converters/bigquery/common.ts

@@ -0,0 +1,14 @@
+import {Converter, StreamContext} from '../converter';
+
+interface BigQueryConfig {
+}
+
+/** BigQuery converter base */
+export abstract class BigQueryConverter extends Converter {
+  source = 'BigQuery';
+
+
+  protected bigqueryConfig(ctx: StreamContext): BigQueryConfig {
+    return ctx.config?.source_specific_configs?.bigquery;
+  }
+}

destinations/airbyte-faros-destination/src/converters/bigquery/deployments.ts

@@ -0,0 +1,97 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord} from '../converter';
+import {BigQueryConverter} from './common';
+
+export class Deployments extends BigQueryConverter {
+  id(record: AirbyteRecord): string {
+    return record.record.data.id;
+  }
+  readonly destinationModels: ReadonlyArray<DestinationModel> = [
+    'cicd_Build',
+    'cicd_Deployment',
+    'vcs_Commit',
+    'cicd_DeploymentChangeset'
+  ];
+
+  async convert(
+    record: AirbyteRecord
+  ): Promise<ReadonlyArray<DestinationRecord>> {
+    const res: DestinationRecord[] = [];
+
+    //cicd_Deployment
+    const uid = record.record.data.uid;
+    const startedAt = record.record.data.started_at;
+    const endedAt = record.record.data.ended_at;
+    const env = record.record.data.env;
+    const status = record.record.data.status;
+    const url = record.record.data.url;
+    const source = record.record.data.source || this.source;
+
+    //cicd_Build
+    const buildUid = `${record.record.data.build_uid}`;
+    const buildNumber = record.record.data.build_number;
+    const buildUrl = record.record.data.url;
+
+    //vcs_Commit & cicd_DeploymentChangeset
+    const commit = record.record.data.commit_sha;
+
+    if (uid) {
+      res.push({
+        model: 'cicd_Deployment',
+        record: {
+          uid: uid,
+          startedAt: startedAt,
+          endedAt: endedAt,
+          env: env,
+          status: status,
+          url: url,
+          source: source,
+          build: {
+            uid: buildUid,
+          },
+          changeset: {
+            commit: {
+              sha: commit,
+            },
+          },
+        },
+      });
+    }
+
+    if (buildUid) {
+      res.push({
+        model: 'cicd_Build',
+        record: {
+          uid: buildUid,
+          number: buildNumber,
+          url: buildUrl,
+        },
+      });
+    }
+
+    if (commit) {
+      res.push({
+        model: 'vcs_Commit',
+        record: {
+          sha: commit,
+        },
+      });
+
+      res.push({
+        model: 'cicd_DeploymentChangeset',
+        record: {
+          commit: {
+            sha: commit,
+          },
+          deployment: {
+            uid: uid,
+            source: source,
+          },
+        },
+      });
+    }
+
+    return res;
+  }
+}

destinations/airbyte-faros-destination/test/converters/__snapshots__/bigquery.test.ts.snap

@@ -0,0 +1,12 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`bigquery basic test process records from all streams 1`] = `
+Array [
+  "Processed 7 records",
+  "Processed records by stream: {\\\\\\"bigquery__bigquery__deployments\\\\\\":7}\\"},\\"type\\":\\"LOG\\"}",
+  "Would write 28 records",
+  "Would write records by model: {\\\\\\"cicd_Build\\\\\\":7,\\\\\\"cicd_Deployment\\\\\\":7,\\\\\\"cicd_DeploymentChangeset\\\\\\":7,\\\\\\"vcs_Commit\\\\\\":7}\\"},\\"type\\":\\"LOG\\"}",
+  "Skipped 0 records",
+  "Errored 0 records",
+]
+`;

destinations/airbyte-faros-destination/test/converters/bigquery.test.ts

@@ -0,0 +1,3 @@
+import {generateBasicTestSuite} from './utils';
+
+generateBasicTestSuite({sourceName: 'bigquery'});

destinations/airbyte-faros-destination/test/converters/data.ts

@@ -140,3 +140,6 @@ export const zendeskAllStreamsLog = readTestResourceFile(
 export const trelloAllStreamsLog = readTestResourceFile(
   'trello/all-streams.log'
 );
+export const bigqueryAllStreamsLog = readTestResourceFile(
+  'bigquery/all-streams.log'
+);

destinations/airbyte-faros-destination/test/resources/bigquery/all-streams.log

@@ -0,0 +1,7 @@
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-08-03T21:01:13Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid189","env":"Staging","status":"success","build_uid":107808,"build_number":107808,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"f78b22e9ac7c610a3f5a04bd5ac8df680d9881e3","client_build_number":18900900107808},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-14T22:20:52Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid202","env":"Production","status":"success","build_uid":133645,"build_number":133645,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"5fd2f9bea33f6a9545fc92a8947013e86f66c292","client_build_number":20301000133645},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-06T19:46:35Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid203","env":"Development","status":"failure","build_uid":132209,"build_number":132209,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"209352e0595c86e93759b5c11b6284c2be13edcd","client_build_number":20301000132209},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T19:13:23Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid204","env":"Staging","status":"failure","build_uid":132502,"build_number":132502,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dfbdf1b9b97e77dd9cc321556adb4c55f0cd27a1","client_build_number":20301000132502},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T15:20:12Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid205","env":"Production","status":"failure","build_uid":132450,"build_number":132450,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"33693814967195dc940e987265797f5980c81e70","client_build_number":20301000132450},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T00:13:27Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid206","env":"Development","status":"success","build_uid":132354,"build_number":132354,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dafcc8a156fc30fed062f5ddd859ab34c712a827","client_build_number":20301000132354},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-09T21:14:34Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid207","env":"Staging","status":"success","build_uid":132962,"build_number":132962,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"9f4a555ab16b8c8f77229a470dddcb7145b0e2df","client_build_number":20301000132962},"emitted_at":1737154317860}}

destinations/airbyte-faros-destination/test/resources/bigquery/catalog.json

@@ -0,0 +1,10 @@
+{
+    "streams": [
+      {
+        "stream": {
+          "name": "bigquery__bigquery__deployments"
+        },
+        "destination_sync_mode": "append"
+      }
+    ]
+  }

source.json

@@ -0,0 +1,7 @@
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-08-03T21:01:13Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid189","env":"Staging","status":"success","build_uid":107808,"build_number":107808,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"f78b22e9ac7c610a3f5a04bd5ac8df680d9881e3","client_build_number":18900900107808},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-14T22:20:52Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid202","env":"Production","status":"success","build_uid":133645,"build_number":133645,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"5fd2f9bea33f6a9545fc92a8947013e86f66c292","client_build_number":20301000133645},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-06T19:46:35Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid203","env":"Development","status":"failure","build_uid":132209,"build_number":132209,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"209352e0595c86e93759b5c11b6284c2be13edcd","client_build_number":20301000132209},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T19:13:23Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid204","env":"Staging","status":"failure","build_uid":132502,"build_number":132502,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dfbdf1b9b97e77dd9cc321556adb4c55f0cd27a1","client_build_number":20301000132502},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T15:20:12Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid205","env":"Production","status":"failure","build_uid":132450,"build_number":132450,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"33693814967195dc940e987265797f5980c81e70","client_build_number":20301000132450},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T00:13:27Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid206","env":"Development","status":"success","build_uid":132354,"build_number":132354,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dafcc8a156fc30fed062f5ddd859ab34c712a827","client_build_number":20301000132354},"emitted_at":1737154317860}}
+{"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-09T21:14:34Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid207","env":"Staging","status":"success","build_uid":132962,"build_number":132962,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"9f4a555ab16b8c8f77229a470dddcb7145b0e2df","client_build_number":20301000132962},"emitted_at":1737154317860}}