[FAI-14747] Support ingesting deployments data from bigquery #1892
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| { | ||
| "title": "BigQuery", | ||
| "type": "object", | ||
| "oneOf": [ | ||
| { | ||
| "type": "object", | ||
| "title": "Configuration", | ||
| "properties": { | ||
| "source_type": { | ||
| "type": "string", | ||
| "const": "BigQuery", | ||
| "order": 0 | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| import {Converter, StreamContext} from '../converter'; | ||
|
|
||
| interface BigQueryConfig { | ||
| } | ||
|
|
||
| /** BigQuery converter base */ | ||
| export abstract class BigQueryConverter extends Converter { | ||
| source = 'BigQuery'; | ||
|
|
||
|
|
||
| protected bigqueryConfig(ctx: StreamContext): BigQueryConfig { | ||
| return ctx.config?.source_specific_configs?.bigquery; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| import {AirbyteRecord} from 'faros-airbyte-cdk'; | ||
|
|
||
| import {DestinationModel, DestinationRecord} from '../converter'; | ||
| import {BigQueryConverter} from './common'; | ||
|
|
||
| export class Deployments extends BigQueryConverter { | ||
| id(record: AirbyteRecord): string { | ||
| return record.record.data.id; | ||
| } | ||
| readonly destinationModels: ReadonlyArray<DestinationModel> = [ | ||
| 'cicd_Build', | ||
| 'cicd_Deployment', | ||
| 'vcs_Commit', | ||
| 'cicd_DeploymentChangeset' | ||
| ]; | ||
|
|
||
| async convert( | ||
| record: AirbyteRecord | ||
| ): Promise<ReadonlyArray<DestinationRecord>> { | ||
| const res: DestinationRecord[] = []; | ||
|
|
||
| //cicd_Deployment | ||
| const uid = record.record.data.uid; | ||
| const startedAt = record.record.data.started_at; | ||
| const endedAt = record.record.data.ended_at; | ||
| const env = record.record.data.env; | ||
| const status = record.record.data.status; | ||
| const url = record.record.data.url; | ||
| const source = record.record.data.source || this.source; | ||
|
|
||
| //cicd_Build | ||
| const buildUid = `${record.record.data.build_uid}`; | ||
| const buildNumber = record.record.data.build_number; | ||
| const buildUrl = record.record.data.url; | ||
|
|
||
| //vcs_Commit & cicd_DeploymentChangeset | ||
| const commit = record.record.data.commit_sha; | ||
|
|
||
| if (uid) { | ||
| res.push({ | ||
| model: 'cicd_Deployment', | ||
| record: { | ||
|
There was a problem hiding this comment. typically, for the changeset feed to work, we need the application; i understand we may not have it in the example, but maybe we should have support for it? |
||
| uid: uid, | ||
| startedAt: startedAt, | ||
| endedAt: endedAt, | ||
| env: env, | ||
| status: status, | ||
| url: url, | ||
| source: source, | ||
| build: { | ||
| uid: buildUid, | ||
| }, | ||
| changeset: { | ||
|
There was a problem hiding this comment. shouldn't those line be removed since we add the DeploymentChangeset object below? |
||
| commit: { | ||
| sha: commit, | ||
|
There was a problem hiding this comment. Typically, the commit sha of the deployment goes in a different object: vcs_Commit through a cicd_ArtifactCommitAssociation object through a dummy cicd_Artifact object through a dummy cicd_ArtifactDeployment object See what the changeset feed queries to find it here: https://github.com/faros-ai/feeds/blob/master/feeds/cicd/changeset-feed/resources/graphql/deployments.gql#L28 |
||
| }, | ||
| }, | ||
| }, | ||
| }); | ||
| } | ||
|
|
||
| if (buildUid) { | ||
| res.push({ | ||
| model: 'cicd_Build', | ||
| record: { | ||
| uid: buildUid, | ||
| number: buildNumber, | ||
| url: buildUrl, | ||
| }, | ||
| }); | ||
| } | ||
|
|
||
| if (commit) { | ||
| res.push({ | ||
| model: 'vcs_Commit', | ||
| record: { | ||
| sha: commit, | ||
| }, | ||
| }); | ||
|
|
||
| res.push({ | ||
| model: 'cicd_DeploymentChangeset', | ||
|
There was a problem hiding this comment. assuming you write the commit sha through a vcs_Commit object as stated above, I would remove this and let the changeset feed set it |
||
| record: { | ||
| commit: { | ||
| sha: commit, | ||
| }, | ||
| deployment: { | ||
| uid: uid, | ||
| source: source, | ||
| }, | ||
| }, | ||
| }); | ||
| } | ||
|
|
||
| return res; | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| // Jest Snapshot v1, https://goo.gl/fbAQLP | ||
|
|
||
| exports[`bigquery basic test process records from all streams 1`] = ` | ||
| Array [ | ||
| "Processed 7 records", | ||
| "Processed records by stream: {\\\\\\"bigquery__bigquery__deployments\\\\\\":7}\\"},\\"type\\":\\"LOG\\"}", | ||
| "Would write 28 records", | ||
| "Would write records by model: {\\\\\\"cicd_Build\\\\\\":7,\\\\\\"cicd_Deployment\\\\\\":7,\\\\\\"cicd_DeploymentChangeset\\\\\\":7,\\\\\\"vcs_Commit\\\\\\":7}\\"},\\"type\\":\\"LOG\\"}", | ||
| "Skipped 0 records", | ||
| "Errored 0 records", | ||
| ] | ||
| `; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| import {generateBasicTestSuite} from './utils'; | ||
|
|
||
| generateBasicTestSuite({sourceName: 'bigquery'}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-08-03T21:01:13Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid189","env":"Staging","status":"success","build_uid":107808,"build_number":107808,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"f78b22e9ac7c610a3f5a04bd5ac8df680d9881e3","client_build_number":18900900107808},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-14T22:20:52Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid202","env":"Production","status":"success","build_uid":133645,"build_number":133645,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"5fd2f9bea33f6a9545fc92a8947013e86f66c292","client_build_number":20301000133645},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-06T19:46:35Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid203","env":"Development","status":"failure","build_uid":132209,"build_number":132209,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"209352e0595c86e93759b5c11b6284c2be13edcd","client_build_number":20301000132209},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T19:13:23Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid204","env":"Staging","status":"failure","build_uid":132502,"build_number":132502,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dfbdf1b9b97e77dd9cc321556adb4c55f0cd27a1","client_build_number":20301000132502},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T15:20:12Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid205","env":"Production","status":"failure","build_uid":132450,"build_number":132450,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"33693814967195dc940e987265797f5980c81e70","client_build_number":20301000132450},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T00:13:27Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid206","env":"Development","status":"success","build_uid":132354,"build_number":132354,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dafcc8a156fc30fed062f5ddd859ab34c712a827","client_build_number":20301000132354},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-09T21:14:34Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid207","env":"Staging","status":"success","build_uid":132962,"build_number":132962,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"9f4a555ab16b8c8f77229a470dddcb7145b0e2df","client_build_number":20301000132962},"emitted_at":1737154317860}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| { | ||
| "streams": [ | ||
| { | ||
| "stream": { | ||
| "name": "bigquery__bigquery__deployments" | ||
| }, | ||
| "destination_sync_mode": "append" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-08-03T21:01:13Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid189","env":"Staging","status":"success","build_uid":107808,"build_number":107808,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"f78b22e9ac7c610a3f5a04bd5ac8df680d9881e3","client_build_number":18900900107808},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-14T22:20:52Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid202","env":"Production","status":"success","build_uid":133645,"build_number":133645,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"5fd2f9bea33f6a9545fc92a8947013e86f66c292","client_build_number":20301000133645},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-06T19:46:35Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/133645","ended_at":"2023-08-03T21:01:13Z","uid":"uid203","env":"Development","status":"failure","build_uid":132209,"build_number":132209,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"209352e0595c86e93759b5c11b6284c2be13edcd","client_build_number":20301000132209},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T19:13:23Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid204","env":"Staging","status":"failure","build_uid":132502,"build_number":132502,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dfbdf1b9b97e77dd9cc321556adb4c55f0cd27a1","client_build_number":20301000132502},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T15:20:12Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid205","env":"Production","status":"failure","build_uid":132450,"build_number":132450,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"33693814967195dc940e987265797f5980c81e70","client_build_number":20301000132450},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-07T00:13:27Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid206","env":"Development","status":"success","build_uid":132354,"build_number":132354,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"dafcc8a156fc30fed062f5ddd859ab34c712a827","client_build_number":20301000132354},"emitted_at":1737154317860}} | ||
| {"type":"RECORD","record":{"stream":"bigquery__bigquery__deployments","data":{"started_at":"2023-11-09T21:14:34Z","url":"https://buildkite.com/foobar/foobar-android-rn-js/builds/107808","ended_at":"2023-08-03T21:01:13Z","uid":"uid207","env":"Staging","status":"success","build_uid":132962,"build_number":132962,"pipeline_name":"foobar-android-rn-js","pipeline_uid":"foobar-android-rn-js","commit_sha":"9f4a555ab16b8c8f77229a470dddcb7145b0e2df","client_build_number":20301000132962},"emitted_at":1737154317860}} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
are we documenting anywhere what is the structure of the expected record?