diff --git a/sources/azure-workitems-source/resources/spec.json b/sources/azure-workitems-source/resources/spec.json index 523641aed..c9996c0b1 100644 --- a/sources/azure-workitems-source/resources/spec.json +++ b/sources/azure-workitems-source/resources/spec.json @@ -7,20 +7,33 @@ "required": ["access_token", "organization"], "additionalProperties": true, "properties": { + "api_url": { + "order": 0, + "type": "string", + "title": "API URL", + "default": "https://dev.azure.com" + }, + "graph_api_url": { + "order": 1, + "type": "string", + "title": "Graph API URL", + "default": "https://vssps.dev.azure.com" + }, "access_token": { + "order": 2, "type": "string", "title": "Azure Access Token", "description": "Your unaltered Azure Access Token", "airbyte_secret": true }, "organization": { - "order": 0, + "order": 3, "type": "string", "title": "Organization", "description": "Azure Organization" }, "projects": { - "order": 1, + "order": 4, "type": "array", "items": { "type": "string", @@ -30,12 +43,14 @@ "description": "Azure Projects. If empty or '*' all available projects will be processed." }, "project": { + "order": 5, "type": "string", "title": "Azure Project", "deprecated": true, "description": "Deprecated: Please use 'projects' array instead" }, "additional_fields": { + "order": 6, "type": "array", "title": "Additional Fields", "items": { @@ -49,26 +64,37 @@ ] }, "cutoff_days": { + "order": 7, "type": "integer", "title": "Cutoff Days", "description": "The threshold after which data should be synced.", "default": 90 }, + "api_version": { + "order": 8, + "type": "string", + "title": "API Version", + "default": "7.1" + }, "graph_version": { + "order": 9, "type": "string", "title": "Graph API Version", "default": "7.1-preview.1" }, + "reject_unauthorized": { + "order": 10, + "type": "boolean", + "title": "Reject Unauthorized", + "description": "If true, requests to Azure DevOps API with self-signed certificates will be rejected.", + "default": true + }, "request_timeout": { + "order": 11, "type": "integer", "title": "Request Timeout", "description": "The max time in milliseconds to wait for a request to complete (0 - no timeout).", "default": 60000 - }, - "api_version": { - "type": "string", - "title": "API Version", - "default": "7.1" } } } diff --git a/sources/azure-workitems-source/src/azure-workitems.ts b/sources/azure-workitems-source/src/azure-workitems.ts index 7bac51d2c..b978f5bc7 100644 --- a/sources/azure-workitems-source/src/azure-workitems.ts +++ b/sources/azure-workitems-source/src/azure-workitems.ts @@ -14,7 +14,10 @@ import { WorkItemResponse, WorkItemUpdatesResponse, } from './models'; + +const DEFAULT_API_URL = 'https://dev.azure.com'; const DEFAULT_API_VERSION = '7.1'; +const DEFAULT_GRAPH_API_URL = 'https://vssps.dev.azure.com'; const DEFAULT_GRAPH_VERSION = '7.1-preview.1'; const MAX_BATCH_SIZE = 200; export const DEFAULT_REQUEST_TIMEOUT = 60000; @@ -37,14 +40,16 @@ const WORK_ITEM_TYPES = [ export interface AzureWorkitemsConfig { readonly access_token: string; + readonly api_url: string; + readonly graph_api_url: string; readonly organization: string; readonly project: string; readonly projects: string[]; readonly additional_fields: string[]; readonly cutoff_days?: number; readonly api_version?: string; - readonly request_timeout?: number; readonly graph_version?: string; + readonly request_timeout?: number; } export class AzureWorkitems { @@ -63,22 +68,15 @@ export class AzureWorkitems { ): Promise { if (AzureWorkitems.azure_Workitems) return AzureWorkitems.azure_Workitems; - if (!config.access_token) { - throw new VError('access_token must not be an empty string'); - } - - if (!config.organization) { - throw new VError('organization must not be an empty string'); - } - - const accessToken = base64Encode(`:${config.access_token}`); + AzureWorkitems.validateConfig(config); + const apiUrl = AzureWorkitems.cleanUrl(config.api_url) ?? DEFAULT_API_URL; const version = config.api_version ?? DEFAULT_API_VERSION; + const accessToken = base64Encode(`:${config.access_token}`); const httpClient = makeAxiosInstanceWithRetry( { - // baseURL: `https://dev.azure.com/${config.organization}/${config.project}/_apis`, - baseURL: `https://dev.azure.com/${config.organization}`, + baseURL: `${apiUrl}/${config.organization}`, timeout: config.request_timeout ?? DEFAULT_REQUEST_TIMEOUT, maxContentLength: Infinity, //default is 2000 bytes params: { @@ -93,8 +91,11 @@ export class AzureWorkitems { 1000 ); + const graphApiUrl = + AzureWorkitems.cleanUrl(config.graph_api_url) ?? DEFAULT_GRAPH_API_URL; + const graphClient = axios.create({ - baseURL: `https://vssps.dev.azure.com/${config.organization}/_apis/graph`, + baseURL: `${graphApiUrl}/${config.organization}/_apis/graph`, timeout: config.request_timeout ?? DEFAULT_REQUEST_TIMEOUT, maxContentLength: Infinity, maxBodyLength: Infinity, @@ -136,6 +137,31 @@ export class AzureWorkitems { return AzureWorkitems.azure_Workitems; } + static validateConfig(config: AzureWorkitemsConfig) { + if (!config.access_token) { + throw new VError('access_token must not be an empty string'); + } + + if (!config.organization) { + throw new VError('organization must not be an empty string'); + } + + // If using a custom API URL for Server, a custom Graph API URL must also be provided + const apiUrl = AzureWorkitems.cleanUrl(config.api_url) ?? DEFAULT_API_URL; + const graphApiUrl = + AzureWorkitems.cleanUrl(config.graph_api_url) ?? DEFAULT_GRAPH_API_URL; + + if (apiUrl !== DEFAULT_API_URL && graphApiUrl === DEFAULT_GRAPH_API_URL) { + throw new VError( + 'When using a custom API URL, a custom Graph API URL must also be provided' + ); + } + } + + static cleanUrl(url?: string): string | undefined { + return url?.trim().endsWith('/') ? url.trim().slice(0, -1) : url?.trim(); + } + async checkConnection(): Promise { try { const iter = this.getUsers(); diff --git a/sources/azure-workitems-source/src/index.ts b/sources/azure-workitems-source/src/index.ts index b15787447..f4d62c296 100644 --- a/sources/azure-workitems-source/src/index.ts +++ b/sources/azure-workitems-source/src/index.ts @@ -35,11 +35,11 @@ export class AzureWorkitemsSource extends AirbyteSourceBase { try { - const azureActiveDirectory = await AzureWorkitems.instance( + const azureWorkItems = await AzureWorkitems.instance( config, this.logger ); - await azureActiveDirectory.checkConnection(); + await azureWorkItems.checkConnection(); } catch (err: any) { return [false, err]; } diff --git a/sources/azure-workitems-source/test/__snapshots__/index.test.ts.snap b/sources/azure-workitems-source/test/__snapshots__/index.test.ts.snap index 29576a265..d55140734 100644 --- a/sources/azure-workitems-source/test/__snapshots__/index.test.ts.snap +++ b/sources/azure-workitems-source/test/__snapshots__/index.test.ts.snap @@ -1,5 +1,44 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`index check connection - custom api_url with no graph_api_url 1`] = ` +AirbyteConnectionStatusMessage { + "connectionStatus": { + "message": "When using a custom API URL, a custom Graph API URL must also be provided", + "status": "FAILED", + }, + "type": "CONNECTION_STATUS", +} +`; + +exports[`index check connection - no access token 1`] = ` +AirbyteConnectionStatusMessage { + "connectionStatus": { + "message": "access_token must not be an empty string", + "status": "FAILED", + }, + "type": "CONNECTION_STATUS", +} +`; + +exports[`index check connection - no organization 1`] = ` +AirbyteConnectionStatusMessage { + "connectionStatus": { + "message": "organization must not be an empty string", + "status": "FAILED", + }, + "type": "CONNECTION_STATUS", +} +`; + +exports[`index check connection - valid config 1`] = ` +AirbyteConnectionStatusMessage { + "connectionStatus": { + "status": "SUCCEEDED", + }, + "type": "CONNECTION_STATUS", +} +`; + exports[`index streams - iterations, use full_refresh sync mode 1`] = ` [ { diff --git a/sources/azure-workitems-source/test/index.test.ts b/sources/azure-workitems-source/test/index.test.ts index 75d5033e6..a3e003bf0 100644 --- a/sources/azure-workitems-source/test/index.test.ts +++ b/sources/azure-workitems-source/test/index.test.ts @@ -2,10 +2,10 @@ import { AirbyteLogLevel, AirbyteSourceLogger, AirbyteSpec, + sourceCheckTest, SyncMode, } from 'faros-airbyte-cdk'; import fs from 'fs-extra'; -import {VError} from 'verror'; import {AzureWorkitems} from '../src/azure-workitems'; import * as sut from '../src/index'; @@ -22,6 +22,8 @@ describe('index', () => { : AirbyteLogLevel.INFO ); + const source = new sut.AzureWorkitemsSource(logger); + beforeEach(() => { AzureWorkitems.instance = azureWorkitem; }); @@ -41,18 +43,63 @@ describe('index', () => { ); }); + test('check connection - valid config', async () => { + AzureWorkitems.instance = jest.fn().mockImplementation(() => { + const usersResource: any[] = readTestResourceFile('users.json'); + return new AzureWorkitems( + null, + { + get: jest.fn().mockResolvedValue({ + data: {value: usersResource}, + }), + } as any, + new Map(), + logger + ); + }); + + await sourceCheckTest({ + source, + configOrPath: { + access_token: 'access_token', + organization: 'organization', + project: 'project', + }, + }); + }); + test('check connection - no access token', async () => { - const source = new sut.AzureWorkitemsSource(logger); - await expect( - source.checkConnection({ + await sourceCheckTest({ + source, + configOrPath: { access_token: '', organization: 'organization', project: 'project', - } as any) - ).resolves.toStrictEqual([ - false, - new VError('access_token must not be an empty string'), - ]); + }, + }); + }); + + test('check connection - no organization', async () => { + await sourceCheckTest({ + source, + configOrPath: { + access_token: 'access_token', + organization: '', + project: 'project', + }, + }); + }); + + test('check connection - custom api_url with no graph_api_url', async () => { + await sourceCheckTest({ + source, + configOrPath: { + access_token: 'access_token', + organization: 'organization', + api_url: 'https://azure.myorg.com', + project: 'project', + }, + }); }); test('streams - users, use full_refresh sync mode', async () => { diff --git a/sources/azurepipeline-source/resources/spec.json b/sources/azurepipeline-source/resources/spec.json index 5576fcce4..3e6fd1781 100644 --- a/sources/azurepipeline-source/resources/spec.json +++ b/sources/azurepipeline-source/resources/spec.json @@ -25,50 +25,61 @@ "title": "Project IDs or project names", "description": "Azure Projects. If empty or '*' all available projects will be processed." }, - "access_token": { + "api_url": { "order": 2, "type": "string", + "title": "API URL", + "default": "https://dev.azure.com" + }, + "vsrm_api_url": { + "order": 3, + "type": "string", + "title": "VSRM API URL", + "default": "https://vsrm.dev.azure.com" + }, + "api_version": { + "order": 4, + "type": "string", + "title": "API Version", + "default": "6.0" + }, + "access_token": { + "order": 5, + "type": "string", "title": "Azure Access Token", "description": "Your unaltered Azure Access Token", "airbyte_secret": true }, "cutoff_days": { - "order": 3, + "order": 6, "type": "integer", "title": "Cutoff Days", "default": 90, "description": "Fetch builds and releases created within the specified number of days" }, "page_size": { - "order": 4, + "order": 7, "type": "integer", "title": "Page Size", "description": "Page size to use when requesting records from Azure API", "default": 100 }, - "api_version": { - "order": 5, - "type": "string", - "title": "API Version", - "description": "Azure API Version", - "default": "6.0" - }, "api_timeout": { - "order": 6, + "order": 8, "type": "integer", "title": "API Timeout", "description": "Timeout (in milliseconds) to use when making requests to Azure API. 0 means no timeout.", "default": 0 }, "max_retries": { - "order": 7, + "order": 9, "type": "integer", "title": "Max Number of Retries", "description": "The max number of retries before giving up on retrying requests to the Azure API.", "default": 3 }, "api_retry_delay": { - "order": 8, + "order": 10, "type": "integer", "title": "API Retry Delay", "description": "Delay (in milliseconds) to use when retrying Azure API calls.", diff --git a/sources/azurepipeline-source/src/azurepipeline.ts b/sources/azurepipeline-source/src/azurepipeline.ts index 8e6881c8a..9c7728fed 100644 --- a/sources/azurepipeline-source/src/azurepipeline.ts +++ b/sources/azurepipeline-source/src/azurepipeline.ts @@ -14,7 +14,10 @@ import {makeAxiosInstanceWithRetry, Utils} from 'faros-js-client'; import {Dictionary} from 'ts-essentials'; import {VError} from 'verror'; +const DEFAULT_API_URL = 'https://dev.azure.com'; +const DEFAULT_VSRM_API_URL = 'https://vsrm.dev.azure.com'; const DEFAULT_API_VERSION = '6.0'; + const DEFAULT_CUTOFF_DAYS = 90; const DEFAULT_PAGE_SIZE = 100; const DEFAULT_API_TIMEOUT_MS = 0; // 0 means no timeout @@ -28,6 +31,8 @@ export interface AzurePipelineConfig { readonly access_token: string; readonly cutoff_days?: number; readonly page_size?: number; + readonly api_url?: string; + readonly vsrm_api_url?: string; readonly api_version?: string; readonly api_timeout?: number; readonly max_retries?: number; @@ -70,6 +75,7 @@ export class AzurePipeline { ): Promise { if (AzurePipeline.azurePipeline) return AzurePipeline.azurePipeline; + // TODO - Move to common utility for Azure clients if (!config.access_token) { throw new VError('Please provide an access token'); } @@ -89,15 +95,16 @@ export class AzurePipeline { const startDate = new Date(); startDate.setDate(startDate.getDate() - cutoff_days); - const version = config.api_version ?? DEFAULT_API_VERSION; + const apiUrl = config.api_url ?? DEFAULT_API_URL; + const apiVersion = config.api_version ?? DEFAULT_API_VERSION; const httpClient = makeAxiosInstanceWithRetry( { - baseURL: `https://dev.azure.com/${config.organization}`, + baseURL: `${apiUrl}/${config.organization}`, timeout: config.api_timeout ?? DEFAULT_API_TIMEOUT_MS, maxContentLength: Infinity, //default is 2000 bytes params: { - 'api-version': version, + 'api-version': apiVersion, }, headers: { Authorization: `Basic ${accessToken}`, @@ -108,13 +115,15 @@ export class AzurePipeline { config.api_retry_delay ?? DEFAULT_RETRY_DELAY_MS ); + + const vsrmApiUrl = config.vsrm_api_url ?? DEFAULT_VSRM_API_URL; const httpVSRMClient = makeAxiosInstanceWithRetry( { - baseURL: `https://vsrm.dev.azure.com/${config.organization}`, + baseURL: `${vsrmApiUrl}/${config.organization}`, timeout: config.api_timeout ?? DEFAULT_API_TIMEOUT_MS, maxContentLength: Infinity, //default is 2000 bytes params: { - 'api-version': version, + 'api-version': apiVersion, }, headers: { Authorization: `Basic ${accessToken}`,