add support for azure server urls

Signed-off-by: Chalenge Masekera <[email protected]>

sources/azure-workitems-source/resources/spec.json

@@ -7,20 +7,33 @@
     "required": ["access_token", "organization"],
     "additionalProperties": true,
     "properties": {
+      "api_url": {
+        "order": 0,
+        "type": "string",
+        "title": "API URL",
+        "default": "https://dev.azure.com"
+      },
+      "graph_api_url": {
+        "order": 1,
+        "type": "string",
+        "title": "Graph API URL",
+        "default": "https://vssps.dev.azure.com"
+      },
       "access_token": {
+        "order": 2,
         "type": "string",
         "title": "Azure Access Token",
         "description": "Your unaltered Azure Access Token",
         "airbyte_secret": true
       },
       "organization": {
-        "order": 0,
+        "order": 3,
         "type": "string",
         "title": "Organization",
         "description": "Azure Organization"
       },
       "projects": {
-        "order": 1,
+        "order": 4,
         "type": "array",
         "items": {
           "type": "string",
@@ -30,12 +43,14 @@
         "description": "Azure Projects. If empty or '*' all available projects will be processed."
       },
       "project": {
+        "order": 5,
         "type": "string",
         "title": "Azure Project",
         "deprecated": true,
         "description": "Deprecated: Please use 'projects' array instead"
       },
       "additional_fields": {
+        "order": 6,
         "type": "array",
         "title": "Additional Fields",
         "items": {
@@ -49,26 +64,37 @@
         ]
       },
       "cutoff_days": {
+        "order": 7,
         "type": "integer",
         "title": "Cutoff Days",
         "description": "The threshold after which data should be synced.",
         "default": 90
       },
+      "api_version": {
+        "order": 8,
+        "type": "string",
+        "title": "API Version",
+        "default": "7.1"
+      },
       "graph_version": {
+        "order": 9,
         "type": "string",
         "title": "Graph API Version",
         "default": "7.1-preview.1"
       },
+      "reject_unauthorized": {
+        "order": 10,
+        "type": "boolean",
+        "title": "Reject Unauthorized",
+        "description": "If true, requests to Azure DevOps API with self-signed certificates will be rejected.",
+        "default": true
+      },
       "request_timeout": {
+        "order": 11,
         "type": "integer",
         "title": "Request Timeout",
         "description": "The max time in milliseconds to wait for a request to complete (0 - no timeout).",
         "default": 60000
-      },
-      "api_version": {
-        "type": "string",
-        "title": "API Version",
-        "default": "7.1"
       }
     }
   }

sources/azure-workitems-source/src/azure-workitems.ts

@@ -14,7 +14,10 @@ import {
   WorkItemResponse,
   WorkItemUpdatesResponse,
 } from './models';
+
+const DEFAULT_API_URL = 'https://dev.azure.com';
 const DEFAULT_API_VERSION = '7.1';
+const DEFAULT_GRAPH_API_URL = 'https://vssps.dev.azure.com';
 const DEFAULT_GRAPH_VERSION = '7.1-preview.1';
 const MAX_BATCH_SIZE = 200;
 export const DEFAULT_REQUEST_TIMEOUT = 60000;
@@ -37,14 +40,16 @@ const WORK_ITEM_TYPES = [
 
 export interface AzureWorkitemsConfig {
   readonly access_token: string;
+  readonly api_url: string;
+  readonly graph_api_url: string;
   readonly organization: string;
   readonly project: string;
   readonly projects: string[];
   readonly additional_fields: string[];
   readonly cutoff_days?: number;
   readonly api_version?: string;
-  readonly request_timeout?: number;
   readonly graph_version?: string;
+  readonly request_timeout?: number;
 }
 
 export class AzureWorkitems {
@@ -63,22 +68,15 @@ export class AzureWorkitems {
   ): Promise<AzureWorkitems> {
     if (AzureWorkitems.azure_Workitems) return AzureWorkitems.azure_Workitems;
 
-    if (!config.access_token) {
-      throw new VError('access_token must not be an empty string');
-    }
-
-    if (!config.organization) {
-      throw new VError('organization must not be an empty string');
-    }
-
-    const accessToken = base64Encode(`:${config.access_token}`);
+    AzureWorkitems.validateConfig(config);
 
+    const apiUrl = config.api_url ?? DEFAULT_API_URL;
     const version = config.api_version ?? DEFAULT_API_VERSION;
+    const accessToken = base64Encode(`:${config.access_token}`);
 
     const httpClient = makeAxiosInstanceWithRetry(
       {
-        // baseURL: `https://dev.azure.com/${config.organization}/${config.project}/_apis`,
-        baseURL: `https://dev.azure.com/${config.organization}`,
+        baseURL: `${apiUrl}/${config.organization}`,
         timeout: config.request_timeout ?? DEFAULT_REQUEST_TIMEOUT,
         maxContentLength: Infinity, //default is 2000 bytes
         params: {
@@ -93,8 +91,10 @@ export class AzureWorkitems {
       1000
     );
 
+    const graphApiUrl = config.graph_api_url ?? DEFAULT_GRAPH_API_URL;
+
     const graphClient = axios.create({
-      baseURL: `https://vssps.dev.azure.com/${config.organization}/_apis/graph`,
+      baseURL: `${graphApiUrl}/${config.organization}/_apis/graph`,
       timeout: config.request_timeout ?? DEFAULT_REQUEST_TIMEOUT,
       maxContentLength: Infinity,
       maxBodyLength: Infinity,
@@ -136,7 +136,30 @@ export class AzureWorkitems {
     return AzureWorkitems.azure_Workitems;
   }
 
+  static validateConfig(config: AzureWorkitemsConfig) {
+    if (!config.access_token) {
+      throw new VError('access_token must not be an empty string');
+    }
+
+    if (!config.organization) {
+      throw new VError('organization must not be an empty string');
+    }
+
+    // If using a custom API URL for Server, a custom Graph API URL must also be provided
+    const apiUrl =
+      config.api_url?.replace(/\/+$/, '').trim() ?? DEFAULT_API_URL;
+    const graphApiUrl =
+      config.graph_api_url?.replace(/\/+$/, '').trim() ?? DEFAULT_GRAPH_API_URL;
+
+    if (apiUrl !== DEFAULT_API_URL && graphApiUrl === DEFAULT_GRAPH_API_URL) {
+      throw new VError(
+        'When using a custom API URL, a custom Graph API URL must also be provided'
+      );
+    }
+  }
+
   async checkConnection(): Promise<void> {
+    let iter2: AsyncGenerator<User>;
     try {
       const iter = this.getUsers();
       await iter.next();

sources/azure-workitems-source/src/index.ts

@@ -35,11 +35,11 @@ export class AzureWorkitemsSource extends AirbyteSourceBase<AzureWorkitemsConfig
     config: AzureWorkitemsConfig
   ): Promise<[boolean, VError]> {
     try {
-      const azureActiveDirectory = await AzureWorkitems.instance(
+      const azureWorkItems = await AzureWorkitems.instance(
         config,
         this.logger
       );
-      await azureActiveDirectory.checkConnection();
+      await azureWorkItems.checkConnection();
     } catch (err: any) {
       return [false, err];
     }

sources/azure-workitems-source/test/__snapshots__/index.test.ts.snap

@@ -1,5 +1,44 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`index check connection - custom api_url with no graph_api_url 1`] = `
+AirbyteConnectionStatusMessage {
+  "connectionStatus": {
+    "message": "When using a custom API URL, a custom Graph API URL must also be provided",
+    "status": "FAILED",
+  },
+  "type": "CONNECTION_STATUS",
+}
+`;
+
+exports[`index check connection - no access token 1`] = `
+AirbyteConnectionStatusMessage {
+  "connectionStatus": {
+    "message": "access_token must not be an empty string",
+    "status": "FAILED",
+  },
+  "type": "CONNECTION_STATUS",
+}
+`;
+
+exports[`index check connection - no organization 1`] = `
+AirbyteConnectionStatusMessage {
+  "connectionStatus": {
+    "message": "organization must not be an empty string",
+    "status": "FAILED",
+  },
+  "type": "CONNECTION_STATUS",
+}
+`;
+
+exports[`index check connection - valid config 1`] = `
+AirbyteConnectionStatusMessage {
+  "connectionStatus": {
+    "status": "SUCCEEDED",
+  },
+  "type": "CONNECTION_STATUS",
+}
+`;
+
 exports[`index streams - iterations, use full_refresh sync mode 1`] = `
 [
   {

sources/azure-workitems-source/test/index.test.ts

@@ -2,10 +2,10 @@ import {
   AirbyteLogLevel,
   AirbyteSourceLogger,
   AirbyteSpec,
+  sourceCheckTest,
   SyncMode,
 } from 'faros-airbyte-cdk';
 import fs from 'fs-extra';
-import {VError} from 'verror';
 
 import {AzureWorkitems} from '../src/azure-workitems';
 import * as sut from '../src/index';
@@ -22,6 +22,8 @@ describe('index', () => {
       : AirbyteLogLevel.INFO
   );
 
+  const source = new sut.AzureWorkitemsSource(logger);
+
   beforeEach(() => {
     AzureWorkitems.instance = azureWorkitem;
   });
@@ -41,18 +43,63 @@ describe('index', () => {
     );
   });
 
+  test('check connection - valid config', async () => {
+    AzureWorkitems.instance = jest.fn().mockImplementation(() => {
+      const usersResource: any[] = readTestResourceFile('users.json');
+      return new AzureWorkitems(
+        null,
+        {
+          get: jest.fn().mockResolvedValue({
+            data: {value: usersResource},
+          }),
+        } as any,
+        new Map(),
+        logger
+      );
+    });
+
+    await sourceCheckTest({
+      source,
+      configOrPath: {
+        access_token: 'access_token',
+        organization: 'organization',
+        project: 'project',
+      },
+    });
+  });
+
   test('check connection - no access token', async () => {
-    const source = new sut.AzureWorkitemsSource(logger);
-    await expect(
-      source.checkConnection({
+    await sourceCheckTest({
+      source,
+      configOrPath: {
         access_token: '',
         organization: 'organization',
         project: 'project',
-      } as any)
-    ).resolves.toStrictEqual([
-      false,
-      new VError('access_token must not be an empty string'),
-    ]);
+      },
+    });
+  });
+
+  test('check connection - no organization', async () => {
+    await sourceCheckTest({
+      source,
+      configOrPath: {
+        access_token: 'access_token',
+        organization: '',
+        project: 'project',
+      },
+    });
+  });
+
+  test('check connection - custom api_url with no graph_api_url', async () => {
+    await sourceCheckTest({
+      source,
+      configOrPath: {
+        access_token: 'access_token',
+        organization: 'organization',
+        api_url: 'https://azure.myorg.com',
+        project: 'project',
+      },
+    });
   });
 
   test('streams - users, use full_refresh sync mode', async () => {