Save github PR list in csv (elastic#123276)

* save PR list for release testing in csv

* use paginate to simplify code

* fix

* Update src/dev/github/get_prs_cli.ts

Co-authored-by: Spencer <[email protected]>

* review fix

* update example with OR logic

* fix optional flags check

Co-authored-by: Spencer <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

scripts/download_pr_list.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/github/download_pr_list_cli').downloadPullRequests();

src/dev/github/download_pr_list_cli.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { run, createFlagError, Flags } from '@kbn/dev-utils';
+import fs from 'fs';
+import Path from 'path';
+import { savePrsToCsv } from './search_and_save_pr_list';
+
+function getLabelsPath(flags: Flags) {
+  if (typeof flags.path !== 'string') {
+    throw createFlagError('please provide a single --path flag');
+  }
+
+  if (!fs.existsSync(Path.resolve(flags.path))) {
+    throw createFlagError('please provide an existing json file with --path flag');
+  }
+
+  return Path.resolve(flags.path);
+}
+
+export async function downloadPullRequests() {
+  run(
+    async ({ log, flags }) => {
+      const githubToken = process.env.GITHUB_TOKEN;
+
+      if (!githubToken) {
+        throw new Error('GITHUB_TOKEN was not provided.');
+      }
+
+      const labelsPath = getLabelsPath(flags);
+
+      if (typeof flags.dest !== 'string') {
+        throw createFlagError('please provide csv path in --dest flag');
+      }
+
+      const query = flags.query || undefined;
+      if (query !== undefined && typeof query !== 'string') {
+        throw createFlagError('please provide valid string in --query flag');
+      }
+
+      const mergedSince = flags['merged-since'] || undefined;
+      if (
+        mergedSince !== undefined &&
+        (typeof mergedSince !== 'string' || !/\d{4}-\d{2}-\d{2}/.test(mergedSince))
+      ) {
+        throw createFlagError(
+          `please provide a past date in 'yyyy-mm-dd' format in --merged-since flag`
+        );
+      }
+
+      fs.mkdirSync(flags.dest, { recursive: true });
+      const filename = Path.resolve(
+        flags.dest,
+        `kibana_prs_${new Date().toISOString().split('T').join('-')}.csv`
+      );
+      await savePrsToCsv(log, githubToken, labelsPath, filename, query, mergedSince);
+    },
+    {
+      description: `
+        Create a csv file with PRs to be tests for upcoming release,
+        require GITHUB_TOKEN variable to be set in advance
+    `,
+      flags: {
+        string: ['path', 'dest', 'query', 'merged-since'],
+        help: `
+        --path             Required, path to json file with labels to operate on, see src/dev/example.json
+        --dest             Required, generated csv file location
+        --query            Optional, overrides default query
+        --merged-since     Optional, start date in 'yyyy-mm-dd' format
+      `,
+      },
+    }
+  );
+}

src/dev/github/example.json

@@ -0,0 +1,12 @@
+{
+    "include": [
+        "v8.0.0",
+        "\"Team:ResponseOps\",\"Feature:Canvas\""
+    ],
+    "exclude": [
+        "failed-test",
+        "Feature:Unit Testing",
+        "Feature:Functional Testing",
+        "release_note:skip"
+    ]
+}

src/dev/github/search_and_save_pr_list.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ToolingLog } from '@kbn/dev-utils';
+import { Octokit } from '@octokit/rest';
+import fs from 'fs';
+
+interface Labels {
+  include: string[];
+  exclude: string[];
+}
+
+interface PR {
+  title: string;
+  url: string;
+  releaseLabel: string;
+}
+
+export async function savePrsToCsv(
+  log: ToolingLog,
+  githubToken: string,
+  labelsPath: string,
+  filename: string,
+  query: string | undefined,
+  mergedSince: string | undefined
+) {
+  const repo = `repo:"elastic/kibana"`;
+  const defaultQuery = 'is:pull-request+is:merged+sort:updated-desc';
+  const perPage = 100;
+  const searchApiLimit = 1000;
+
+  let q = repo + '+' + (query ?? defaultQuery) + (mergedSince ? `+merged:>=${mergedSince}` : '');
+
+  const rawData = fs.readFileSync(labelsPath, 'utf8');
+  const labels = JSON.parse(rawData) as Labels;
+
+  labels.include.map((label) => (q += `+label:${label}`));
+  labels.exclude.map((label) => (q += ` -label:"${label}"`));
+
+  log.debug(`Github query: ${q}`);
+
+  const octokit = new Octokit({
+    auth: githubToken,
+  });
+  const items: PR[] = await octokit.paginate(
+    'GET /search/issues',
+    { q, per_page: perPage },
+    (response) =>
+      response.data.map((item: Octokit.SearchIssuesAndPullRequestsResponseItemsItem) => {
+        return {
+          title: item.title,
+          url: item.html_url,
+          releaseLabel: item.labels
+            .filter((label) => label.name.trim().startsWith('release_note'))
+            .map((label) => label.name)
+            .join(','),
+        } as PR;
+      })
+  );
+
+  // https://docs.github.com/en/rest/reference/search
+  if (items.length >= searchApiLimit) {
+    log.warning(
+      `Search API limit is 1000 results per search, try to adjust the query. Saving first 1000 PRs`
+    );
+  } else {
+    log.info(`Found ${items.length} PRs`);
+  }
+
+  let csv = '';
+  for (const i of items) {
+    csv += `${i.title}\t${i.url}\t${i.releaseLabel}\r\n`;
+  }
+
+  fs.writeFileSync(filename, csv);
+  log.info(`Saved to ${filename}`);
+}