[Utilization] Add url link to utilization report page

torchci/clickhouse_queries/oss_ci_util_metadata_info/params.json

@@ -0,0 +1,7 @@
+{
+  "params": {
+    "workflowId": "UInt64",
+    "repo": "String"
+  },
+  "tests":[]
+}

torchci/clickhouse_queries/oss_ci_util_metadata_info/query.sql

@@ -0,0 +1,12 @@
+SELECT
+    workflow_id,
+    job_id,
+    workflow_name,
+    job_name,
+    run_attempt,
+    repo,
+FROM
+    misc.oss_ci_utilization_metadata
+WHERE
+    workflow_id = { workflowId: UInt64}
+    AND repo = {repo: String }

torchci/components/WorkflowBox.tsx

@@ -1,9 +1,15 @@
+import { Button, styled } from "@mui/material";
 import styles from "components/commit.module.css";
-import { fetcher } from "lib/GeneralUtils";
+import { fetcher, fetcherHandleError } from "lib/GeneralUtils";
 import { isFailedJob } from "lib/jobUtils";
 import { getSearchRes, LogSearchResult } from "lib/searchLogs";
 import { Artifact, IssueData, JobData } from "lib/types";
+import {
+  ListUtilizationMetadataInfoAPIResponse,
+  UtilizationMetadataInfo,
+} from "lib/utilization/types";
 import React, { useEffect, useState } from "react";
+import { IoMdArrowDropdown } from "react-icons/io";
 import useSWR from "swr";
 import { getConclusionSeverityForSorting } from "../lib/JobClassifierUtil";
 import { TestInfo } from "./additionalTestInfo/TestInfo";
@@ -24,15 +30,24 @@ function sortJobsByConclusion(jobA: JobData, jobB: JobData): number {
   // Jobs with the same conclusion are sorted alphabetically
   return ("" + jobA.jobName).localeCompare("" + jobB.jobName); // the '' forces the type to be a string
 }
-
+const CustomIoMdArrowDropdownShow = styled(IoMdArrowDropdown)({
+  fontSize: "20px",
+});
+const JobButton = styled(Button)({
+  fontSize: "10px",
+  color: "green",
+  margin: "5px",
+});
 function WorkflowJobSummary({
   job,
+  utilMetadata,
   artifacts,
   artifactsToShow,
   setArtifactsToShow,
   unstableIssues,
 }: {
   job: JobData;
+  utilMetadata?: UtilizationMetadataInfo[];
   artifacts?: Artifact[];
   artifactsToShow: Set<string>;
   setArtifactsToShow: any;
@@ -73,15 +88,34 @@ function WorkflowJobSummary({
 
   if (hasArtifacts) {
     subInfo.push(
-      <a onClick={() => setArtifactsToShowHelper()}>Show artifacts</a>
+      <JobButton variant="outlined" onClick={() => setArtifactsToShowHelper()}>
+        <CustomIoMdArrowDropdownShow /> artifacts
+      </JobButton>
     );
   }
-
   if (job.logUrl) {
     subInfo.push(
-      <a target="_blank" rel="noreferrer" href={job.logUrl}>
+      <JobButton variant="outlined" href={job.logUrl}>
         Raw logs
-      </a>
+      </JobButton>
+    );
+  }
+  if (utilMetadata && utilMetadata.length > 0) {
+    if (utilMetadata.length > 1) {
+      console.log(
+        `Multiple util metadata found for job ${job.id}, currently only showing the first one`
+      );
+    }
+    const m = utilMetadata[0];
+    subInfo.push(
+      <>
+        <JobButton
+          variant="outlined"
+          href={`/utilization/${m.workflow_id}/${m.job_id}/${m.run_attempt}`}
+        >
+          Utilization Report{" "}
+        </JobButton>
+      </>
     );
   }
 
@@ -95,7 +129,7 @@ function WorkflowJobSummary({
           return (
             <span key={ind}>
               {info}
-              {ind < subInfo.length - 1 && ", "}
+              {ind < subInfo.length - 1}
             </span>
           );
         })}
@@ -124,18 +158,19 @@ export default function WorkflowBox({
   setWide: any;
   repoFullName: string;
 }) {
+  const workflowId = jobs[0].workflowId;
   const isFailed = jobs.some(isFailedJob) !== false;
   const workflowClass = isFailed
     ? styles.workflowBoxFail
     : styles.workflowBoxSuccess;
 
-  const workflowId = jobs[0].workflowId;
   const anchorName = encodeURIComponent(workflowName.toLowerCase());
+  const { utilMetadataList, metaError } = fetchMetadata(workflowId);
+  const groupUtilMetadataList = groupMetadataByJobId(utilMetadataList);
 
   const { artifacts, error } = useArtifacts(workflowId);
   const [artifactsToShow, setArtifactsToShow] = useState(new Set<string>());
   const groupedArtifacts = groupArtifacts(jobs, artifacts);
-
   const [searchString, setSearchString] = useState("");
   const [searchRes, setSearchRes] = useState<{
     results: Map<string, LogSearchResult>;
@@ -144,6 +179,7 @@ export default function WorkflowBox({
     results: new Map(),
     info: undefined,
   });
+
   useEffect(() => {
     getSearchRes(jobs, searchString, setSearchRes);
   }, [jobs, searchString]);
@@ -216,6 +252,11 @@ export default function WorkflowBox({
           <div key={job.id} id={`${job.id}-box`}>
             <WorkflowJobSummary
               job={job}
+              utilMetadata={
+                job.id
+                  ? groupUtilMetadataList.get(job.id.toString())
+                  : undefined
+              }
               artifacts={groupedArtifacts?.get(job.id?.toString())}
               artifactsToShow={artifactsToShow}
               setArtifactsToShow={setArtifactsToShow}
@@ -236,6 +277,46 @@ export default function WorkflowBox({
   );
 }
 
+function fetchMetadata(workflowId: string | undefined): {
+  utilMetadataList: UtilizationMetadataInfo[];
+  metaError: any;
+} {
+  if (!workflowId) {
+    return { utilMetadataList: [], metaError: "No workflow ID" };
+  }
+  const { data, error } = useSWR<ListUtilizationMetadataInfoAPIResponse>(
+    `/api/list_utilization_metadata_info/${workflowId}`,
+    fetcherHandleError,
+    {
+      refreshInterval: 60 * 1000, // refresh every minute
+      // Refresh even when the user isn't looking, so that switching to the tab
+      // will always have fresh info.
+      refreshWhenHidden: true,
+    }
+  );
+
+  if (error != null) {
+    console.log(`Error occured when list_utilization_metadata_info for ${workflowId}`,error, error.status);
+  }
+
+  if (data == null) {
+    return { utilMetadataList: [], metaError: "Loading..." };
+  }
+
+  if (data.metadata_list == null) {
+    return { utilMetadataList: [], metaError: "No metadata list found" };
+  }
+
+  if (error != null) {
+    return {
+      utilMetadataList: [],
+      metaError: "Error occured while fetching util metadata",
+    };
+  }
+
+  return { utilMetadataList: data.metadata_list, metaError: null };
+}
+
 function useArtifacts(workflowId: string | undefined): {
   artifacts: Artifact[];
   error: any;
@@ -260,6 +341,25 @@ function useArtifacts(workflowId: string | undefined): {
   return { artifacts: data, error };
 }
 
+function groupMetadataByJobId(
+  utilMetadataList: UtilizationMetadataInfo[]
+): Map<string, UtilizationMetadataInfo[]> {
+  const grouping = new Map<string, UtilizationMetadataInfo[]>();
+  for (const utilMetadata of utilMetadataList) {
+    if (!utilMetadata.job_id) {
+      continue;
+    }
+
+    const jobId = utilMetadata.job_id.toString();
+    if (grouping.has(jobId)) {
+      grouping.get(jobId)!.push(utilMetadata);
+    } else {
+      grouping.set(jobId, [utilMetadata]);
+    }
+  }
+  return grouping;
+}
+
 function groupArtifacts(jobs: JobData[], artifacts: Artifact[]) {
   // Group artifacts by job id if possible
   const jobIds = jobs.map((job) => job.id?.toString());

torchci/lib/utilization/fetchListUtilizationMetadataInfo.ts

@@ -0,0 +1,31 @@
+import { queryClickhouseSaved } from "lib/clickhouse";
+import {
+  ListUtilizationMetadataInfoAPIResponse,
+  ListUtilizationMetadataInfoParams,
+  UTILIZATION_DEFAULT_REPO,
+  UtilizationMetadataInfo,
+} from "./types";
+
+export default async function fetchListUtilizationMetadataInfo(
+  params: ListUtilizationMetadataInfoParams
+): Promise<ListUtilizationMetadataInfoAPIResponse | null> {
+  const meta_resp = await getUtilizationMetadataInfo(
+    params.workflow_id,
+    params.repo
+  );
+
+  return {
+    metadata_list: meta_resp ? meta_resp : ([] as UtilizationMetadataInfo[]),
+  };
+}
+
+async function getUtilizationMetadataInfo(
+  workflow_id: string,
+  repo: string = UTILIZATION_DEFAULT_REPO
+) {
+  const response = await queryClickhouseSaved("oss_ci_util_metadata_info", {
+    workflowId: workflow_id,
+    repo: repo,
+  });
+  return response;
+}

torchci/lib/utilization/fetchUtilization.ts

@@ -3,14 +3,14 @@ import {
   TimeSeriesDataPoint,
   TimeSeriesDbData,
   TimeSeriesWrapper,
+  UTIL_METADATA_QUERY_FOLDER_NAME,
+  UTILIZATION_DEFAULT_REPO,
   UtilizationAPIResponse,
   UtilizationMetadata,
   UtilizationParams,
 } from "./types";
 
-const DEFAULT_REPO = "pytorch/pytorch";
 const UTIL_TS_QUERY_FOLDER_NAME = "oss_ci_util_ts";
-const UTIL_METADATA_QUERY_FOLDER_NAME = "oss_ci_util_metadata";
 const UTILIZATION_TYPE = "utilization";
 
 export default async function fetchUtilization(
@@ -19,7 +19,8 @@ export default async function fetchUtilization(
   const meta_resp: UtilizationMetadata[] = await getUtilizationMetadata(
     params.workflow_id,
     params.job_id,
-    params.run_attempt
+    params.run_attempt,
+    params.repo
   );
 
   const metadata = getLatestMetadata(meta_resp);
@@ -40,7 +41,8 @@ export default async function fetchUtilization(
   const resp: TimeSeriesDbData[] = await getUtilTimesSeries(
     params.workflow_id,
     params.job_id,
-    params.run_attempt
+    params.run_attempt,
+    params.repo
   );
   const tsMap = flattenTS(resp);
 
@@ -61,29 +63,31 @@ export default async function fetchUtilization(
 async function getUtilTimesSeries(
   workflow_id: string,
   job_id: string,
-  run_attempt: string
+  run_attempt: string,
+  repo: string = UTILIZATION_DEFAULT_REPO
 ) {
   const response = await queryClickhouseSaved(UTIL_TS_QUERY_FOLDER_NAME, {
     workflowId: workflow_id,
     jobId: job_id,
     runAttempt: run_attempt,
     type: UTILIZATION_TYPE,
-    repo: DEFAULT_REPO,
+    repo: repo,
   });
   return response;
 }
 
 async function getUtilizationMetadata(
   workflow_id: string,
   job_id: string,
-  run_attempt: string
+  run_attempt: string,
+  repo: string = UTILIZATION_DEFAULT_REPO
 ) {
   const response = await queryClickhouseSaved(UTIL_METADATA_QUERY_FOLDER_NAME, {
     workflowId: workflow_id,
     jobId: job_id,
     runAttempt: run_attempt,
     type: UTILIZATION_TYPE,
-    repo: DEFAULT_REPO,
+    repo: repo,
   });
   return response;
 }

torchci/lib/utilization/types.ts

@@ -1,7 +1,12 @@
+export const UTILIZATION_DEFAULT_REPO = "pytorch/pytorch";
+
+export const UTIL_METADATA_QUERY_FOLDER_NAME = "oss_ci_util_metadata";
+
 export interface UtilizationParams {
   workflow_id: string;
   job_id: string;
   run_attempt: string;
+  repo?: string;
 }
 
 export interface TimeSeriesDbData {
@@ -48,3 +53,21 @@ export interface TimeSeriesDataPoint {
   ts: string;
   value: number;
 }
+
+export interface UtilizationMetadataInfo {
+  workflow_id: string;
+  job_id: string;
+  run_attempt: string;
+  workflow_name: string;
+  job_name: string;
+  repo: string;
+}
+
+export interface ListUtilizationMetadataInfoParams {
+  workflow_id: string;
+  repo?: string;
+}
+
+export interface ListUtilizationMetadataInfoAPIResponse {
+  metadata_list: UtilizationMetadataInfo[];
+}

torchci/pages/api/list_utilization_metadata_info/[workflowId].ts

@@ -0,0 +1,31 @@
+import { getErrorMessage } from "lib/error_utils";
+import fetchListUtilizationMetadataInfo from "lib/utilization/fetchListUtilizationMetadataInfo";
+import { ListUtilizationMetadataInfoParams } from "lib/utilization/types";
+import { NextApiRequest, NextApiResponse } from "next";
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse
+) {
+  const { workflowId } = req.query;
+  if (!workflowId) {
+    return res.status(400).json({ error: "workflowId is required" });
+  }
+
+  const params: ListUtilizationMetadataInfoParams = {
+    workflow_id: workflowId as string,
+  };
+
+  try {
+    const resp = await fetchListUtilizationMetadataInfo(params);
+    if (resp == null) {
+      return res
+        .status(404)
+        .json({ error: `No data found for params ${JSON.stringify(params)}` });
+    }
+    return res.status(200).json(resp);
+  } catch (error) {
+    const err_msg = getErrorMessage(error);
+    return res.status(500).json({ error: err_msg });
+  }
+}