diff --git a/torchci/clickhouse_queries/failed_workflow_jobs/params.json b/torchci/clickhouse_queries/failed_workflow_jobs/params.json index a0b5563e33..cfaab3e9f5 100644 --- a/torchci/clickhouse_queries/failed_workflow_jobs/params.json +++ b/torchci/clickhouse_queries/failed_workflow_jobs/params.json @@ -1,7 +1,6 @@ { "branch": "String", - "count": "Int64", "repo": "String", "startTime": "DateTime64(3)", "stopTime": "DateTime64(3)" -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/failed_workflow_jobs/query.sql b/torchci/clickhouse_queries/failed_workflow_jobs/query.sql index 71cf6b7fbf..c88d76e09e 100644 --- a/torchci/clickhouse_queries/failed_workflow_jobs/query.sql +++ b/torchci/clickhouse_queries/failed_workflow_jobs/query.sql @@ -1,60 +1,26 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted -WITH repeats AS ( - SELECT - array_agg(j.id) AS ids - FROM - workflow_run w - JOIN workflow_job j ON w.id = j.run_id HINT(join_strategy = lookup) - WHERE - j._event_time >= PARSE_DATETIME_ISO8601(: startTime) - AND j._event_time < PARSE_DATETIME_ISO8601(: stopTime) - AND w.head_repository.full_name = : repo - AND w.head_branch = : branch - AND w.event != 'workflow_run' - AND w.event != 'repository_dispatch' - GROUP BY - j.head_sha, - j.name, - w.name - HAVING - count(*) > : count - AND bool_or( - j.conclusion IN ( - 'failure', 'cancelled', 'time_out' - ) - ) -), -ids AS ( - SELECT - ids.id - FROM - repeats, - UNNEST(repeats.ids AS id) AS ids -) -SELECT - job.head_sha AS sha, - CONCAT(w.name, ' / ', job.name) AS jobName, - job.id, - job.conclusion, - job.html_url AS htmlUrl, - CONCAT( - 'https://ossci-raw-job-status.s3.amazonaws.com/log/', - CAST(job.id AS string) - ) AS logUrl, - DATE_DIFF( - 'SECOND', - PARSE_TIMESTAMP_ISO8601(job.started_at), - PARSE_TIMESTAMP_ISO8601(job.completed_at) - ) AS durationS, - w.repository.full_name AS repo, - ARRAY_CREATE(job.torchci_classification.line) AS failureLines, - job.torchci_classification.captures AS failureCaptures, - ARRAY_CREATE(job.torchci_classification.line_num) AS failureLineNumbers, +-- This query is used to annotate job on HUD +SELECT DISTINCT + j.head_sha AS sha, + CONCAT(w.name, ' / ', j.name) AS jobName, + j.id, + j.conclusion, + j.html_url AS htmlUrl, + CONCAT( + 'https://ossci-raw-job-status.s3.amazonaws.com/log/', + j.id + ) AS logUrl, + DATE_DIFF('SECOND', j.started_at, j.completed_at) AS durationS, + array(j.torchci_classification. 'line') AS failureLines, + j.torchci_classification. 'captures' AS failureCaptures, + array(j.torchci_classification. 'line_num') AS failureLineNumbers FROM - ids - JOIN workflow_job job on job.id = ids.id - INNER JOIN workflow_run w on w.id = job.run_id + workflow_job j FINAL + JOIN workflow_run w FINAL on w.id = j.run_id WHERE - job.conclusion IN ( - 'failure', 'cancelled', 'time_out' - ) \ No newline at end of file + j.created_at >= {startTime: DateTime64(3) } + AND j.created_at < {stopTime: DateTime64(3) } + AND w.head_repository. 'full_name' = {repo: String } + AND w.head_branch = {branch: String } + AND w.event != 'workflow_run' + AND w.event != 'repository_dispatch' + AND j.conclusion IN ('failure', 'cancelled', 'time_out') diff --git a/torchci/components/JobAnnotationToggle.tsx b/torchci/components/JobAnnotationToggle.tsx index 40c4c50323..c5fd4ec403 100644 --- a/torchci/components/JobAnnotationToggle.tsx +++ b/torchci/components/JobAnnotationToggle.tsx @@ -1,4 +1,5 @@ import { ToggleButton, ToggleButtonGroup } from "@mui/material"; +import _ from "lodash"; import { useSession } from "next-auth/react"; import React from "react"; import { JobAnnotation, JobData } from "../lib/types"; @@ -15,7 +16,10 @@ export default function JobAnnotationToggle({ repo?: string | null; }) { const allJobs = similarJobs ?? []; - allJobs.push(job); + // Double check if the job exists before adding it + if (!_.find(allJobs, (j: JobData) => j.id === job.id)) { + allJobs.push(job); + } const [state, setState] = React.useState( (annotation ?? "null") as JobAnnotation diff --git a/torchci/pages/api/job_annotation/[repoOwner]/[repoName]/failures/[queryParams].ts b/torchci/pages/api/job_annotation/[repoOwner]/[repoName]/failures/[queryParams].ts index 36cb502a24..bc55561310 100644 --- a/torchci/pages/api/job_annotation/[repoOwner]/[repoName]/failures/[queryParams].ts +++ b/torchci/pages/api/job_annotation/[repoOwner]/[repoName]/failures/[queryParams].ts @@ -1,22 +1,12 @@ +import { queryClickhouseSaved } from "lib/clickhouse"; import { getDynamoClient } from "lib/dynamo"; -import getRocksetClient, { RocksetParam } from "lib/rockset"; import { JobData } from "lib/types"; import { NextApiRequest, NextApiResponse } from "next"; -import rocksetVersions from "rockset/prodVersions.json"; -async function fetchFailureJobs( - queryParams: RocksetParam[] -): Promise { - const rocksetClient = getRocksetClient(); - const failedJobs = await rocksetClient.queryLambdas.executeQueryLambda( - "commons", - "failed_workflow_jobs", - rocksetVersions.commons.failed_workflow_jobs, - { - parameters: queryParams, - } - ); - return failedJobs.results ?? []; +async function fetchFailureJobs(queryParams: { + [key: string]: any; +}): Promise { + return await queryClickhouseSaved("failed_workflow_jobs", queryParams); } export default async function handler( diff --git a/torchci/pages/failedjobs/[repoOwner]/[repoName]/[branch]/[[...page]].tsx b/torchci/pages/failedjobs/[repoOwner]/[repoName]/[branch]/[[...page]].tsx index 9f6558e637..a08b95bc5c 100644 --- a/torchci/pages/failedjobs/[repoOwner]/[repoName]/[branch]/[[...page]].tsx +++ b/torchci/pages/failedjobs/[repoOwner]/[repoName]/[branch]/[[...page]].tsx @@ -6,7 +6,6 @@ import LogViewer from "components/LogViewer"; import dayjs from "dayjs"; import { fetcher } from "lib/GeneralUtils"; import { isRerunDisabledTestsJob, isUnstableJob } from "lib/jobUtils"; -import { RocksetParam } from "lib/rockset"; import { JobAnnotation, JobData } from "lib/types"; import _ from "lodash"; import { useRouter } from "next/router"; @@ -38,14 +37,17 @@ function SimilarFailedJobs({ {showDetail ? "▼ " : "▶ "} Failing {similarJobs.length} times - {showDetail && - _.map(similarJobs, (job) => ( - - ))} +
    + {showDetail && + _.map(similarJobs, (job) => ( + + ))} +
); } @@ -125,7 +127,7 @@ function FailedJobs({ repoName, repoOwner, }: { - queryParams: RocksetParam[]; + queryParams: { [key: string]: any }; repoName: string; repoOwner: string; }) { @@ -133,7 +135,7 @@ function FailedJobs({ // their annotation is not a scalable solution because the list of failures // could be longer than the browser-dependent URL-length limit. The workaround // here is to send the query param over to another annotation API that will then - // make a query to Rockset to get the list of failed jobs itself and return the + // make a query to the db to get the list of failed jobs itself and return the // list to the caller here const { data: failedJobsWithAnnotations } = useSWR( `/api/job_annotation/${repoOwner}/${repoName}/failures/${encodeURIComponent( @@ -233,33 +235,12 @@ export default function Page() { const [stopTime, setStopTime] = useState(dayjs()); const [timeRange, setTimeRange] = useState(7); - const queryParams: RocksetParam[] = [ - { - name: "startTime", - type: "string", - value: startTime, - }, - { - name: "stopTime", - type: "string", - value: stopTime, - }, - { - name: "repo", - type: "string", - value: `${repoOwner}/${repoName}`, - }, - { - name: "branch", - type: "string", - value: `${branch}`, - }, - { - name: "count", - type: "int", - value: "0", // Set the count to 0 to query all failures - }, - ]; + const queryParams: { [key: string]: any } = { + branch: branch, + repo: `${repoOwner}/${repoName}`, + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + }; return (