Nightlies page use clickhouse, deprecate rockset (#6090)

Rewrite Nightlies page using clickhouse

torchci/clickhouse_queries/nightly_jobs_red/params.json

@@ -2,6 +2,5 @@
   "granularity": "String",
   "repo": "String",
   "startTime": "DateTime64(3)",
-  "stopTime": "DateTime64(3)",
-  "timezone": "String"
-}
+  "stopTime": "DateTime64(3)"
+}

torchci/clickhouse_queries/nightly_jobs_red/query.sql

@@ -1,37 +1,37 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
 SELECT
-    FORMAT_ISO8601(
-        DATE_TRUNC(:granularity, time)
+    DATE_TRUNC(
+        {granularity: String },
+        time
     ) AS granularity_bucket,
     AVG(
         CASE
             when conclusion = 'failure' THEN 1
             when conclusion = 'timed_out' THEN 1
             when conclusion = 'cancelled' THEN 1
-             when conclusion = 'skipped' THEN 1
+            when conclusion = 'skipped' THEN 1
             ELSE 0
         END
-    ) as red,
+    ) as red
 FROM
-    (
-        SELECT
-            job._event_time AT TIME ZONE :timezone as time,
-            job.conclusion as conclusion,
-        FROM
-            commons.workflow_job job
-            JOIN commons.workflow_run workflow on workflow.id = job.run_id
-            JOIN push on workflow.head_commit.id = push.head_commit.id
-        WHERE
-            job.name NOT LIKE '%generate-matrix%'
-            AND job.name NOT LIKE '%unittests%'
-            AND workflow.name NOT IN ('cron', 'Bandit', 'tests')
-            AND push.ref = 'refs/heads/nightly'
-            AND push.repository.owner.name = 'pytorch'
-            AND push.repository.name = :repo
-            AND job._event_time >= PARSE_DATETIME_ISO8601(:startTime)
-            AND job._event_time < PARSE_DATETIME_ISO8601(:stopTime)
-    ) as all_job
+(
+SELECT
+    job.created_at AS time,
+    job.conclusion AS conclusion
+FROM
+    workflow_job job
+    JOIN workflow_run workflow ON workflow.id = job.run_id
+    JOIN push ON workflow.head_commit.'id' = push.head_commit.'id'
+WHERE
+    job.name NOT LIKE '%generate-matrix%'
+    AND job.name NOT LIKE '%unittests%'
+    AND workflow.name NOT IN ('cron', 'Bandit', 'tests')
+    AND push.ref = 'refs/heads/nightly'
+    AND push.repository.'owner'.'name' = 'pytorch'
+    AND push.repository.'name' = {repo: String }
+    AND job.created_at >= {startTime: DateTime64(3)}
+    AND job.created_at < {stopTime: DateTime64(3)}
+)
 GROUP BY
-    DATE_TRUNC(:granularity, time)
+    granularity_bucket
 ORDER BY
-    DATE_TRUNC(:granularity, time) ASC
+    granularity_bucket ASC

torchci/clickhouse_queries/nightly_jobs_red_by_name/query.sql

@@ -1,20 +1,19 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
-SELECT
-            COUNT(*) COUNT, workflow.name
+  SELECT
+            COUNT(*) AS COUNT,
+            workflow.name as name
         FROM
-            commons.workflow_job job
-            JOIN commons.workflow_run workflow on workflow.id = job.run_id
-            JOIN push on workflow.head_commit.id = push.head_commit.id
+            workflow_job job
+            JOIN workflow_run workflow on workflow.id = job.run_id
+            JOIN push ON workflow.head_commit.'id' = push.head_commit.'id'
         WHERE
             job.name NOT LIKE '%generate-matrix%'
             AND job.name NOT LIKE '%unittests%'
             AND workflow.name NOT IN ('cron', 'Bandit', 'tests', 'Lint')
             AND push.ref = 'refs/heads/nightly'
-            AND push.repository.owner.name = 'pytorch'
-            AND push.repository.name in ('pytorch', 'vision', 'audio', 'text')
-            AND job._event_time >= PARSE_DATETIME_ISO8601(:startTime)
-            AND job._event_time < PARSE_DATETIME_ISO8601(:stopTime)
-            AND job.conclusion in ('failure', 'timed_out', 'cancelled')
+            AND push.repository.'owner'.'name' = 'pytorch'
+            AND push.repository.'name' IN ('pytorch', 'vision', 'audio')
+            AND job.created_at >= {startTime: DateTime64(3)}
+            AND job.created_at < {stopTime: DateTime64(3)}
+            AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
   GROUP BY
   workflow.name
-  ORDER BY COUNT DESC

torchci/clickhouse_queries/nightly_jobs_red_by_platform/params.json

@@ -1,4 +1,5 @@
 {
   "startTime": "DateTime64(3)",
-  "stopTime": "DateTime64(3)"
-}
+  "stopTime": "DateTime64(3)",
+  "repo": "String"
+}

torchci/clickhouse_queries/nightly_jobs_red_by_platform/query.sql

@@ -1,38 +1,30 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
 WITH all_failed_jobs AS (
-  SELECT
-            COUNT(*) COUNT, workflow.path
+    SELECT
+            COUNT(*) AS COUNT, workflow.path
         FROM
-            commons.workflow_job job
-            JOIN commons.workflow_run workflow on workflow.id = job.run_id
-            JOIN push on workflow.head_commit.id = push.head_commit.id
+            workflow_job job
+            JOIN workflow_run workflow on workflow.id = job.run_id
+            JOIN push ON workflow.head_commit.'id' = push.head_commit.'id'
         WHERE
             job.name NOT LIKE '%generate-matrix%'
             AND job.name NOT LIKE '%unittests%'
             AND workflow.name NOT IN ('cron', 'Bandit', 'tests', 'Lint')
             AND push.ref = 'refs/heads/nightly'
-            AND push.repository.owner.name = 'pytorch'
-            AND push.repository.name in ('pytorch', 'vision', 'audio', 'text')
-            AND job._event_time >= PARSE_DATETIME_ISO8601(:startTime)
-            AND job._event_time < PARSE_DATETIME_ISO8601(:stopTime)
-            AND job.conclusion in ('failure', 'timed_out', 'cancelled')
+            AND push.repository.'owner'.'name' = 'pytorch'
+            AND push.repository.'name' = {repo: String }
+            AND job.created_at >= {startTime: DateTime64(3)}
+            AND job.created_at < {stopTime: DateTime64(3)}
+            AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
   GROUP BY
   workflow.path )
-SELECT
-    SUM(COUNT) as Count, 'Conda' as Platform
-FROM
-    all_failed_jobs
-where path like '%conda%'
-UNION
 SELECT
     SUM(COUNT) as Count, 'Wheel' as Platform
 FROM
     all_failed_jobs
-where path like '%wheel%'
-UNION 
+where workflow.path like '%wheel%'
+UNION ALL
 SELECT
     SUM(COUNT) as Count, 'Libtorch' as Platform
 FROM
     all_failed_jobs
-where path like '%libtorch%'
-
+where workflow.path like '%libtorch%'

torchci/clickhouse_queries/nightly_jobs_red_past_day/query.sql

@@ -1,20 +1,18 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
 SELECT
-  COUNT(*) COUNT,
-  job.name
+  COUNT(*) AS COUNT,
+  job.name as name
 FROM
-  commons.workflow_job job
-  JOIN commons.workflow_run workflow on workflow.id = job.run_id
-  JOIN push on workflow.head_commit.id = push.head_commit.id
+  workflow_job job
+  JOIN workflow_run workflow ON workflow.id = job.run_id
+  join push on push.head_commit.'id' = workflow.head_commit.'id'
 WHERE
   job.name NOT LIKE '%generate-matrix%'
   AND job.name NOT LIKE '%unittests%'
   AND workflow.name NOT IN ('cron', 'Bandit', 'tests')
   AND push.ref = 'refs/heads/nightly'
-  AND push.repository.owner.name = 'pytorch'
-  AND push.repository.name = :repo
-  AND job.conclusion in ('failure', 'timed_out', 'cancelled') 
-  AND job._event_time >= CURRENT_DATE() - INTERVAL 1 DAY
+  AND push.repository.'owner'.'name' = 'pytorch'
+  AND push.repository.'name' = {repo: String }
+  AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
+  AND job.completed_at >= today() - 1
 GROUP BY job.name
-ORDER BY COUNT
-
+ORDER BY COUNT;

torchci/clickhouse_queries/validation_jobs_red_past_day/query.sql

@@ -1,17 +1,15 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
 SELECT
-  COUNT(*) COUNT,
-  job.name
+  COUNT(*) AS COUNT,
+  job.name as name
 FROM
-  commons.workflow_job job
-  JOIN commons.workflow_run workflow on workflow.id = job.run_id
+  workflow_job job
+  JOIN workflow_run workflow ON workflow.id = job.run_id
 WHERE
-  job.head_branch = 'main' 
-  AND workflow.name = 'cron' 
-  AND workflow.event = 'schedule' 
-  AND job.conclusion in ('failure', 'timed_out', 'cancelled') 
-  AND job.name like CONCAT('%',:channel,'%') 
-  AND workflow.repository.full_name = 'pytorch/builder' 
-  AND job._event_time >= CURRENT_DATE() - INTERVAL 1 DAY
+  job.head_branch = 'main'
+  AND workflow.name like '%Binaries Validations%'
+  AND workflow.event = 'schedule'
+  AND job.name like concat('%', {channel: String}, '%')
+  AND job.conclusion IN ('failure', 'timed_out', 'cancelled')
+  AND job.completed_at >= today() - 1
 GROUP BY job.name
-ORDER BY COUNT DESC
+ORDER BY COUNT DESC;