Skip to content

Commit

Permalink
[ch] disablebot aka flaky test bot (#5749)
Browse files Browse the repository at this point in the history
Migrate queries for flaky test bot

Not bothering to gate

Sanity checked that outputs of the queries had same results over the
same 3 hour window, but only checked for 1 window
  • Loading branch information
clee2000 authored Oct 8, 2024
1 parent dd1a19e commit d420136
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 250 deletions.
115 changes: 36 additions & 79 deletions torchci/clickhouse_queries/flaky_tests/query.sql
Original file line number Diff line number Diff line change
@@ -1,93 +1,50 @@
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
SELECT
test_run.name,
test_run.classname as suite,
test_run.file,
test_run.invoking_file,
SUM(
ELEMENT_AT(
JSON_PARSE(
REPLACE(test_run.skipped.message, 'True', 'true')
),
'num_green'
)
) as numGreen,
SUM(
ELEMENT_AT(
JSON_PARSE(
REPLACE(test_run.skipped.message, 'True', 'true')
),
'num_red'
)
) as numRed,
ARRAY_AGG(job.name) as jobNames,
ARRAY_AGG(job.id) as jobIds,
ARRAY_AGG(workflow.id) as workflowIds,
ARRAY_AGG(workflow.name) as workflowNames,
ARRAY_AGG(workflow.head_branch) as branches,
ARRAY_AGG(test_run.workflow_run_attempt) as runAttempts,
ARBITRARY(
if(
TYPEOF(test_run.rerun) = 'object',
test_run.rerun.text,
test_run.rerun[1].text
)
) as sampleTraceback
FROM
commons.workflow_job job
INNER JOIN commons.test_run_s3 test_run ON test_run.job_id = job.id HINT(join_strategy = lookup)
INNER JOIN commons.workflow_run workflow ON job.run_id = workflow.id
WHERE
test_run.skipped.message LIKE '{%"flaky": _rue%'
AND test_run._event_time > (CURRENT_TIMESTAMP() - HOURs(:numHours))
AND test_run.name LIKE :name
AND test_run.classname LIKE :suite
AND test_run.file LIKE :file
AND job.name NOT LIKE '%rerun_disabled_tests%'
GROUP BY
name,
suite,
file,
invoking_file
UNION
with flaky_tests as (
select
*
from
default .test_run_s3 test_run
where
LENGTH(test_run.rerun) != 0
AND LENGTH(test_run.failure) = 0
AND test_run.name LIKE {name: String }
AND test_run.classname LIKE {suite: String }
AND test_run.file LIKE {file: String }
and test_run.time_inserted > (CURRENT_TIMESTAMP() - interval {numHours: Int64} hour)
)
select
test_run.name,
test_run.name as name,
test_run.classname as suite,
test_run.file,
test_run.invoking_file,
test_run.file as file,
test_run.invoking_file as invoking_file,
COUNT(*) as numGreen,
SUM(
if(
TYPEOF(test_run.rerun) = 'object',
1,
Length(test_run.rerun)
)
) as numRed,
SUM(Length(test_run.rerun)) as numRed,
ARRAY_AGG(job.name) as jobNames,
ARRAY_AGG(job.id) as jobIds,
ARRAY_AGG(workflow.id) as workflowIds,
ARRAY_AGG(workflow.name) as workflowNames,
ARRAY_AGG(workflow.head_branch) as branches,
ARRAY_AGG(test_run.workflow_run_attempt) as runAttempts,
ARBITRARY(
if(
TYPEOF(test_run.rerun) = 'object',
test_run.rerun.text,
test_run.rerun[1].text
)
) as sampleTraceback
any(test_run.rerun [ 1 ].'text') as sampleTraceback
FROM
commons.workflow_job job
INNER JOIN commons.test_run_s3 test_run ON test_run.job_id = job.id HINT(join_strategy = lookup)
INNER JOIN commons.workflow_run workflow ON job.run_id = workflow.id
default .workflow_job job final
INNER JOIN flaky_tests test_run ON test_run.job_id = job.id
INNER JOIN default .workflow_run workflow final ON job.run_id = workflow.id
where
test_run.rerun is not null
and test_run.failure is null
AND test_run._event_time > (CURRENT_TIMESTAMP() - HOURs(:numHours))
AND test_run.name LIKE :name
AND test_run.classname LIKE :suite
AND test_run.file LIKE :file
AND job.name NOT LIKE '%rerun_disabled_tests%'
workflow.id in (
select
workflow_id
from
flaky_tests
)
and job.id in (
select
job_id
from
flaky_tests
)
and workflow.head_branch = 'main'
and workflow.repository. 'full_name' = 'pytorch/pytorch'
and job.name not like '%rerun_disabled_tests%'
GROUP BY
name,
suite,
Expand Down
Loading

0 comments on commit d420136

Please sign in to comment.