diff --git a/torchci/clickhouse_queries/individual_test_stats_per_workflow_per_oncall/params.json b/torchci/clickhouse_queries/individual_test_stats_per_workflow_per_oncall/params.json deleted file mode 100644 index 1358c35218..0000000000 --- a/torchci/clickhouse_queries/individual_test_stats_per_workflow_per_oncall/params.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "oncall": "String", - "queryDate": "String", - "thresholdInSecond": "Int64", - "workflow_name": "String" -} \ No newline at end of file diff --git a/torchci/clickhouse_queries/individual_test_stats_per_workflow_per_oncall/query.sql b/torchci/clickhouse_queries/individual_test_stats_per_workflow_per_oncall/query.sql deleted file mode 100644 index 7e3f77bbcf..0000000000 --- a/torchci/clickhouse_queries/individual_test_stats_per_workflow_per_oncall/query.sql +++ /dev/null @@ -1,143 +0,0 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted -WITH - workflow_summed_table AS ( - SELECT - workflow_id, - -- sum by job name to get total over all shards - SUM(sum_duration_in_second) as sum_duration_in_second, - oncalls, - date, - workflow_name, - test_class, - test_file, - config_job_name, - config_shard_name, - FROM - metrics.aggregated_test_metrics_with_preproc - WHERE - DATE_TRUNC('DAY', date) = DATE_TRUNC('DAY', PARSE_DATETIME_ISO8601(: queryDate)) - AND workflow_name =: workflow_name - GROUP BY - workflow_id, - workflow_name, - test_class, - test_file, - date, - oncalls, - config_job_name, - config_shard_name - ), - filtered_table AS ( - SELECT - AVG(sum_duration_in_second) as avg_duration_in_second, - COUNT(DISTINCT(workflow_id)) as workflow_occurences, - oncalls, - date, - workflow_name, - test_class, - test_file, - config_job_name, - config_shard_name, - FROM - workflow_summed_table - WHERE - DATE_TRUNC('DAY', date) = DATE_TRUNC('DAY', PARSE_DATETIME_ISO8601(: queryDate)) - GROUP BY - workflow_name, - test_class, - test_file, - date, - oncalls, - config_job_name, - config_shard_name - ), - filtered_with_costs AS ( - SELECT - t.avg_duration_in_second as avg_duration_in_second, - t.oncalls, - t.date, - t.workflow_name, - t.test_class, - t.test_file, - t.workflow_occurences as workflow_occurences, - t.config_job_name, - t.config_shard_name, - CASE - WHEN p.price IS NULL THEN 0 - ELSE CAST(p.price AS float) * t.avg_duration_in_second / 60 / 60 - END as estimated_price - FROM - filtered_table as t - LEFT JOIN commons.price_per_config p ON ( - t.config_job_name = p.job_name - AND t.config_shard_name = p.shard_name - ) - ), - test_runs AS ( - SELECT - count(*) as the_count, - REPLACE(REPLACE(t2.oncall, 'module: ', ''), 'oncall: ', '') as oncall, - t.workflow_name as workflow_type, - -- summing over job name here as it contains information on each shard - SUM(t.avg_duration_in_second) AS avg_duration_in_second, - SUM(t.estimated_price) as estimated_price_per_run_in_dollars, - t.date as granularity_bucket, - t.test_class as test_class, - t.test_file as test_file, - t.workflow_occurences, - t.config_job_name, - t.config_shard_name - FROM - filtered_with_costs as t, - UNNEST(t.oncalls AS oncall) AS t2 - WHERE - REPLACE(REPLACE(t2.oncall, 'module: ', ''), 'oncall: ', '') =: oncall - GROUP BY - t2.oncall, - t.config_job_name, - t.config_shard_name, - t.date, - t.test_class, - t.test_file, - t.workflow_name, - t.workflow_occurences - ORDER BY - avg_duration_in_second DESC - ), - test_runs_averaged as ( - SELECT - oncall, - workflow_type, - granularity_bucket, - test_class, - test_file, - TRUNC( - SUM( - estimated_price_per_run_in_dollars * workflow_occurences - ) / SUM(workflow_occurences), - 2 - ) as est_cost_per_run, - TRUNC( - SUM( - estimated_price_per_run_in_dollars * workflow_occurences - ), - 2 - ) as est_cost_per_day, - SUM(avg_duration_in_second) as avg_duration_in_second - FROM - test_runs - GROUP BY - oncall, - workflow_type, - granularity_bucket, - test_class, - test_file - ) -SELECT - *, -FROM - test_runs_averaged -WHERE - avg_duration_in_second >=: thresholdInSecond -ORDER BY - avg_duration_in_second DESC diff --git a/torchci/clickhouse_queries/individual_test_times_per_oncall_per_workflow/params.json b/torchci/clickhouse_queries/individual_test_times_per_oncall_per_workflow/params.json deleted file mode 100644 index e7e859ac9d..0000000000 --- a/torchci/clickhouse_queries/individual_test_times_per_oncall_per_workflow/params.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "classname": "String", - "oncall": "String", - "queryDate": "String", - "thresholdInSecond": "Float64", - "workflow_name": "String" -} \ No newline at end of file diff --git a/torchci/clickhouse_queries/individual_test_times_per_oncall_per_workflow/query.sql b/torchci/clickhouse_queries/individual_test_times_per_oncall_per_workflow/query.sql deleted file mode 100644 index ccdde4a1bd..0000000000 --- a/torchci/clickhouse_queries/individual_test_times_per_oncall_per_workflow/query.sql +++ /dev/null @@ -1,90 +0,0 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted -WITH - oncalls_table as ( - SELECT - test_file, - oncalls - FROM - commons.test_file_to_oncall_mapping - WHERE - DATE_TRUNC('DAY', PARSE_DATETIME_ISO8601(date)) >= DATE_TRUNC( - 'DAY', - CAST(PARSE_DATETIME_ISO8601(:queryDate) as date) - ) - ), - filtered_oncalls as ( - SELECT - f.test_file, - oncall - FROM - (oncalls_table f - CROSS JOIN UNNEST(oncalls AS oncall)) - WHERE - REPLACE(REPLACE(oncall, 'module: ', ''), 'oncall: ', '') LIKE: oncall - GROUP BY - f.test_file, - oncall - ), - workflow_id_table as ( - SELECT - CAST(workflow_id as STRING) as workflow_id, - workflow_name - from - commons.workflow_ids_from_test_aggregates - WHERE - workflow_name = :workflow_name - AND DATE_TRUNC('DAY', date) = DATE_TRUNC( - 'DAY', - CAST(PARSE_DATETIME_ISO8601(:queryDate) as date) - ) - LIMIT - 3 - ), test_times_filtered as ( - SELECT - test_runs.time, - test_runs.classname, - test_runs.invoking_file, - test_runs.name, - wid.workflow_name, - wid.workflow_id, - FROM - commons.test_run_s3 test_runs - INNER JOIN workflow_id_table wid ON (test_runs.workflow_id = wid.workflow_id) HINT(join_strategy = lookup) - WHERE - test_runs.workflow_run_attempt = 1 - AND test_runs.classname IS NOT NULL - AND test_runs.classname LIKE :classname - ), - test_times_with_oncalls as ( - SELECT - time, - classname, - workflow_name, - workflow_id, - name, - invoking_file, - oncall - FROM - test_times_filtered test_runs - INNER JOIN filtered_oncalls oncalls ON (test_runs.invoking_file = oncalls.test_file) HINT(join_strategy = lookup) - ) -SELECT - AVG(time) as avg_time_in_seconds, - SUM(time) / COUNT(DISTINCT(workflow_id)) as time_per_wokflow_in_seconds, - classname as test_class, - invoking_file as test_file, - name as test_name, - oncall, - workflow_name -FROM - test_times_with_oncalls -GROUP BY - oncall, - workflow_name, - invoking_file, - classname, - name -HAVING - AVG(time) >= :thresholdInSecond -ORDER BY - avg_time_in_seconds DESC diff --git a/torchci/clickhouse_queries/test_insights_latest_runs/params.json b/torchci/clickhouse_queries/test_insights_latest_runs/params.json deleted file mode 100644 index 7246538a39..0000000000 --- a/torchci/clickhouse_queries/test_insights_latest_runs/params.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "jobName": "String", - "limit": "Int64", - "startTime": "DateTime64(3)", - "stopTime": "DateTime64(3)", - "testClass": "String", - "testFile": "String", - "workflowName": "String" -} \ No newline at end of file diff --git a/torchci/clickhouse_queries/test_insights_latest_runs/query.sql b/torchci/clickhouse_queries/test_insights_latest_runs/query.sql deleted file mode 100644 index e34cf89de1..0000000000 --- a/torchci/clickhouse_queries/test_insights_latest_runs/query.sql +++ /dev/null @@ -1,27 +0,0 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted -SELECT DISTINCT - test_run_summary.workflow_id, - test_run_summary.job_id, - test_run_summary._event_time, - test_run_summary.time, - test_run_summary.tests, - test_run_summary.skipped, - test_run_summary.failures, - test_run_summary.errors -FROM - commons.test_run_summary - JOIN commons.workflow_run on test_run_summary.workflow_id = CAST(workflow_run.id as string) - JOIN commons.workflow_job on test_run_summary.job_id = workflow_job.id -WHERE - test_run_summary._event_time >= PARSE_DATETIME_ISO8601(:startTime) - AND test_run_summary._event_time < PARSE_DATETIME_ISO8601(:stopTime) - AND test_run_summary.workflow_run_attempt = 1 - AND workflow_run.name = :workflowName - AND workflow_job.name = :jobName - AND test_run_summary.invoking_file = :testFile - AND test_run_summary.classname = :testClass - AND workflow_run.head_branch = 'main' -ORDER BY - test_run_summary._event_time DESC -LIMIT - :limit diff --git a/torchci/clickhouse_queries/test_insights_overview/params.json b/torchci/clickhouse_queries/test_insights_overview/params.json deleted file mode 100644 index 601daf12e3..0000000000 --- a/torchci/clickhouse_queries/test_insights_overview/params.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "startTime": "DateTime64(3)", - "stopTime": "DateTime64(3)", - "testClass": "String", - "testFile": "String", - "thresholdInSecond": "Int64", - "workflowName": "String" -} \ No newline at end of file diff --git a/torchci/clickhouse_queries/test_insights_overview/query.sql b/torchci/clickhouse_queries/test_insights_overview/query.sql deleted file mode 100644 index cf936af717..0000000000 --- a/torchci/clickhouse_queries/test_insights_overview/query.sql +++ /dev/null @@ -1,58 +0,0 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted -WITH test_runs AS ( - SELECT - workflow_run.name AS workflow_name, - workflow_job.name AS job_name, - test_run_summary.invoking_file AS test_file, - test_run_summary.classname AS test_class, - test_run_summary.tests AS tests, - test_run_summary.errors AS errors, - test_run_summary.failures AS failures, - test_run_summary.skipped AS skipped, - test_run_summary.time AS duration_in_second, - FROM - commons.test_run_summary - JOIN commons.workflow_run on test_run_summary.workflow_id = CAST(workflow_run.id as string) - JOIN commons.workflow_job on test_run_summary.job_id = workflow_job.id - WHERE - test_run_summary._event_time >= PARSE_DATETIME_ISO8601(: startTime) - AND test_run_summary._event_time < PARSE_DATETIME_ISO8601(: stopTime) - AND test_run_summary.workflow_run_attempt = 1 - AND workflow_run.name = : workflowName - AND workflow_run.head_branch = 'main' - AND test_run_summary.invoking_file LIKE : testFile - AND test_run_summary.classname LIKE : testClass -), -aggregated_test_runs AS ( - SELECT - workflow_name, - job_name, - test_file, - test_class, - CAST( - AVG(duration_in_second) AS int - ) avg_duration_in_second, - CAST( - AVG(tests) AS int - ) AS avg_tests, - MAX(failures) AS max_failures, - MAX(errors) AS max_errors, - CAST( - AVG(skipped) AS int - ) AS avg_skipped, - FROM - test_runs - GROUP BY - workflow_name, - job_name, - test_file, - test_class -) -SELECT - * -FROM - aggregated_test_runs -WHERE - avg_duration_in_second >= : thresholdInSecond -ORDER BY - avg_duration_in_second DESC diff --git a/torchci/components/NavBar.tsx b/torchci/components/NavBar.tsx index 9b2481d68b..41553bcccf 100644 --- a/torchci/components/NavBar.tsx +++ b/torchci/components/NavBar.tsx @@ -89,10 +89,6 @@ function NavBar() { name: "Disabled Tests", href: "/disabled", }, - { - name: "Testing Overhead", - href: "/testing_overhead", - }, { name: "Cost Analysis", href: "/cost_analysis", diff --git a/torchci/components/metrics/panels/GenerateIndividualTestsLeaderboard.tsx b/torchci/components/metrics/panels/GenerateIndividualTestsLeaderboard.tsx deleted file mode 100644 index 96508e175a..0000000000 --- a/torchci/components/metrics/panels/GenerateIndividualTestsLeaderboard.tsx +++ /dev/null @@ -1,133 +0,0 @@ -import { Grid } from "@mui/material"; -import { - GridRenderCellParams, - GridValueFormatterParams, -} from "@mui/x-data-grid"; -import TablePanel from "components/metrics/panels/TablePanel"; -import { durationDisplay } from "components/TimeUtils"; -import dayjs from "dayjs"; -import { RocksetParam } from "lib/rockset"; -import { useState } from "react"; - -const ROW_HEIGHT = 500; - -export default function GenerateIndividualTestsLeaderboard({ - oncallName = "%", - workflowName, - thresholdInSecond, - classname = "%", -}: { - oncallName?: string; - workflowName: string; - thresholdInSecond: number; - classname?: string; -}) { - const [queryDate, _setQueryDate] = useState(dayjs().subtract(2, "day")); - const queryParamsForLongTestTable: RocksetParam[] = [ - { - name: "oncall", - type: "string", - value: oncallName, - }, - { - name: "queryDate", - type: "string", - value: queryDate, - }, - { - name: "workflow_name", - type: "string", - value: workflowName, - }, - { - name: "thresholdInSecond", - type: "int", - value: thresholdInSecond, - }, - { - name: "classname", - type: "string", - value: classname, - }, - ]; - return ( - - ) => { - const oncall = params.value; - return ( - - {oncall} - - ); - }, - }, - { - field: "test_name", - headerName: "Test Name", - flex: 1, - filterable: false, - }, - { - field: "test_class", - headerName: "Test Class", - flex: 1, - renderCell: (params: GridRenderCellParams) => { - const testFile = params.row.test_file; - const testClass = params.value; - return ( - - {testClass} - - ); - }, - }, - { - field: "test_file", - headerName: "Test File", - flex: 1, - filterable: false, - }, - { - field: "avg_time_in_seconds", - headerName: "Avg Duration per Runner", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - durationDisplay(params.value), - filterable: false, - }, - { - field: "time_per_wokflow_in_seconds", - headerName: "Total Duration per Workflow", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - durationDisplay(params.value), - filterable: false, - }, - ]} - dataGridProps={{ - getRowId: (e: any) => - e.oncall + - e.workflow_name + - e.test_file + - e.test_class + - e.test_name, - initialState: {}, - }} - /> - - ); -} diff --git a/torchci/components/metrics/panels/GenerateTestInsightsOverviewTable.tsx b/torchci/components/metrics/panels/GenerateTestInsightsOverviewTable.tsx deleted file mode 100644 index 2673cadfbd..0000000000 --- a/torchci/components/metrics/panels/GenerateTestInsightsOverviewTable.tsx +++ /dev/null @@ -1,173 +0,0 @@ -import { Grid } from "@mui/material"; -import { - GridRenderCellParams, - GridValueFormatterParams, -} from "@mui/x-data-grid"; -import TablePanel from "components/metrics/panels/TablePanel"; -import { durationDisplay } from "components/TimeUtils"; -import dayjs from "dayjs"; -import { RocksetParam } from "lib/rockset"; - -const ROW_HEIGHT = 500; - -export default function GenerateTestInsightsOverviewTable({ - workflowName, - startTime, - stopTime, - thresholdInSeconds, - testFile = "%", - testClass = "%", -}: { - workflowName: string; - startTime: dayjs.Dayjs; - stopTime: dayjs.Dayjs; - thresholdInSeconds: number; - testFile?: string; - testClass?: string; -}) { - const queryParams: RocksetParam[] = [ - { - name: "startTime", - type: "string", - value: startTime, - }, - { - name: "stopTime", - type: "string", - value: stopTime, - }, - { - name: "workflowName", - type: "string", - value: workflowName, - }, - { - name: "thresholdInSecond", - type: "int", - value: thresholdInSeconds, - }, - { - name: "testFile", - type: "string", - value: testFile, - }, - { - name: "testClass", - type: "string", - value: testClass, - }, - ]; - - return ( - - ) => - durationDisplay(params.value), - filterable: false, - }, - { - field: "avg_tests", - headerName: "# Test cases", - flex: 1, - filterable: false, - }, - { - field: "job_name", - headerName: "Full job name", - flex: 1, - }, - { - field: "simplified_job_name", - headerName: "Job name", - flex: 1, - valueGetter: (params) => { - const match = params.row.job_name.match( - new RegExp("^(.+)\\s\\/\\s.+$") - ); - return match ? match[1] : params.value; - }, - }, - { - field: "shard", - headerName: "Shard", - flex: 1, - valueGetter: (params) => { - const match = params.row.job_name.match( - new RegExp("^.+\\s\\/\\s.+\\(([^,]+),\\s([^,]+),.+\\)$") - ); - return match ? `${match[1]} (${match[2]})` : params.row.job_name; - }, - }, - { - field: "runner", - headerName: "Runner", - flex: 1, - valueGetter: (params) => { - const match = params.row.job_name.match( - new RegExp("^.+,\\s([^,]+)\\)$") - ); - return match ? `${match[1]}` : params.row.job_name; - }, - }, - { - field: "test_file", - headerName: "Test file", - flex: 1, - }, - { - field: "test_class", - headerName: "Test class", - flex: 1, - renderCell: (params: GridRenderCellParams) => { - const jobName = encodeURIComponent(params.row.job_name); - const testFile_param = params.row.test_file; - const testClass_param = params.value; - const detailParams = `workflowName=${workflowName}&jobName=${jobName}&testFile=${testFile_param}&testClass=${testClass_param}`; - - return ( - {testClass_param} - ); - }, - }, - { - field: "max_failures", - headerName: "# Test failures", - flex: 1, - filterable: false, - }, - { - field: "max_errors", - headerName: "# Unexpected errors", - flex: 1, - filterable: false, - }, - { - field: "avg_skipped", - headerName: "# Test skipped", - flex: 1, - filterable: false, - }, - ]} - dataGridProps={{ - getRowId: (e: any) => e.job_name + e.test_file + e.test_class, - initialState: { - columns: { - columnVisibilityModel: { - job_name: false, - }, - }, - }, - }} - /> - - ); -} diff --git a/torchci/pages/testing_overhead.tsx b/torchci/pages/testing_overhead.tsx deleted file mode 100644 index 53a908e1f1..0000000000 --- a/torchci/pages/testing_overhead.tsx +++ /dev/null @@ -1,182 +0,0 @@ -import { Grid } from "@mui/material"; -import { - GridRenderCellParams, - GridValueFormatterParams, -} from "@mui/x-data-grid"; -import TablePanel from "components/metrics/panels/TablePanel"; -import TimeSeriesPanel from "components/metrics/panels/TimeSeriesPanel"; -import WorkflowPicker, { - WORKFLOWS, -} from "components/metrics/panels/WorkflowPicker"; -import { durationDisplay } from "components/TimeUtils"; -import dayjs from "dayjs"; -import { RocksetParam } from "lib/rockset"; -import { useState } from "react"; -import GenerateIndividualTestsLeaderboard from "../components/metrics/panels/GenerateIndividualTestsLeaderboard"; - -const ROW_HEIGHT = 240; -const THRESHOLD_IN_SECOND = 10; - -export default function TestingOverhead() { - // Looking at data from the past six months - const [startTime, _setStartTime] = useState(dayjs().subtract(1, "month")); - const [stopTime, _setStopTime] = useState(dayjs()); - const [workflow, setWorkFlow] = useState(Object.keys(WORKFLOWS)[0]); - const timeParams: RocksetParam[] = [ - { - name: "startTime", - type: "string", - value: startTime, - }, - { - name: "stopTime", - type: "string", - value: stopTime, - }, - ]; - - function GenerateOncallTestingOverheadLeaderboard({ - workflowName, - }: { - workflowName: string; - }) { - const [startTime, _setStartTime] = useState(dayjs().subtract(2, "day")); - const [endTime, _setEndTime] = useState(dayjs().subtract(1, "day")); - const queryParams: RocksetParam[] = [ - { - name: "oncall", - type: "string", - value: "%", - }, - { - name: "startDate", - type: "string", - value: startTime, - }, - { - name: "endDate", - type: "string", - value: endTime, - }, - { - name: "workflow_type", - type: "string", - value: workflowName, - }, - ]; - - return ( - - ) => { - const oncall = params.value; - return ( - - {oncall} - - ); - }, - }, - { - field: "time_in_seconds", - headerName: "Total duration", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - durationDisplay(params.value), - filterable: false, - }, - { - field: "percentage_of_time", - headerName: "% of workflow time", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - params.value + "%", - filterable: false, - }, - { - field: "estimated_price_per_run_in_dollars", - headerName: "Estimated price per run", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - `$${params.value.toFixed(2)}`, - filterable: false, - }, - { - field: "estimated_price_per_day_in_dollars", - headerName: "Estimated price per day", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - `$${params.value.toFixed(2)}`, - filterable: false, - }, - { - field: "percentage_of_cost", - headerName: "% of workflow cost", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - params.value + "%", - filterable: false, - }, - ]} - dataGridProps={{ - getRowId: (e: any) => - e.oncall + e.date + e.workflow_name + e.time_in_seconds, - initialState: {}, - }} - /> - - ); - } - - return ( - <> - <> - - - - durationDisplay(unit)} - groupByFieldName={"workflow_type"} - additionalOptions={{ yAxis: { scale: true } }} - /> - - - - - - - ); -} diff --git a/torchci/pages/testing_overhead/insights.tsx b/torchci/pages/testing_overhead/insights.tsx deleted file mode 100644 index eb67e3ccbf..0000000000 --- a/torchci/pages/testing_overhead/insights.tsx +++ /dev/null @@ -1,73 +0,0 @@ -import { Grid, Stack, Typography } from "@mui/material"; -import dayjs from "dayjs"; -import { useRouter } from "next/router"; -import { TimeRangePicker } from "pages/metrics"; -import { useState } from "react"; -import GenerateTestInsightsOverviewTable from "../../components/metrics/panels/GenerateTestInsightsOverviewTable"; - -const THRESHOLD_IN_SECOND = 60; - -export default function IndividualTestInsights() { - const [startTime, setStartTime] = useState(dayjs().subtract(1, "week")); - const [stopTime, setStopTime] = useState(dayjs()); - const [timeRange, setTimeRange] = useState(7); - - const router = useRouter(); - const testFile = router.query.testFile as string; - const testClass = router.query.testClass as string; - return ( -
- - - Test Insights for {testFile}.{testClass} - - - - - - - - - - - - - -
- ); -} diff --git a/torchci/pages/testing_overhead/oncall_insights.tsx b/torchci/pages/testing_overhead/oncall_insights.tsx deleted file mode 100644 index 31048fbd93..0000000000 --- a/torchci/pages/testing_overhead/oncall_insights.tsx +++ /dev/null @@ -1,178 +0,0 @@ -import { Grid } from "@mui/material"; -import { - GridRenderCellParams, - GridValueFormatterParams, -} from "@mui/x-data-grid"; -import GenerateIndividualTestsLeaderboard from "components/metrics/panels/GenerateIndividualTestsLeaderboard"; -import TablePanel from "components/metrics/panels/TablePanel"; -import TimeSeriesPanel from "components/metrics/panels/TimeSeriesPanel"; -import WorkflowPicker, { - WORKFLOWS, -} from "components/metrics/panels/WorkflowPicker"; -import { durationDisplay } from "components/TimeUtils"; -import dayjs from "dayjs"; -import { RocksetParam } from "lib/rockset"; -import { useRouter } from "next/router"; -import { useState } from "react"; - -const ROW_HEIGHT = 500; -const THRESHOLD_IN_SECOND = 60; - -function GenerateOncallTestInsightsOverviewTable({ - workflowName, -}: { - workflowName: string; -}) { - const router = useRouter(); - const oncall = router.query.oncall as string; - const [startTime, _setStartTime] = useState(dayjs().subtract(2, "day")); - const queryParams: RocksetParam[] = [ - { - name: "queryDate", - type: "string", - value: startTime, - }, - { - name: "workflowName", - type: "string", - value: workflowName, - }, - { - name: "thresholdInSecond", - type: "int", - value: THRESHOLD_IN_SECOND, - }, - { - name: "oncall", - type: "string", - value: `${oncall}`, - }, - ]; - - return ( - - ) => - durationDisplay(params.value), - filterable: false, - }, - { - field: "est_cost_per_run", - headerName: "Estimated cost per workflow run on all runners", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - `$${params.value.toFixed(2)}`, - filterable: false, - }, - { - field: "est_cost_per_day", - headerName: "Estimated cost per day on all runners", - flex: 1, - valueFormatter: (params: GridValueFormatterParams) => - `$${params.value.toFixed(2)}`, - filterable: false, - }, - { - field: "test_file", - headerName: "Test file", - flex: 1, - }, - { - field: "test_class", - headerName: "Test class", - flex: 1, - renderCell: (params: GridRenderCellParams) => { - const testFile = params.row.test_file; - const testClass = params.value; - return ( - - {testClass} - - ); - }, - }, - ]} - dataGridProps={{ - getRowId: (e: any) => e.test_file + e.test_class, - initialState: {}, - }} - /> - - ); -} - -export default function TestingOverhead() { - const router = useRouter(); - const oncall = router.query.oncall as string; - // Looking at data from the past six months - const [startTime, _setStartTime] = useState(dayjs().subtract(1, "month")); - const [endTime, _setEndTime] = useState(dayjs()); - const [workflow, setWorkFlow] = useState(Object.keys(WORKFLOWS)[0]); - return ( - <> - <> - - - - durationDisplay(unit)} - groupByFieldName={"workflow_name"} - additionalOptions={{ yAxis: { scale: true } }} - /> - - - - - - - - - - ); -} diff --git a/torchci/pages/tests.tsx b/torchci/pages/tests.tsx deleted file mode 100644 index c757056994..0000000000 --- a/torchci/pages/tests.tsx +++ /dev/null @@ -1,61 +0,0 @@ -import { Grid, Stack, Typography } from "@mui/material"; -import dayjs from "dayjs"; -import { useState } from "react"; -import GenerateTestInsightsOverviewTable from "../components/metrics/panels/GenerateTestInsightsOverviewTable"; -import { TimeRangePicker } from "./metrics"; - -const THRESHOLD_IN_SECOND = 60; - -export default function GatherTestsInfo() { - const [startTime, setStartTime] = useState(dayjs().subtract(1, "week")); - const [stopTime, setStopTime] = useState(dayjs()); - const [timeRange, setTimeRange] = useState(7); - - return ( -
- - - Test Insights - - - - - - - - - - - - - -
- ); -} diff --git a/torchci/rockset/commons/__sql/individual_test_stats_per_workflow_per_oncall.sql b/torchci/rockset/commons/__sql/individual_test_stats_per_workflow_per_oncall.sql deleted file mode 100644 index d04f834c47..0000000000 --- a/torchci/rockset/commons/__sql/individual_test_stats_per_workflow_per_oncall.sql +++ /dev/null @@ -1,142 +0,0 @@ -WITH - workflow_summed_table AS ( - SELECT - workflow_id, - -- sum by job name to get total over all shards - SUM(sum_duration_in_second) as sum_duration_in_second, - oncalls, - date, - workflow_name, - test_class, - test_file, - config_job_name, - config_shard_name, - FROM - metrics.aggregated_test_metrics_with_preproc - WHERE - DATE_TRUNC('DAY', date) = DATE_TRUNC('DAY', PARSE_DATETIME_ISO8601(: queryDate)) - AND workflow_name =: workflow_name - GROUP BY - workflow_id, - workflow_name, - test_class, - test_file, - date, - oncalls, - config_job_name, - config_shard_name - ), - filtered_table AS ( - SELECT - AVG(sum_duration_in_second) as avg_duration_in_second, - COUNT(DISTINCT(workflow_id)) as workflow_occurences, - oncalls, - date, - workflow_name, - test_class, - test_file, - config_job_name, - config_shard_name, - FROM - workflow_summed_table - WHERE - DATE_TRUNC('DAY', date) = DATE_TRUNC('DAY', PARSE_DATETIME_ISO8601(: queryDate)) - GROUP BY - workflow_name, - test_class, - test_file, - date, - oncalls, - config_job_name, - config_shard_name - ), - filtered_with_costs AS ( - SELECT - t.avg_duration_in_second as avg_duration_in_second, - t.oncalls, - t.date, - t.workflow_name, - t.test_class, - t.test_file, - t.workflow_occurences as workflow_occurences, - t.config_job_name, - t.config_shard_name, - CASE - WHEN p.price IS NULL THEN 0 - ELSE CAST(p.price AS float) * t.avg_duration_in_second / 60 / 60 - END as estimated_price - FROM - filtered_table as t - LEFT JOIN commons.price_per_config p ON ( - t.config_job_name = p.job_name - AND t.config_shard_name = p.shard_name - ) - ), - test_runs AS ( - SELECT - count(*) as the_count, - REPLACE(REPLACE(t2.oncall, 'module: ', ''), 'oncall: ', '') as oncall, - t.workflow_name as workflow_type, - -- summing over job name here as it contains information on each shard - SUM(t.avg_duration_in_second) AS avg_duration_in_second, - SUM(t.estimated_price) as estimated_price_per_run_in_dollars, - t.date as granularity_bucket, - t.test_class as test_class, - t.test_file as test_file, - t.workflow_occurences, - t.config_job_name, - t.config_shard_name - FROM - filtered_with_costs as t, - UNNEST(t.oncalls AS oncall) AS t2 - WHERE - REPLACE(REPLACE(t2.oncall, 'module: ', ''), 'oncall: ', '') =: oncall - GROUP BY - t2.oncall, - t.config_job_name, - t.config_shard_name, - t.date, - t.test_class, - t.test_file, - t.workflow_name, - t.workflow_occurences - ORDER BY - avg_duration_in_second DESC - ), - test_runs_averaged as ( - SELECT - oncall, - workflow_type, - granularity_bucket, - test_class, - test_file, - TRUNC( - SUM( - estimated_price_per_run_in_dollars * workflow_occurences - ) / SUM(workflow_occurences), - 2 - ) as est_cost_per_run, - TRUNC( - SUM( - estimated_price_per_run_in_dollars * workflow_occurences - ), - 2 - ) as est_cost_per_day, - SUM(avg_duration_in_second) as avg_duration_in_second - FROM - test_runs - GROUP BY - oncall, - workflow_type, - granularity_bucket, - test_class, - test_file - ) -SELECT - *, -FROM - test_runs_averaged -WHERE - avg_duration_in_second >=: thresholdInSecond -ORDER BY - avg_duration_in_second DESC diff --git a/torchci/rockset/commons/__sql/individual_test_times_per_oncall_per_workflow.sql b/torchci/rockset/commons/__sql/individual_test_times_per_oncall_per_workflow.sql deleted file mode 100644 index c17bef0539..0000000000 --- a/torchci/rockset/commons/__sql/individual_test_times_per_oncall_per_workflow.sql +++ /dev/null @@ -1,89 +0,0 @@ -WITH - oncalls_table as ( - SELECT - test_file, - oncalls - FROM - commons.test_file_to_oncall_mapping - WHERE - DATE_TRUNC('DAY', PARSE_DATETIME_ISO8601(date)) >= DATE_TRUNC( - 'DAY', - CAST(PARSE_DATETIME_ISO8601(:queryDate) as date) - ) - ), - filtered_oncalls as ( - SELECT - f.test_file, - oncall - FROM - (oncalls_table f - CROSS JOIN UNNEST(oncalls AS oncall)) - WHERE - REPLACE(REPLACE(oncall, 'module: ', ''), 'oncall: ', '') LIKE: oncall - GROUP BY - f.test_file, - oncall - ), - workflow_id_table as ( - SELECT - CAST(workflow_id as STRING) as workflow_id, - workflow_name - from - commons.workflow_ids_from_test_aggregates - WHERE - workflow_name = :workflow_name - AND DATE_TRUNC('DAY', date) = DATE_TRUNC( - 'DAY', - CAST(PARSE_DATETIME_ISO8601(:queryDate) as date) - ) - LIMIT - 3 - ), test_times_filtered as ( - SELECT - test_runs.time, - test_runs.classname, - test_runs.invoking_file, - test_runs.name, - wid.workflow_name, - wid.workflow_id, - FROM - commons.test_run_s3 test_runs - INNER JOIN workflow_id_table wid ON (test_runs.workflow_id = wid.workflow_id) HINT(join_strategy = lookup) - WHERE - test_runs.workflow_run_attempt = 1 - AND test_runs.classname IS NOT NULL - AND test_runs.classname LIKE :classname - ), - test_times_with_oncalls as ( - SELECT - time, - classname, - workflow_name, - workflow_id, - name, - invoking_file, - oncall - FROM - test_times_filtered test_runs - INNER JOIN filtered_oncalls oncalls ON (test_runs.invoking_file = oncalls.test_file) HINT(join_strategy = lookup) - ) -SELECT - AVG(time) as avg_time_in_seconds, - SUM(time) / COUNT(DISTINCT(workflow_id)) as time_per_wokflow_in_seconds, - classname as test_class, - invoking_file as test_file, - name as test_name, - oncall, - workflow_name -FROM - test_times_with_oncalls -GROUP BY - oncall, - workflow_name, - invoking_file, - classname, - name -HAVING - AVG(time) >= :thresholdInSecond -ORDER BY - avg_time_in_seconds DESC diff --git a/torchci/rockset/commons/individual_test_stats_per_workflow_per_oncall.lambda.json b/torchci/rockset/commons/individual_test_stats_per_workflow_per_oncall.lambda.json deleted file mode 100644 index e826b02016..0000000000 --- a/torchci/rockset/commons/individual_test_stats_per_workflow_per_oncall.lambda.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "sql_path": "__sql/individual_test_stats_per_workflow_per_oncall.sql", - "default_parameters": [ - { - "name": "oncall", - "type": "string", - "value": "functorch" - }, - { - "name": "queryDate", - "type": "string", - "value": "2023-04-11T00:06:32.839Z" - }, - { - "name": "thresholdInSecond", - "type": "int", - "value": "60" - }, - { - "name": "workflow_name", - "type": "string", - "value": "pull" - } - ], - "description": "" -} \ No newline at end of file diff --git a/torchci/rockset/commons/individual_test_times_per_oncall_per_workflow.lambda.json b/torchci/rockset/commons/individual_test_times_per_oncall_per_workflow.lambda.json deleted file mode 100644 index 0fd2337b6c..0000000000 --- a/torchci/rockset/commons/individual_test_times_per_oncall_per_workflow.lambda.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "sql_path": "__sql/individual_test_times_per_oncall_per_workflow.sql", - "default_parameters": [ - { - "name": "classname", - "type": "string", - "value": "%" - }, - { - "name": "oncall", - "type": "string", - "value": "%" - }, - { - "name": "queryDate", - "type": "string", - "value": "2023-05-06T21:08:08.862Z" - }, - { - "name": "thresholdInSecond", - "type": "float", - "value": "10" - }, - { - "name": "workflow_name", - "type": "string", - "value": "pull" - } - ], - "description": "" -} \ No newline at end of file diff --git a/torchci/rockset/commons/test_insights_latest_runs.lambda.json b/torchci/rockset/commons/test_insights_latest_runs.lambda.json deleted file mode 100644 index e66501a1c1..0000000000 --- a/torchci/rockset/commons/test_insights_latest_runs.lambda.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "sql_path": "__sql/test_insights_latest_runs.sql", - "default_parameters": [ - { - "name": "jobName", - "type": "string", - "value": "linux-bionic-cuda11.6-py3.10-gcc7 / test (default, 1, 4, linux.4xlarge.nvidia.gpu)" - }, - { - "name": "limit", - "type": "int", - "value": "10" - }, - { - "name": "startTime", - "type": "string", - "value": "2022-08-30T00:00:00.000Z" - }, - { - "name": "stopTime", - "type": "string", - "value": "2022-09-06T00:00:00.000Z" - }, - { - "name": "testClass", - "type": "string", - "value": "TestCommonCUDA" - }, - { - "name": "testFile", - "type": "string", - "value": "test_ops" - }, - { - "name": "workflowName", - "type": "string", - "value": "pull" - } - ], - "description": "" -} \ No newline at end of file diff --git a/torchci/rockset/commons/test_insights_overview.lambda.json b/torchci/rockset/commons/test_insights_overview.lambda.json deleted file mode 100644 index ab37910e7d..0000000000 --- a/torchci/rockset/commons/test_insights_overview.lambda.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "sql_path": "__sql/test_insights_overview.sql", - "default_parameters": [ - { - "name": "startTime", - "type": "string", - "value": "2023-04-02T00:00:00.000Z" - }, - { - "name": "stopTime", - "type": "string", - "value": "2023-04-04T00:00:00.000Z" - }, - { - "name": "testClass", - "type": "string", - "value": "%" - }, - { - "name": "testFile", - "type": "string", - "value": "%" - }, - { - "name": "thresholdInSecond", - "type": "int", - "value": "1800" - }, - { - "name": "workflowName", - "type": "string", - "value": "pull" - } - ], - "description": "" -} \ No newline at end of file diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json index 52b897f9c2..a857ad71d2 100644 --- a/torchci/rockset/prodVersions.json +++ b/torchci/rockset/prodVersions.json @@ -9,8 +9,6 @@ "flaky_tests": "be53d5f27248e365", "flaky_tests_across_jobs": "474e5454bda0c5bb", "get_relevant_alerts": "727014a49bef2c20", - "individual_test_stats_per_workflow_per_oncall": "559b6735965f1eb2", - "individual_test_times_per_oncall_per_workflow": "6b63c3dde3032bea", "flaky_workflows_jobs": "3ac657ca40327f94", "failed_workflow_jobs": "a91753fbbf82d470", "get_workflow_jobs": "6ed2029b19691a4b", @@ -25,8 +23,6 @@ "recent_pr_workflows_query": "db6b28f729e52892", "reverted_prs_with_reason": "751f01cba16364f0", "unclassified": "1b31a2d8f4ab7230", - "test_insights_overview": "42dbd5232f45fd53", - "test_insights_latest_runs": "1871833a91cb8b1b", "master_commit_red_jobs": "4869b467679a616a", "weekly_force_merge_stats": "d2264131599bcf6e", "pr_commits": "bbbbdf0c62db15b1",