From dc85aacf3dca21fa254738d82a7c1b02e9773569 Mon Sep 17 00:00:00 2001 From: Marcelo Robert Santos Date: Fri, 22 Nov 2024 17:29:03 -0300 Subject: [PATCH] feat: add test path filtering Part of #524 --- backend/kernelCI_app/utils.py | 7 +++ .../kernelCI_app/views/treeCommitsHistory.py | 3 ++ .../kernelCI_app/views/treeDetailsSlowView.py | 10 ++++ .../src/components/BootsTable/BootsTable.tsx | 46 ++++++++++++------- .../src/components/TestsTable/TestsTable.tsx | 45 +++++++++++------- .../pages/TreeDetails/Tabs/Boots/BootsTab.tsx | 16 +++++++ .../pages/TreeDetails/Tabs/Tests/TestsTab.tsx | 16 +++++++ .../pages/TreeDetails/TreeDetailsFilter.tsx | 1 + .../hardwareDetails/Tabs/Boots/BootsTab.tsx | 20 +++++++- .../Tabs/Tests/HardwareDetailsTestsTable.tsx | 2 + .../hardwareDetails/Tabs/Tests/TestsTab.tsx | 21 ++++++++- .../src/types/hardware/hardwareDetails.ts | 2 + dashboard/src/types/tree/TreeDetails.tsx | 2 + dashboard/src/utils/filters.ts | 1 + 14 files changed, 158 insertions(+), 34 deletions(-) diff --git a/backend/kernelCI_app/utils.py b/backend/kernelCI_app/utils.py index 77a75898..02a19f8e 100644 --- a/backend/kernelCI_app/utils.py +++ b/backend/kernelCI_app/utils.py @@ -106,8 +106,11 @@ class FilterParams: "gte": ["gte", ">="], "lt": ["lt", "<"], "lte": ["lte", "<="], + "like": ["like", "LIKE"], } + string_like_filters = ["test.path"] + def __init__(self, request): self.filters = [] self.create_filters_from_req(request) @@ -129,6 +132,10 @@ def create_filters_from_req(self, request): self.add_filter(field, value, "in") continue + if filter_term in self.string_like_filters: + self.add_filter(filter_term, request.GET.get(k), "like") + continue + match = self.filter_reg.match(filter_term) if match: field = match.group(1) diff --git a/backend/kernelCI_app/views/treeCommitsHistory.py b/backend/kernelCI_app/views/treeCommitsHistory.py index a1360baf..2724c9c8 100644 --- a/backend/kernelCI_app/views/treeCommitsHistory.py +++ b/backend/kernelCI_app/views/treeCommitsHistory.py @@ -70,6 +70,9 @@ def __treat_unknown_filter(self, table_field, op, value_name, filter): self.field_values[value_name] = filter['value'] if op == "IN": clause += f" = ANY(%({value_name})s)" + elif op == "LIKE": + self.field_values[value_name] = f"%{filter['value']}%" + clause += f" {op} %({value_name})s" else: clause += f" {op} %({value_name})s" diff --git a/backend/kernelCI_app/views/treeDetailsSlowView.py b/backend/kernelCI_app/views/treeDetailsSlowView.py index c0b2f075..0ad99420 100644 --- a/backend/kernelCI_app/views/treeDetailsSlowView.py +++ b/backend/kernelCI_app/views/treeDetailsSlowView.py @@ -27,6 +27,7 @@ def __init__(self): self.filterTreeDetailsCompiler = set() self.filterArchitecture = set() self.filterHardware = set() + self.filterPath = "" self.filter_handlers = { "boot.status": self.__handle_boot_status, "boot.duration": self.__handle_boot_duration, @@ -36,6 +37,7 @@ def __init__(self): "compiler": self.__handle_compiler, "architecture": self.__handle_architecture, "test.hardware": self.__handle_hardware, + "test.path": self.__handle_path, } self.testHistory = [] @@ -95,6 +97,9 @@ def __handle_architecture(self, current_filter): def __handle_hardware(self, current_filter): self.filterHardware.add(current_filter["value"]) + def __handle_path(self, current_filter): + self.filterPath = current_filter["value"] + def __processFilters(self, request): try: filter_params = FilterParams(request) @@ -470,8 +475,13 @@ def get(self, request, commit_hash: str | None): ) = currentRowData self.hardwareUsed.add(testEnvironmentCompatible) + if ( ( + self.filterPath != "" + and (self.filterPath not in path) + ) + or ( len(self.filterHardware) > 0 and (testEnvironmentCompatible not in self.filterHardware) ) diff --git a/dashboard/src/components/BootsTable/BootsTable.tsx b/dashboard/src/components/BootsTable/BootsTable.tsx index 58baedd9..bf6d66d8 100644 --- a/dashboard/src/components/BootsTable/BootsTable.tsx +++ b/dashboard/src/components/BootsTable/BootsTable.tsx @@ -118,6 +118,7 @@ interface IBootsTable { filter: TestsTableFilter; getRowLink: (testId: TestHistory['id']) => LinkProps; onClickFilter: (newFilter: TestsTableFilter) => void; + updatePathFilter: (pathFilter: string) => void; } const TableCellComponent = ({ @@ -196,6 +197,7 @@ export function BootsTable({ filter, getRowLink, onClickFilter, + updatePathFilter, }: IBootsTable): JSX.Element { const [sorting, setSorting] = useState([]); const [pagination, setPagination] = useState({ @@ -311,20 +313,40 @@ export function BootsTable({ ?.setFilterValue(filter !== 'all' ? filter : undefined); }, [filter, table]); + // TODO: there should be a filtering for the frontend before the backend AND that filtering should consider the individual tests inside each test "batch" (the data from individualTestsTables), not only the rows of the external table const onSearchChange = useCallback( - (e: React.ChangeEvent) => - table.setGlobalFilter(String(e.target.value)), - [table], + (e: React.ChangeEvent) => { + if (e.target.value !== undefined) { + updatePathFilter(e.target.value); + } + // TODO: only use the frontend filtering when the backend filter function is undefined (like in BuildDetails page) + table.setGlobalFilter(String(e.target.value)); + }, + [table, updatePathFilter], ); const groupHeaders = table.getHeaderGroups()[0]?.headers; const tableHeaders = useMemo((): JSX.Element[] => { return groupHeaders.map(header => { + const headerComponent = header.isPlaceholder + ? null + : flexRender(header.column.columnDef.header, header.getContext()); return ( - - {header.isPlaceholder - ? null - : flexRender(header.column.columnDef.header, header.getContext())} + + {header.id === 'path' ? ( +
+ {headerComponent} + {/* TODO: add startingValue with the currentPathFilter from the diffFilter param, same for TestsTable */} + +
+ ) : ( + headerComponent + )} ); }); @@ -416,15 +438,7 @@ export function BootsTable({ navigationLogsActions={navigationLogsActions} onOpenChange={onOpenChange} > -
- - -
+ {tableRows} diff --git a/dashboard/src/components/TestsTable/TestsTable.tsx b/dashboard/src/components/TestsTable/TestsTable.tsx index 3b456fdb..a26c77ef 100644 --- a/dashboard/src/components/TestsTable/TestsTable.tsx +++ b/dashboard/src/components/TestsTable/TestsTable.tsx @@ -47,15 +47,17 @@ export interface ITestsTable { columns?: ColumnDef[]; innerColumns?: ColumnDef[]; getRowLink: (testId: TestHistory['id']) => LinkProps; + updatePathFilter?: (pathFilter: string) => void; } export function TestsTable({ testHistory, onClickFilter, filter, - getRowLink, columns = defaultColumns, innerColumns = defaultInnerColumns, + getRowLink, + updatePathFilter, }: ITestsTable): JSX.Element { const [sorting, setSorting] = useState([]); const [expanded, setExpanded] = useState({}); @@ -253,20 +255,39 @@ export function TestsTable({ [filterCount, intl, filter], ); + // TODO: there should be a filtering for the frontend before the backend AND that filtering should consider the individual tests inside each test "batch" (the data from individualTestsTables), not only the rows of the external table const onSearchChange = useCallback( - (e: React.ChangeEvent) => - table.setGlobalFilter(String(e.target.value)), - [table], + (e: React.ChangeEvent) => { + if (e.target.value !== undefined && updatePathFilter) { + updatePathFilter(e.target.value); + } + // TODO: remove this frontend filter when the hardwareDetails backend filtering gets in place + table.setGlobalFilter(String(e.target.value)); + }, + [table, updatePathFilter], ); const groupHeaders = table.getHeaderGroups()[0]?.headers; const tableHeaders = useMemo((): JSX.Element[] => { return groupHeaders.map(header => { + const headerComponent = header.isPlaceholder + ? null + : flexRender(header.column.columnDef.header, header.getContext()); return ( - {header.isPlaceholder - ? null - : flexRender(header.column.columnDef.header, header.getContext())} + {header.id === 'path_group' ? ( +
+ {headerComponent} + +
+ ) : ( + headerComponent + )} ); }); @@ -316,15 +337,7 @@ export function TestsTable({ return (
-
- - -
+ {tableRows} diff --git a/dashboard/src/pages/TreeDetails/Tabs/Boots/BootsTab.tsx b/dashboard/src/pages/TreeDetails/Tabs/Boots/BootsTab.tsx index a73a7120..08256731 100644 --- a/dashboard/src/pages/TreeDetails/Tabs/Boots/BootsTab.tsx +++ b/dashboard/src/pages/TreeDetails/Tabs/Boots/BootsTab.tsx @@ -38,6 +38,21 @@ const BootsTab = ({ reqFilter }: BootsTabProps): JSX.Element => { const navigate = useNavigate({ from: '/tree/$treeId/' }); + const updatePathFilter = useCallback( + (pathFilter: string) => { + navigate({ + search: previousSearch => ({ + ...previousSearch, + diffFilter: { + ...previousSearch.diffFilter, + path: pathFilter === '' ? undefined : { [pathFilter]: true }, + }, + }), + }); + }, + [navigate], + ); + const onClickFilter = useCallback( (newFilter: TestsTableFilter): void => { navigate({ @@ -165,6 +180,7 @@ const BootsTab = ({ reqFilter }: BootsTabProps): JSX.Element => { onClickFilter={onClickFilter} testHistory={data.bootHistory} getRowLink={getRowLink} + updatePathFilter={updatePathFilter} />
); diff --git a/dashboard/src/pages/TreeDetails/Tabs/Tests/TestsTab.tsx b/dashboard/src/pages/TreeDetails/Tabs/Tests/TestsTab.tsx index d079815a..2c112f70 100644 --- a/dashboard/src/pages/TreeDetails/Tabs/Tests/TestsTab.tsx +++ b/dashboard/src/pages/TreeDetails/Tabs/Tests/TestsTab.tsx @@ -44,6 +44,21 @@ const TestsTab = ({ reqFilter }: TestsTabProps): JSX.Element => { const navigate = useNavigate({ from: '/tree/$treeId' }); + const updatePathFilter = useCallback( + (pathFilter: string) => { + navigate({ + search: previousSearch => ({ + ...previousSearch, + diffFilter: { + ...previousSearch.diffFilter, + path: pathFilter === '' ? undefined : { [pathFilter]: true }, + }, + }), + }); + }, + [navigate], + ); + const getRowLink = useCallback( (bootId: string): LinkProps => { return { @@ -170,6 +185,7 @@ const TestsTab = ({ reqFilter }: TestsTabProps): JSX.Element => { onClickFilter={onClickFilter} filter={tableFilter.testsTable} getRowLink={getRowLink} + updatePathFilter={updatePathFilter} /> ); diff --git a/dashboard/src/pages/TreeDetails/TreeDetailsFilter.tsx b/dashboard/src/pages/TreeDetails/TreeDetailsFilter.tsx index 8df92be2..6bacaa71 100644 --- a/dashboard/src/pages/TreeDetails/TreeDetailsFilter.tsx +++ b/dashboard/src/pages/TreeDetails/TreeDetailsFilter.tsx @@ -47,6 +47,7 @@ const filterFieldMap = { 'test.duration_[gte]': 'testDurationMin', 'test.duration_[lte]': 'testDurationMax', 'test.hardware': 'hardware', + 'test.path': 'path', } as const satisfies Record; export const mapFilterToReq = ( diff --git a/dashboard/src/pages/hardwareDetails/Tabs/Boots/BootsTab.tsx b/dashboard/src/pages/hardwareDetails/Tabs/Boots/BootsTab.tsx index da8705e4..6012d586 100644 --- a/dashboard/src/pages/hardwareDetails/Tabs/Boots/BootsTab.tsx +++ b/dashboard/src/pages/hardwareDetails/Tabs/Boots/BootsTab.tsx @@ -145,7 +145,9 @@ const ErrorsSummary = ({ export const MemoizedErrorsSummary = memo(ErrorsSummary); const BootsTab = ({ boots, hardwareId }: TBootsTab): JSX.Element => { - const { tableFilter } = useSearch({ from: '/hardware/$hardwareId' }); + const { tableFilter } = useSearch({ + from: '/hardware/$hardwareId', + }); const getRowLink = useCallback( (bootId: string): LinkProps => ({ @@ -161,6 +163,21 @@ const BootsTab = ({ boots, hardwareId }: TBootsTab): JSX.Element => { const navigate = useNavigate({ from: '/hardware/$hardwareId' }); + const updatePathFilter = useCallback( + (pathFilter: string) => { + navigate({ + search: previousSearch => ({ + ...previousSearch, + diffFilter: { + ...previousSearch.diffFilter, + path: pathFilter === '' ? undefined : { [pathFilter]: true }, + }, + }), + }); + }, + [navigate], + ); + const onClickFilter = useCallback( (newFilter: TestsTableFilter): void => { navigate({ @@ -227,6 +244,7 @@ const BootsTab = ({ boots, hardwareId }: TBootsTab): JSX.Element => { filter={tableFilter.bootsTable} testHistory={boots.history} onClickFilter={onClickFilter} + updatePathFilter={updatePathFilter} /> ); diff --git a/dashboard/src/pages/hardwareDetails/Tabs/Tests/HardwareDetailsTestsTable.tsx b/dashboard/src/pages/hardwareDetails/Tabs/Tests/HardwareDetailsTestsTable.tsx index 756eb3ae..78e4868c 100644 --- a/dashboard/src/pages/hardwareDetails/Tabs/Tests/HardwareDetailsTestsTable.tsx +++ b/dashboard/src/pages/hardwareDetails/Tabs/Tests/HardwareDetailsTestsTable.tsx @@ -94,6 +94,7 @@ const HardwareDetailsTestTable = ({ onClickFilter, testHistory, hardwareId, + updatePathFilter, }: IHardwareDetailsTestTable): JSX.Element => { const getRowLink = useCallback( (bootId: string): LinkProps => ({ @@ -114,6 +115,7 @@ const HardwareDetailsTestTable = ({ testHistory={testHistory} innerColumns={innerColumns} getRowLink={getRowLink} + updatePathFilter={updatePathFilter} /> ); }; diff --git a/dashboard/src/pages/hardwareDetails/Tabs/Tests/TestsTab.tsx b/dashboard/src/pages/hardwareDetails/Tabs/Tests/TestsTab.tsx index 42155d77..9ac9c174 100644 --- a/dashboard/src/pages/hardwareDetails/Tabs/Tests/TestsTab.tsx +++ b/dashboard/src/pages/hardwareDetails/Tabs/Tests/TestsTab.tsx @@ -27,10 +27,28 @@ interface TTestsTab { } const TestsTab = ({ tests, hardwareId }: TTestsTab): JSX.Element => { - const { tableFilter } = useSearch({ from: '/hardware/$hardwareId' }); + const { tableFilter } = useSearch({ + from: '/hardware/$hardwareId', + }); const navigate = useNavigate({ from: '/hardware/$hardwareId' }); + // TODO: move inside the same hook for the tables, passing navigate, in order to not repeat the same code in each tab of each monitor + const updatePathFilter = useCallback( + (pathFilter: string) => { + navigate({ + search: previousSearch => ({ + ...previousSearch, + diffFilter: { + ...previousSearch.diffFilter, + path: pathFilter === '' ? undefined : { [pathFilter]: true }, + }, + }), + }); + }, + [navigate], + ); + const onClickFilter = useCallback( (newFilter: TestsTableFilter): void => { navigate({ @@ -97,6 +115,7 @@ const TestsTab = ({ tests, hardwareId }: TTestsTab): JSX.Element => { filter={tableFilter.testsTable} hardwareId={hardwareId} onClickFilter={onClickFilter} + updatePathFilter={updatePathFilter} /> ); diff --git a/dashboard/src/types/hardware/hardwareDetails.ts b/dashboard/src/types/hardware/hardwareDetails.ts index 2483799f..3155c437 100644 --- a/dashboard/src/types/hardware/hardwareDetails.ts +++ b/dashboard/src/types/hardware/hardwareDetails.ts @@ -66,6 +66,7 @@ export const zFilterObjectsKeys = z.enum([ 'bootStatus', 'testStatus', 'trees', + 'path', ]); export const zFilterNumberKeys = z.enum([ 'buildDurationMin', @@ -96,6 +97,7 @@ export const zDiffFilter = z testDurationMin: zFilterNumberValue, testDurationMax: zFilterNumberValue, trees: zFilterBoolValue, + path: zFilterBoolValue, } satisfies Record), z.record(z.never()), ]) diff --git a/dashboard/src/types/tree/TreeDetails.tsx b/dashboard/src/types/tree/TreeDetails.tsx index 1e71843b..d7a4f2b9 100644 --- a/dashboard/src/types/tree/TreeDetails.tsx +++ b/dashboard/src/types/tree/TreeDetails.tsx @@ -158,6 +158,7 @@ export const zFilterObjectsKeys = z.enum([ 'bootStatus', 'testStatus', 'hardware', + 'path', ]); export const zFilterNumberKeys = z.enum([ 'buildDurationMin', @@ -182,6 +183,7 @@ export const zDiffFilter = z bootStatus: zFilterBoolValue, testStatus: zFilterBoolValue, hardware: zFilterBoolValue, + path: zFilterBoolValue, buildDurationMax: zFilterNumberValue, buildDurationMin: zFilterNumberValue, bootDurationMin: zFilterNumberValue, diff --git a/dashboard/src/utils/filters.ts b/dashboard/src/utils/filters.ts index 9556e8b9..83d93da3 100644 --- a/dashboard/src/utils/filters.ts +++ b/dashboard/src/utils/filters.ts @@ -7,6 +7,7 @@ const requestFilters = { 'test.duration_[gte]', 'test.duration_[lte]', 'test.hardware', + 'test.path', 'boot.status', 'boot.duration_[gte]', 'boot.duration_[lte]',