From ff9c2c5fa674b699f628793b223466c1a6923af5 Mon Sep 17 00:00:00 2001 From: Murilo Geraldini Date: Thu, 21 Nov 2024 15:47:32 -0300 Subject: [PATCH] refactor: remove unused endpoints - Remove `/tree/tests` and `/tree//tests` that were no longer being used, including the views for these endpoints in the backend and the calling of these endpoints in the frontend, including the route for it. Closes #556 --- backend/kernelCI_app/urls.py | 8 - .../kernelCI_app/views/groupedTestsView.py | 98 ------------ backend/kernelCI_app/views/treeTestsView.py | 146 ------------------ backend/requests/boots-get.sh | 80 ---------- backend/requests/grouped-tests.sh | 1 - backend/requests/revision-tests-get.sh | 33 ---- dashboard/src/api/TreeDetails.tsx | 46 ------ dashboard/src/routeTree.gen.ts | 42 ++--- .../src/routes/tree/$treeId/test/route.tsx | 3 - 9 files changed, 10 insertions(+), 447 deletions(-) delete mode 100644 backend/kernelCI_app/views/groupedTestsView.py delete mode 100644 backend/kernelCI_app/views/treeTestsView.py delete mode 100644 backend/requests/boots-get.sh delete mode 100755 backend/requests/grouped-tests.sh delete mode 100644 backend/requests/revision-tests-get.sh delete mode 100644 dashboard/src/routes/tree/$treeId/test/route.tsx diff --git a/backend/kernelCI_app/urls.py b/backend/kernelCI_app/urls.py index fdddd952..bf114283 100644 --- a/backend/kernelCI_app/urls.py +++ b/backend/kernelCI_app/urls.py @@ -28,10 +28,6 @@ def viewCache(view): viewCache(views.TreeViewFast), name="tree-fast" ), - path("tree/tests/", - viewCache(views.groupedTests), - name="treeGroupedTests" - ), path("tree/", viewCache(views.TreeDetails), name="treeDetails" @@ -44,10 +40,6 @@ def viewCache(view): viewCache(views.TreeCommitsHistory), name="treeCommits" ), - path("tree//tests/", - viewCache(views.TreeTestsView), - name="treeTests" - ), path("build/", viewCache(views.BuildDetails), name="buildDetails" diff --git a/backend/kernelCI_app/views/groupedTestsView.py b/backend/kernelCI_app/views/groupedTestsView.py deleted file mode 100644 index af24d433..00000000 --- a/backend/kernelCI_app/views/groupedTestsView.py +++ /dev/null @@ -1,98 +0,0 @@ -from django.http import JsonResponse -from django.views import View -from django.db.models import Q, TextField, Sum, Count, Case, When, F, Func, Value -from django.db.models.functions import Concat -from django.contrib.postgres.aggregates import ArrayAgg - -from utils.validation import validate_required_params -from kernelCI_app.models import Tests -from kernelCI_app.serializers import GroupedTestsSerializer - - -class groupedTests(View): - - def get(self, request): - commit = request.GET.get("git_commit_hash") - branch = request.GET.get("git_repository_branch") - url = request.GET.get("git_repository_url") - origin = request.GET.get("origin") - - get_required_params = [ - "git_commit_hash", - "git_repository_branch", - "git_repository_url", - "origin" - ] - - errorResponse = validate_required_params(request, get_required_params) - - if errorResponse: - return errorResponse - - path = request.GET.get("path", "") - if path: - if not path.endswith("."): - path += "." - - result = ( - Tests.objects.filter( - build__checkout__git_commit_hash=commit, - build__checkout__git_repository_branch=branch, - build__checkout__git_repository_url=url, - build__checkout__origin=origin, - path__startswith=path, - ) - .annotate( - first_path_segment=Func( - F("path"), - Value("."), - Value(1), - function="SPLIT_PART", - output_field=TextField(), - ) - ) - .exclude( - Q(first_path_segment="boot") | Q(first_path_segment__startswith="boot.") - ) - .values("first_path_segment") - .annotate( - fail_tests=Sum(Case(When(status="FAIL", then=1), default=0)), - error_tests=Sum(Case(When(status="ERROR", then=1), default=0)), - miss_tests=Sum(Case(When(status="MISS", then=1), default=0)), - pass_tests=Sum(Case(When(status="PASS", then=1), default=0)), - done_tests=Sum(Case(When(status="DONE", then=1), default=0)), - skip_tests=Sum(Case(When(status="SKIP", then=1), default=0)), - null_tests=Sum(Case(When(status__isnull=True, then=1), default=0)), - total_tests=Count("id"), - path_group=F("first_path_segment"), - individual_tests=ArrayAgg( - Concat( - F("path"), - Value(", "), - F("status"), - Value(", "), - F("start_time"), - Value(", "), - F("duration"), - output_field=TextField(), - ), - distinct=True, - ), - ) - .order_by("first_path_segment") - ) - - for item in result: - item["individual_tests"] = [ - { - "path": test[0], - "status": test[1], - "start_time": test[2], - "duration": test[3], - } - for test_info in item["individual_tests"] - if (test := test_info.split(", ")) - ] - - serializer = GroupedTestsSerializer(result, many=True) - return JsonResponse(serializer.data, safe=False) diff --git a/backend/kernelCI_app/views/treeTestsView.py b/backend/kernelCI_app/views/treeTestsView.py deleted file mode 100644 index 0d61d10d..00000000 --- a/backend/kernelCI_app/views/treeTestsView.py +++ /dev/null @@ -1,146 +0,0 @@ -from collections import defaultdict -from django.http import HttpResponseBadRequest, JsonResponse -from django.views import View -from kernelCI_app.utils import (FilterParams, InvalidComparisonOP, - extract_error_message, extract_platform, getErrorResponseBody) -from kernelCI_app.models import Checkouts - - -class TreeTestsView(View): - # TODO misc_environment is not stable and should be used as a POC only - # use the standardized field when that gets available - valid_filter_fields = ['status', 'duration'] - - def __paramsFromBootAndFilters(self, commit_hash, path_param, filter_params, params): - if path_param: - # TODO: 'boot' and 'boot.', right now is only using 'boot.' - path_filter = "AND t.path LIKE %s" - params.append(f'{path_param}%') - else: - path_filter = "" - - for f in filter_params.filters: - field = f['field'] - if field not in self.valid_filter_fields: - continue - value = f['value'] - value_is_list = isinstance(value, list) - if value_is_list: - placeholder = ",".join( - ['%s' for i in value]) if value_is_list else '%s' - path_filter += f" AND t.{field} IN ({placeholder})" - params.extend(value) - else: - comparison_op = filter_params.get_comparison_op(f, 'raw') - path_filter += f" AND t.{field} {comparison_op} %s" - params.append(value) - - return path_filter, params - - def get(self, request, commit_hash: str | None): - origin_param = request.GET.get("origin") - git_url_param = request.GET.get("git_url") - git_branch_param = request.GET.get("git_branch") - path_param = request.GET.get('path') - path_filter = '' - params = [commit_hash, origin_param, git_url_param, git_branch_param] - - try: - filter_params = FilterParams(request) - except InvalidComparisonOP as e: - return HttpResponseBadRequest(getErrorResponseBody(str(e))) - - print("####", params, path_filter) - path_filter, params = self.__paramsFromBootAndFilters( - commit_hash, path_param, filter_params, params) - print("@@@@", params, path_filter) - - names_map = { - "c.id": "id", - "c.git_repository_url": "git_repository_url", - "c.git_commit_hash": "git_commit_hash", - "t.build_id": "build_id", - "t.start_time": "start_time", - "t.status": "status", - "t.path": "path", - "b.architecture": "architecture", - "b.config_name": "config_name", - "b.compiler": "compiler", - "t.environment_misc": "environment_misc", - "t.environment_comment": "environment_comment", - "t.misc": "misc", - } - - # TODO - Remove the f string here and use parametrized queries - query = Checkouts.objects.raw( - f""" - SELECT c.id, c.git_repository_url, - c.git_commit_hash, t.build_id, t.start_time, - t.status as status, t.path, b.architecture, b.config_name, - b.compiler, t.environment_misc, t.environment_comment, t.misc - FROM checkouts AS c - INNER JOIN builds AS b ON c.id = b.checkout_id - INNER JOIN tests AS t ON t.build_id = b.id - WHERE c.git_commit_hash = %s AND c.origin = %s AND c.git_repository_url = %s AND - c.git_repository_branch = %s {path_filter} - ORDER BY - build_id, - CASE t.status - WHEN 'FAIL' THEN 1 - WHEN 'ERROR' THEN 2 - WHEN 'MISS' THEN 3 - WHEN 'PASS' THEN 4 - ELSE 4 - END, - t.id; - """, - params, - translations=names_map, - ) - - statusCounts = defaultdict(int) - errorCounts = defaultdict(int) - configStatusCounts = defaultdict(lambda: defaultdict(int)) - architectureStatusCounts = defaultdict(lambda: defaultdict(int)) - testHistory = [] - platformsWithError = set() - compilersPerArchitecture = defaultdict(set) - errorMessageCounts = defaultdict(int) - for record in query: - statusCounts[record.status] += 1 - configStatusCounts[record.config_name][record.status] += 1 - architectureStatusCounts[record.architecture][record.status] += 1 - testHistory.append( - {"start_time": record.start_time, "status": record.status} - ) - if ( - record.status == "MISS" - or record.status == "ERROR" - or record.status == "FAIL" - ): - errorCounts[record.status] += 1 - compilersPerArchitecture[record.architecture].add( - record.compiler) - platformsWithError.add( - extract_platform(record.environment_misc)) - currentErrorMessage = extract_error_message(record.misc) - errorMessageCounts[currentErrorMessage] += 1 - errorMessageCounts[currentErrorMessage] += 1 - for architecture in compilersPerArchitecture: - compilersPerArchitecture[architecture] = list( - compilersPerArchitecture[architecture] - ) - - # TODO Validate output - return JsonResponse( - { - "statusCounts": statusCounts, - "errorCounts": errorCounts, - "configStatusCounts": configStatusCounts, - "testHistory": testHistory, - "architectureStatusCounts": architectureStatusCounts, - "compilersPerArchitecture": compilersPerArchitecture, - "platformsWithError": list(platformsWithError), - "errorMessageCounts": errorMessageCounts, - } - ) diff --git a/backend/requests/boots-get.sh b/backend/requests/boots-get.sh deleted file mode 100644 index 673abad5..00000000 --- a/backend/requests/boots-get.sh +++ /dev/null @@ -1,80 +0,0 @@ -# Database tested - `playground_kcidb` -# This should be empty -# http 'http://localhost:8000/api/tree/a-git-commit-hash-that-clearly-does-not-exist/boot/' -# This should find something -http 'http://localhost:8000/api/tree/1dd28064d4164a4dc9096fd1a7990d2de15f2bb6/tests/' - -# { -# "statusCounts": { -# "FAIL": 2, -# "MISS": 1, -# "PASS": 1, -# "ERROR": 1 -# }, -# "errorCounts": { -# "FAIL": 2, -# "MISS": 1, -# "ERROR": 1 -# }, -# "configStatusCounts": { -# "defconfig": { -# "FAIL": 2, -# "PASS": 1 -# }, -# "multi_v7_defconfig": { -# "MISS": 1 -# }, -# "null": { -# "ERROR": 1 -# } -# }, -# "testHistory": [ -# { -# "start_time": "2024-07-06T00:58:20.149Z", -# "status": "FAIL" -# }, -# { -# "start_time": "2024-07-06T00:48:55.924Z", -# "status": "MISS" -# }, -# { -# "start_time": "2024-07-06T01:11:10.816Z", -# "status": "PASS" -# }, -# { -# "start_time": "2024-07-06T00:56:56.659Z", -# "status": "FAIL" -# }, -# { -# "start_time": "2024-07-06T00:26:44.970Z", -# "status": "ERROR" -# } -# ], -# "errorCountPerArchitecture": { -# "arm64": 2, -# "arm": 1, -# "null": 1 -# }, -# "compilersPerArchitecture": { -# "arm64": [ -# "gcc-12" -# ], -# "arm": [ -# "gcc-12" -# ], -# "null": [ -# null -# ] -# }, -# "platformsWithError": [ -# "mt8183-kukui-jacuzzi-juniper-sku16", -# "kubernetes", -# "bcm2711-rpi-4-b", -# "bcm2836-rpi-2-b" -# ], -# "errorMessageCounts": { -# "unknown error": 4, -# "No time left for remaining 1 retries. 2 retries out of 3 failed for auto-login-action": 2, -# "Node timed-out": 2 -# } -# } \ No newline at end of file diff --git a/backend/requests/grouped-tests.sh b/backend/requests/grouped-tests.sh deleted file mode 100755 index 3a651652..00000000 --- a/backend/requests/grouped-tests.sh +++ /dev/null @@ -1 +0,0 @@ -http 'http://localhost:8000/api/tree/tests' git_commit_hash==0ec2cf1e20adc2c8dcc5f58f3ebd40111c280944 origin==maestro git_repository_url==https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git git_repository_branch==linux-6.6.y diff --git a/backend/requests/revision-tests-get.sh b/backend/requests/revision-tests-get.sh deleted file mode 100644 index 42d949b7..00000000 --- a/backend/requests/revision-tests-get.sh +++ /dev/null @@ -1,33 +0,0 @@ -http 'http://localhost:8000/api/tree/tests/?git_commit_hash=001f8443d480773117a013a07f774d252f369bea' - -# { -# "current_path": "kernelci_kver", -# "start_time": "2024-08-08T21:46:15.254000Z", -# "fail_tests": 0, -# "error_tests": 0, -# "miss_tests": 0, -# "pass_tests": 1, -# "done_tests": 0, -# "skip_tests": 0, -# "null_tests": 0, -# "total_tests": 1, -# "origins": [ -# "maestro" -# ] -# }, -# { -# "current_path": "kunit", -# "start_time": "2024-08-08T21:46:24.509000Z", -# "fail_tests": 0, -# "error_tests": 0, -# "miss_tests": 0, -# "pass_tests": 327, -# "done_tests": 0, -# "skip_tests": 44, -# "null_tests": 0, -# "total_tests": 371, -# "origins": [ -# "maestro" -# ] -# } -# ] diff --git a/dashboard/src/api/TreeDetails.tsx b/dashboard/src/api/TreeDetails.tsx index e0032b85..80bed968 100644 --- a/dashboard/src/api/TreeDetails.tsx +++ b/dashboard/src/api/TreeDetails.tsx @@ -13,8 +13,6 @@ import type { LogFilesResponse, } from '@/types/tree/TreeDetails'; -import type { TPathTests } from '@/types/general'; - import { getTargetFilter } from '@/utils/filters'; import http from './api'; @@ -197,50 +195,6 @@ export const useTestsByTreeAndCommitHash = ( }); }; -type TTreeData = { - git_commit_hash?: string; - patchset?: string; - path?: string; - git_repository_url: string; - git_repository_branch: string; - origin: string; -}; - -const fetchTreeTestData = async (params?: TTreeData): Promise => { - const res = await http.get(`/api/tree/tests/`, { - params, - }); - - return res.data; -}; - -export const useTreeTest = ( - git_commit_hash: TTreeData['git_commit_hash'], - path: TTreeData['path'], - git_repository_branch: TTreeData['git_repository_branch'], - git_repository_url: TTreeData['git_repository_url'], - origin: TTreeData['origin'], -): UseQueryResult => { - return useQuery({ - queryKey: [ - 'treeGroupedTests', - git_commit_hash, - path, - git_repository_url, - git_repository_branch, - origin, - ], - queryFn: () => - fetchTreeTestData({ - git_commit_hash, - path, - git_repository_url, - git_repository_branch, - origin, - }), - }); -}; - const fetchTreeCommitHistory = async ( commitHash: string, origin: string, diff --git a/dashboard/src/routeTree.gen.ts b/dashboard/src/routeTree.gen.ts index b27b2a6a..5de41adf 100644 --- a/dashboard/src/routeTree.gen.ts +++ b/dashboard/src/routeTree.gen.ts @@ -20,7 +20,6 @@ import { Route as TreeTreeIdRouteImport } from './routes/tree/$treeId/route' import { Route as HardwareHardwareIdRouteImport } from './routes/hardware/$hardwareId/route' import { Route as TreeTreeIdIndexImport } from './routes/tree/$treeId/index' import { Route as HardwareHardwareIdIndexImport } from './routes/hardware/$hardwareId/index' -import { Route as TreeTreeIdTestRouteImport } from './routes/tree/$treeId/test/route' import { Route as HardwareHardwareIdTestRouteImport } from './routes/hardware/$hardwareId/test/route' import { Route as HardwareHardwareIdBuildRouteImport } from './routes/hardware/$hardwareId/build/route' import { Route as HardwareHardwareIdBootRouteImport } from './routes/hardware/$hardwareId/boot/route' @@ -81,11 +80,6 @@ const HardwareHardwareIdIndexRoute = HardwareHardwareIdIndexImport.update({ getParentRoute: () => HardwareHardwareIdRouteRoute, } as any) -const TreeTreeIdTestRouteRoute = TreeTreeIdTestRouteImport.update({ - path: '/test', - getParentRoute: () => TreeTreeIdRouteRoute, -} as any) - const HardwareHardwareIdTestRouteRoute = HardwareHardwareIdTestRouteImport.update({ path: '/test', @@ -123,8 +117,8 @@ const HardwareHardwareIdBootIndexRoute = } as any) const TreeTreeIdTestTestIdRouteRoute = TreeTreeIdTestTestIdRouteImport.update({ - path: '/$testId', - getParentRoute: () => TreeTreeIdTestRouteRoute, + path: '/test/$testId', + getParentRoute: () => TreeTreeIdRouteRoute, } as any) const TreeTreeIdTestTestIdIndexRoute = TreeTreeIdTestTestIdIndexImport.update({ @@ -230,13 +224,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof HardwareHardwareIdTestRouteImport parentRoute: typeof HardwareHardwareIdRouteImport } - '/tree/$treeId/test': { - id: '/tree/$treeId/test' - path: '/test' - fullPath: '/tree/$treeId/test' - preLoaderRoute: typeof TreeTreeIdTestRouteImport - parentRoute: typeof TreeTreeIdRouteImport - } '/hardware/$hardwareId/': { id: '/hardware/$hardwareId/' path: '/' @@ -253,10 +240,10 @@ declare module '@tanstack/react-router' { } '/tree/$treeId/test/$testId': { id: '/tree/$treeId/test/$testId' - path: '/$testId' + path: '/test/$testId' fullPath: '/tree/$treeId/test/$testId' preLoaderRoute: typeof TreeTreeIdTestTestIdRouteImport - parentRoute: typeof TreeTreeIdTestRouteImport + parentRoute: typeof TreeTreeIdRouteImport } '/hardware/$hardwareId/boot/': { id: '/hardware/$hardwareId/boot/' @@ -344,13 +331,11 @@ export const routeTree = rootRoute.addChildren({ }), TreeRouteRoute: TreeRouteRoute.addChildren({ TreeTreeIdRouteRoute: TreeTreeIdRouteRoute.addChildren({ - TreeTreeIdTestRouteRoute: TreeTreeIdTestRouteRoute.addChildren({ - TreeTreeIdTestTestIdRouteRoute: - TreeTreeIdTestTestIdRouteRoute.addChildren({ - TreeTreeIdTestTestIdIndexRoute, - }), - }), TreeTreeIdIndexRoute, + TreeTreeIdTestTestIdRouteRoute: + TreeTreeIdTestTestIdRouteRoute.addChildren({ + TreeTreeIdTestTestIdIndexRoute, + }), TreeTreeIdBuildBuildIdIndexRoute, }), TreeIndexRoute, @@ -401,8 +386,8 @@ export const routeTree = rootRoute.addChildren({ "filePath": "tree/$treeId/route.tsx", "parent": "/tree", "children": [ - "/tree/$treeId/test", "/tree/$treeId/", + "/tree/$treeId/test/$testId", "/tree/$treeId/build/$buildId/" ] }, @@ -438,13 +423,6 @@ export const routeTree = rootRoute.addChildren({ "/hardware/$hardwareId/test/$testId/" ] }, - "/tree/$treeId/test": { - "filePath": "tree/$treeId/test/route.tsx", - "parent": "/tree/$treeId", - "children": [ - "/tree/$treeId/test/$testId" - ] - }, "/hardware/$hardwareId/": { "filePath": "hardware/$hardwareId/index.tsx", "parent": "/hardware/$hardwareId" @@ -455,7 +433,7 @@ export const routeTree = rootRoute.addChildren({ }, "/tree/$treeId/test/$testId": { "filePath": "tree/$treeId/test/$testId/route.tsx", - "parent": "/tree/$treeId/test", + "parent": "/tree/$treeId", "children": [ "/tree/$treeId/test/$testId/" ] diff --git a/dashboard/src/routes/tree/$treeId/test/route.tsx b/dashboard/src/routes/tree/$treeId/test/route.tsx deleted file mode 100644 index 5ca5b735..00000000 --- a/dashboard/src/routes/tree/$treeId/test/route.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import { createFileRoute } from '@tanstack/react-router'; - -export const Route = createFileRoute('/tree/$treeId/test')();