diff --git a/CHANGELOG.md b/CHANGELOG.md index bf857ac..58cad6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +**09/11/2022** [Version 3.6.1] + - Allow defining a set of KPIs per TestCase + - Implement `/execution//kpis` endpoint + - Avoid exception when sending CSVs to InfluxDb on Python 3.11 + **10/10/2022** [Version 3.6.0] - Implemented Child tasks, flow control: diff --git a/Facility/Loader/testcase_loader.py b/Facility/Loader/testcase_loader.py index 960cdeb..b65e16c 100644 --- a/Facility/Loader/testcase_loader.py +++ b/Facility/Loader/testcase_loader.py @@ -18,12 +18,14 @@ def __init__(self, data: Dict): # V2 only self.Name: (str | None) = data.pop('Name', None) self.Sequence: List[Dict] = data.pop('Sequence', []) + self.KPIs: Dict[str, List[str]] = data.pop('KPIs', {}) class TestCaseLoader(Loader): testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} dashboards: Dict[str, List[DashboardPanel]] = {} + kpis: Dict[str, List[Tuple[str, str]]] = {} parameters: Dict[str, Tuple[str, str]] = {} # For use only while processing data, not necessary afterwards @classmethod @@ -85,6 +87,29 @@ def validateParameters(cls, defs: TestCaseData) -> [(Level, str)]: f"Cannot guarantee consistency.")) return validation + @classmethod + def validateKPIs(cls, key: str, defs: TestCaseData) -> [(Level, str)]: + kpis = [] + validation = [] + + try: + for measurement in sorted(defs.KPIs.keys()): + kpiList = defs.KPIs[measurement] + if not isinstance(kpiList, List): + validation.append( + (Level.ERROR, f"KPIs for '{measurement}' ({key}) are not a list. Found '{kpiList}'")) + elif len(kpiList) == 0: + validation.append( + (Level.ERROR, f"'{measurement}' ({key}) defines an empty listf of KPIs")) + else: + for kpi in sorted(kpiList): + kpis.append((measurement, kpi)) + except Exception as e: + validation.append((Level.ERROR, f"Could not read KPIs dictionary for testcase '{key}': {e}")) + + cls.kpis[key] = kpis + return validation + @classmethod def ProcessData(cls, data: Dict) -> [(Level, str)]: version = str(data.pop('Version', 1)) @@ -140,6 +165,9 @@ def processV2Data(cls, data: Dict) -> [(Level, str)]: validation.extend( cls.createDashboard(defs.Name, defs)) + validation.extend( + cls.validateKPIs(defs.Name, defs)) + validation.extend( cls.validateParameters(defs)) @@ -150,6 +178,7 @@ def Clear(cls): cls.testCases = {} cls.extra = {} cls.dashboards = {} + cls.kpis = {} cls.parameters = {} @classmethod @@ -160,6 +189,10 @@ def GetCurrentTestCases(cls): def GetCurrentTestCaseExtras(cls): return cls.extra + @classmethod + def GetCurrentTestCaseKPIs(cls): + return cls.kpis + @classmethod def GetCurrentDashboards(cls): return cls.dashboards diff --git a/Facility/facility.py b/Facility/facility.py index 1860745..72df8a4 100644 --- a/Facility/facility.py +++ b/Facility/facility.py @@ -24,6 +24,7 @@ class Facility: testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} dashboards: Dict[str, List[DashboardPanel]] = {} + kpis: Dict[str, List[Tuple[str, str]]] = {} resources: Dict[str, Resource] = {} scenarios: Dict[str, Dict] = {} @@ -63,6 +64,7 @@ def Reload(cls): cls.testCases = TestCaseLoader.GetCurrentTestCases() cls.extra = TestCaseLoader.GetCurrentTestCaseExtras() cls.dashboards = TestCaseLoader.GetCurrentDashboards() + cls.kpis = TestCaseLoader.GetCurrentTestCaseKPIs() cls.ues = UeLoader.GetCurrentUEs() cls.scenarios = ScenarioLoader.GetCurrentScenarios() @@ -95,6 +97,10 @@ def GetTestCaseDashboards(cls, id: str) -> List[DashboardPanel]: def GetTestCaseExtra(cls, id: str) -> Dict[str, object]: return cls.extra.get(id, {}) + @classmethod + def GetTestCaseKPIs(cls, id: str) -> List[Tuple[str, str]]: + return cls.kpis.get(id, []) + @classmethod def BusyResources(cls) -> List[Resource]: return [res for res in cls.resources.values() if res.Locked] diff --git a/Helper/influx.py b/Helper/influx.py index 7ae495a..1caaa2f 100644 --- a/Helper/influx.py +++ b/Helper/influx.py @@ -57,7 +57,7 @@ class baseDialect(Dialect): doublequote = False skipinitialspace = False lineterminator = '\r\n' - quotechar = '' + quotechar = '"' quoting = QUOTE_NONE diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index 6a653c7..82b4f27 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -4,6 +4,8 @@ from Scheduler.execution import bp from typing import Union, Optional from Settings import Config +from Data import ExperimentDescriptor +from Facility import Facility from os.path import join, isfile, abspath @@ -126,6 +128,21 @@ def descriptor(executionId: int): return f"Execution {executionId} not found", 404 +@bp.route('/kpis') +def kpis(executionId: int): + execution = executionOrTombstone(executionId) + + if execution is not None: + kpis = set() + descriptor = ExperimentDescriptor(execution.JsonDescriptor) + for testcase in descriptor.TestCases: + kpis.update(Facility.GetTestCaseKPIs(testcase)) + + return jsonify({"KPIs": sorted(kpis)}) + else: + return f"Execution {executionId} not found", 404 + + @bp.route('nextExecutionId') def nextExecutionId(): return jsonify({'NextId': Status.PeekNextId()}) diff --git a/docs/2-1_FACILITY_CONFIGURATION.md b/docs/2-1_FACILITY_CONFIGURATION.md index 895eab7..cf64263 100644 --- a/docs/2-1_FACILITY_CONFIGURATION.md +++ b/docs/2-1_FACILITY_CONFIGURATION.md @@ -91,6 +91,9 @@ Distributed: False Dashboard: {} ```` +V2 TestCases can also contain an optional `KPIs` field, which can be used to define a sub-set of results that are +considered *of interest*. See [TestCase Parameters](/docs/2-2_TESTCASE_PARAMETERS.md) for more information. + ##### - V1 TestCase (`Version: 1` or missing) TestCases using this format follow the same approach as for UE files. The following is an example V1 TestCase: diff --git a/docs/2-2_TESTCASE_PARAMETERS.md b/docs/2-2_TESTCASE_PARAMETERS.md index 3de5b7a..37c84e3 100644 --- a/docs/2-2_TESTCASE_PARAMETERS.md +++ b/docs/2-2_TESTCASE_PARAMETERS.md @@ -8,6 +8,19 @@ be added to the yaml description. These keys are: value is set to an empty list ('[]') the test case is considered public and will appear on the list of Custom experiments for all users of the Portal. If the list contains one or more email addresses, the test case will be visible only to the users with matching emails. + - `KPIs`: Optional dictionary that can be used for defining a sub-set of results, from all of the generated by the +TestCase, that can be considered *of interest*. The following is an example of the format: + ```yaml +"KPIs": + ping: # Each key refers to a 'measurement' (table) + - Success # Each key contains a list of 'kpi' (table columns) + - Delay + iPerf: + - Throughput + ``` +These values (as (`measurement`, `kpi`) pairs) can be retrieved, per experiment execution, by using the +`execution//kpis` endpoint. When multiple TestCases are included as part of an experiment the union +of all KPIs (duplicates removed) are returned. - `Parameters`: Dictionary of dictionaries, where each entry is defined as follows: ```yaml "": diff --git a/docs/A1_ENDPOINTS.md b/docs/A1_ENDPOINTS.md index 043dd12..3563e63 100644 --- a/docs/A1_ENDPOINTS.md +++ b/docs/A1_ENDPOINTS.md @@ -46,6 +46,14 @@ Returns a compressed file that includes the logs and all files generated by the Returns a copy of the Experiment Descriptor that was used to define the execution. +### [GET] '/execution//kpis' + +Returns a dictionary with a single `KPIs` key, containing a list of pairs (`measurement`, `kpi`) that are considered of +interest. + +> These values can be used as part of queries to the [Analytics Module](https://github.com/5genesis/Analytics), in order +> to extract a sub-set of important KPIs from all the generated measurements. + ### [DELETE] `/execution/` > *[GET] `/execution//cancel` (Deprecated)*