Merge pull request #9 from EVOLVED-5G/testcaseKPIs

Testcase KPIs

CHANGELOG.md

@@ -1,3 +1,8 @@
+**09/11/2022** [Version 3.6.1]
+ - Allow defining a set of KPIs per TestCase
+ - Implement `/execution/<id>/kpis` endpoint
+ - Avoid exception when sending CSVs to InfluxDb on Python 3.11
+
 **10/10/2022** [Version 3.6.0]
 
  - Implemented Child tasks, flow control:

Facility/Loader/testcase_loader.py

@@ -18,12 +18,14 @@ def __init__(self, data: Dict):
         # V2 only
         self.Name: (str | None) = data.pop('Name', None)
         self.Sequence: List[Dict] = data.pop('Sequence', [])
+        self.KPIs: Dict[str, List[str]] = data.pop('KPIs', {})
 
 
 class TestCaseLoader(Loader):
     testCases: Dict[str, List[ActionInformation]] = {}
     extra: Dict[str, Dict[str, object]] = {}
     dashboards: Dict[str, List[DashboardPanel]] = {}
+    kpis: Dict[str, List[Tuple[str, str]]] = {}
     parameters: Dict[str, Tuple[str, str]] = {}  # For use only while processing data, not necessary afterwards
 
     @classmethod
@@ -85,6 +87,29 @@ def validateParameters(cls, defs: TestCaseData) -> [(Level, str)]:
                                         f"Cannot guarantee consistency."))
         return validation
 
+    @classmethod
+    def validateKPIs(cls, key: str, defs: TestCaseData) -> [(Level, str)]:
+        kpis = []
+        validation = []
+
+        try:
+            for measurement in sorted(defs.KPIs.keys()):
+                kpiList = defs.KPIs[measurement]
+                if not isinstance(kpiList, List):
+                    validation.append(
+                        (Level.ERROR, f"KPIs for '{measurement}' ({key}) are not a list. Found '{kpiList}'"))
+                elif len(kpiList) == 0:
+                    validation.append(
+                        (Level.ERROR, f"'{measurement}' ({key}) defines an empty listf of KPIs"))
+                else:
+                    for kpi in sorted(kpiList):
+                        kpis.append((measurement, kpi))
+        except Exception as e:
+            validation.append((Level.ERROR, f"Could not read KPIs dictionary for testcase '{key}': {e}"))
+
+        cls.kpis[key] = kpis
+        return validation
+
     @classmethod
     def ProcessData(cls, data: Dict) -> [(Level, str)]:
         version = str(data.pop('Version', 1))
@@ -140,6 +165,9 @@ def processV2Data(cls, data: Dict) -> [(Level, str)]:
         validation.extend(
             cls.createDashboard(defs.Name, defs))
 
+        validation.extend(
+            cls.validateKPIs(defs.Name, defs))
+
         validation.extend(
             cls.validateParameters(defs))
 
@@ -150,6 +178,7 @@ def Clear(cls):
         cls.testCases = {}
         cls.extra = {}
         cls.dashboards = {}
+        cls.kpis = {}
         cls.parameters = {}
 
     @classmethod
@@ -160,6 +189,10 @@ def GetCurrentTestCases(cls):
     def GetCurrentTestCaseExtras(cls):
         return cls.extra
 
+    @classmethod
+    def GetCurrentTestCaseKPIs(cls):
+        return cls.kpis
+
     @classmethod
     def GetCurrentDashboards(cls):
         return cls.dashboards

Facility/facility.py

@@ -24,6 +24,7 @@ class Facility:
     testCases: Dict[str, List[ActionInformation]] = {}
     extra: Dict[str, Dict[str, object]] = {}
     dashboards: Dict[str, List[DashboardPanel]] = {}
+    kpis: Dict[str, List[Tuple[str, str]]] = {}
     resources: Dict[str, Resource] = {}
     scenarios: Dict[str, Dict] = {}
 
@@ -63,6 +64,7 @@ def Reload(cls):
         cls.testCases = TestCaseLoader.GetCurrentTestCases()
         cls.extra = TestCaseLoader.GetCurrentTestCaseExtras()
         cls.dashboards = TestCaseLoader.GetCurrentDashboards()
+        cls.kpis = TestCaseLoader.GetCurrentTestCaseKPIs()
         cls.ues = UeLoader.GetCurrentUEs()
         cls.scenarios = ScenarioLoader.GetCurrentScenarios()
 
@@ -95,6 +97,10 @@ def GetTestCaseDashboards(cls, id: str) -> List[DashboardPanel]:
     def GetTestCaseExtra(cls, id: str) -> Dict[str, object]:
         return cls.extra.get(id, {})
 
+    @classmethod
+    def GetTestCaseKPIs(cls, id: str) -> List[Tuple[str, str]]:
+        return cls.kpis.get(id, [])
+
     @classmethod
     def BusyResources(cls) -> List[Resource]:
         return [res for res in cls.resources.values() if res.Locked]

Helper/influx.py

@@ -57,7 +57,7 @@ class baseDialect(Dialect):
     doublequote = False
     skipinitialspace = False
     lineterminator = '\r\n'
-    quotechar = ''
+    quotechar = '"'
     quoting = QUOTE_NONE
 
 

Scheduler/execution/routes.py

@@ -4,6 +4,8 @@
 from Scheduler.execution import bp
 from typing import Union, Optional
 from Settings import Config
+from Data import ExperimentDescriptor
+from Facility import Facility
 from os.path import join, isfile, abspath
 
 
@@ -126,6 +128,21 @@ def descriptor(executionId: int):
         return f"Execution {executionId} not found", 404
 
 
+@bp.route('<int:executionId>/kpis')
+def kpis(executionId: int):
+    execution = executionOrTombstone(executionId)
+
+    if execution is not None:
+        kpis = set()
+        descriptor = ExperimentDescriptor(execution.JsonDescriptor)
+        for testcase in descriptor.TestCases:
+            kpis.update(Facility.GetTestCaseKPIs(testcase))
+
+        return jsonify({"KPIs": sorted(kpis)})
+    else:
+        return f"Execution {executionId} not found", 404
+
+
 @bp.route('nextExecutionId')
 def nextExecutionId():
     return jsonify({'NextId': Status.PeekNextId()})

docs/2-1_FACILITY_CONFIGURATION.md

@@ -91,6 +91,9 @@ Distributed: False
 Dashboard: {}
 ````
 
+V2 TestCases can also contain an optional `KPIs` field, which can be used to define a sub-set of results that are
+considered *of interest*. See [TestCase Parameters](/docs/2-2_TESTCASE_PARAMETERS.md) for more information.
+
 ##### - V1 TestCase (`Version: 1` or missing)
 
 TestCases using this format follow the same approach as for UE files. The following is an example V1 TestCase:

docs/2-2_TESTCASE_PARAMETERS.md

@@ -8,6 +8,19 @@ be added to the yaml description. These keys are:
    value is set to an empty list ('[]') the test case is considered public and will appear on the list of Custom
    experiments for all users of the Portal. If the list contains one or more email addresses, the test case will be
    visible only to the users with matching emails.
+ - `KPIs`: Optional dictionary that can be used for defining a sub-set of results, from all of the generated by the
+TestCase, that can be considered *of interest*. The following is an example of the format:
+ ```yaml
+"KPIs":
+  ping:  # Each key refers to a 'measurement' (table)
+    - Success  # Each key contains a list of 'kpi' (table columns)
+    - Delay
+  iPerf:
+    - Throughput
+ ```
+These values (as (`measurement`, `kpi`) pairs) can be retrieved, per experiment execution, by using the
+`execution/<id>/kpis` endpoint. When multiple TestCases are included as part of an experiment the union
+of all KPIs (duplicates removed) are returned.
  - `Parameters`: Dictionary of dictionaries, where each entry is defined as follows:
  ```yaml
 "<Parameter Name>":

docs/A1_ENDPOINTS.md

@@ -46,6 +46,14 @@ Returns a compressed file that includes the logs and all files generated by the
 
 Returns a copy of the Experiment Descriptor that was used to define the execution.
 
+### [GET] '/execution/<id>/kpis'
+
+Returns a dictionary with a single `KPIs` key, containing a list of pairs (`measurement`, `kpi`) that are considered of
+interest.
+
+> These values can be used as part of queries to the [Analytics Module](https://github.com/5genesis/Analytics), in order
+> to extract a sub-set of important KPIs from all the generated measurements.
+
 ### [DELETE] `/execution/<id>`
 > *[GET] `/execution/<id>/cancel` (Deprecated)*