diff --git a/CHANGELOG.md b/CHANGELOG.md index 18401c8..a51707d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +**01/04/2022** [Version 3.3.0] + + - TestCase definition Version 2 + **16/02/2022** [Version 3.2.2] - Enhanced PublishFromFile and PublishFromPreviousTaskLog: diff --git a/Facility/Loader/__init__.py b/Facility/Loader/__init__.py new file mode 100644 index 0000000..22cc184 --- /dev/null +++ b/Facility/Loader/__init__.py @@ -0,0 +1,5 @@ +from .loader_base import Loader +from .resource_loader import ResourceLoader +from .scenario_loader import ScenarioLoader +from .ue_loader import UeLoader +from .testcase_loader import TestCaseLoader diff --git a/Facility/Loader/loader_base.py b/Facility/Loader/loader_base.py new file mode 100644 index 0000000..69e53fa --- /dev/null +++ b/Facility/Loader/loader_base.py @@ -0,0 +1,77 @@ +import yaml +from Helper import Level +from typing import List, Dict +from os.path import join +from ..action_information import ActionInformation + + +class Loader: + @classmethod + def EnsureFolder(cls, path: str) -> [(Level, str)]: + from Helper import IO + validation = [] + if not IO.EnsureFolder(path): + validation.append((Level.INFO, f'Auto-generated folder: {path}')) + return validation + + @classmethod + def LoadFolder(cls, path: str, kind: str) -> [(Level, str)]: + from Helper import IO + ignored = [] + validation = [] + for file in IO.ListFiles(path): + if file.endswith('.yml'): + filePath = join(path, file) + try: + validation.append((Level.INFO, f'Loading {kind}: {file}')) + data, v = cls.LoadFile(filePath) + validation.extend(v) + validation.extend(cls.ProcessData(data)) + except Exception as e: + validation.append((Level.ERROR, f"Exception loading {kind} file '{filePath}': {e}")) + else: + ignored.append(file) + if len(ignored) != 0: + validation.append((Level.WARNING, + f'Ignored the following files on the {kind}s folder: {(", ".join(ignored))}')) + return validation + + @classmethod + def LoadFile(cls, path: str) -> ((Dict | None), [(Level, str)]): + try: + with open(path, 'r', encoding='utf-8') as file: + raw = yaml.safe_load(file) + return raw, [] + except Exception as e: + return None, [(Level.ERROR, f"Unable to load file '{path}': {e}")] + + @classmethod + def GetActionList(cls, data: List[Dict]) -> ([ActionInformation], [(Level, str)]): + actionList = [] + validation = [] + + for action in data: + actionInfo = ActionInformation.FromMapping(action) + if actionInfo is not None: + actionList.append(actionInfo) + else: + validation.append((Level.ERROR, f'Action not correctly defined for element (data="{action}").')) + actionList.append(ActionInformation.MessageAction( + 'ERROR', f'Incorrect Action (data="{action}")' + )) + + if len(actionList) == 0: + validation.append((Level.WARNING, 'No actions defined')) + else: + for action in actionList: + validation.append((Level.DEBUG, str(action))) + + return actionList, validation + + @classmethod + def ProcessData(cls, data: Dict) -> [(Level, str)]: + raise NotImplementedError + + @classmethod + def Clear(cls): + raise NotImplementedError diff --git a/Facility/Loader/resource_loader.py b/Facility/Loader/resource_loader.py new file mode 100644 index 0000000..3b1c9f4 --- /dev/null +++ b/Facility/Loader/resource_loader.py @@ -0,0 +1,27 @@ +from Helper import Level +from .loader_base import Loader +from ..resource import Resource +from typing import Dict + + +class ResourceLoader(Loader): + resources: Dict[str, Resource] = {} + + @classmethod + def ProcessData(cls, data: Dict) -> [(Level, str)]: + validation = [] + + resource = Resource(data) + if resource.Id in cls.resources.keys(): + validation.append((Level.WARNING, f'Redefining Resource {resource.Id}')) + cls.resources[resource.Id] = resource + + return validation + + @classmethod + def Clear(cls): + cls.resources = {} + + @classmethod + def GetCurrentResources(cls): + return cls.resources diff --git a/Facility/Loader/scenario_loader.py b/Facility/Loader/scenario_loader.py new file mode 100644 index 0000000..40b326c --- /dev/null +++ b/Facility/Loader/scenario_loader.py @@ -0,0 +1,31 @@ +from Helper import Level +from .loader_base import Loader +from typing import Dict + + +class ScenarioLoader(Loader): + scenarios: Dict[str, Dict] = {} + + @classmethod + def ProcessData(cls, data: Dict) -> [(Level, str)]: + validation = [] + keys = list(data.keys()) + + if len(keys) > 1: + validation.append((Level.WARNING, f'Multiple Scenarios defined on a single file: {keys}')) + + for key, value in data.items(): + if key in cls.scenarios.keys(): + validation.append((Level.WARNING, f'Redefining Scenario {key}')) + cls.scenarios[key] = value + validation.append((Level.DEBUG, f'{key}: {value}')) + + return validation + + @classmethod + def Clear(cls): + cls.scenarios = {} + + @classmethod + def GetCurrentScenarios(cls): + return cls.scenarios diff --git a/Facility/Loader/testcase_loader.py b/Facility/Loader/testcase_loader.py new file mode 100644 index 0000000..960cdeb --- /dev/null +++ b/Facility/Loader/testcase_loader.py @@ -0,0 +1,165 @@ +from Helper import Level +from .loader_base import Loader +from ..action_information import ActionInformation +from ..dashboard_panel import DashboardPanel +from typing import Dict, List, Tuple + + +class TestCaseData: + def __init__(self, data: Dict): + # Shared keys + self.AllKeys: List[str] = list(data.keys()) + self.Dashboard: (Dict | None) = data.pop('Dashboard', None) + self.Standard: (bool | None) = data.pop('Standard', None) + self.Custom: (List[str] | None) = data.pop('Custom', None) + self.Distributed: bool = data.pop('Distributed', False) + self.Parameters: Dict[str, Dict[str, str]] = data.pop('Parameters', {}) + + # V2 only + self.Name: (str | None) = data.pop('Name', None) + self.Sequence: List[Dict] = data.pop('Sequence', []) + + +class TestCaseLoader(Loader): + testCases: Dict[str, List[ActionInformation]] = {} + extra: Dict[str, Dict[str, object]] = {} + dashboards: Dict[str, List[DashboardPanel]] = {} + parameters: Dict[str, Tuple[str, str]] = {} # For use only while processing data, not necessary afterwards + + @classmethod + def getPanelList(cls, data: Dict) -> ([DashboardPanel], [(Level, str)]): + validation = [] + panelList = [] + for panel in data: + try: + parsedPanel = DashboardPanel(panel) + valid, error = parsedPanel.Validate() + if not valid: + validation.append((Level.ERROR, f'Could not validate panel (data={panel}) - {error}')) + else: + panelList.append(parsedPanel) + except Exception as e: + validation.append((Level.ERROR, f"Unable to parse Dashboard Panel (data={panel}), ignored. {e}")) + + validation.append((Level.DEBUG, f'Defined {len(panelList)} dashboard panels')) + return panelList, validation + + @classmethod + def handleExperimentType(cls, defs: TestCaseData) -> [(Level, str)]: + validation = [] + if defs.Standard is None: + defs.Standard = (defs.Custom is None) + validation.append((Level.WARNING, f'Standard not defined, assuming {defs.Standard}. Keys: {defs.AllKeys}')) + return validation + + @classmethod + def createDashboard(cls, key: str, defs: TestCaseData) -> [(Level, str)]: + validation = [] + if defs.Dashboard is not None: + cls.dashboards[key], validation = cls.getPanelList(defs.Dashboard) + return validation + + @classmethod + def createExtra(cls, key: str, defs: TestCaseData): + cls.extra[key] = { + 'Standard': defs.Standard, + 'PublicCustom': (defs.Custom is not None and len(defs.Custom) == 0), + 'PrivateCustom': defs.Custom if defs.Custom is not None else [], + 'Parameters': defs.Parameters, + 'Distributed': defs.Distributed + } + + @classmethod + def validateParameters(cls, defs: TestCaseData) -> [(Level, str)]: + validation = [] + for name, info in defs.Parameters.items(): + type, desc = (info['Type'], info['Description']) + if name not in cls.parameters.keys(): + cls.parameters[name] = (type, desc) + else: + oldType, oldDesc = cls.parameters[name] + if type != oldType or desc != oldDesc: + validation.append( + (Level.WARNING, f"Redefined parameter '{name}' with different settings: " + f"'{oldType}' - '{type}'; '{oldDesc}' - '{desc}'. " + f"Cannot guarantee consistency.")) + return validation + + @classmethod + def ProcessData(cls, data: Dict) -> [(Level, str)]: + version = str(data.pop('Version', 1)) + + match version: + case '1': return cls.processV1Data(data) + case '2': return cls.processV2Data(data) + case _: raise RuntimeError(f"Unknown testcase version '{version}'.") + + @classmethod + def processV1Data(cls, data: Dict) -> [(Level, str)]: + validation = [] + defs = TestCaseData(data) + + if defs.Dashboard is None: + validation.append((Level.WARNING, f'Dashboard not defined. Keys: {defs.AllKeys}')) + + validation.extend( + cls.handleExperimentType(defs)) + + keys = list(data.keys()) + + if len(keys) > 1: + validation.append((Level.ERROR, f'Multiple TestCases defined on a single file: {list(keys)}')) + + for key in keys: + cls.testCases[key], v = cls.GetActionList(data[key]) + validation.extend(v) + + cls.createExtra(key, defs) + + validation.extend( + cls.createDashboard(key, defs)) + + validation.extend( + cls.validateParameters(defs)) + + return validation + + @classmethod + def processV2Data(cls, data: Dict) -> [(Level, str)]: + validation = [] + defs = TestCaseData(data) + + validation.extend( + cls.handleExperimentType(defs)) + + cls.testCases[defs.Name], v = cls.GetActionList(defs.Sequence) + validation.extend(v) + + cls.createExtra(defs.Name, defs) + + validation.extend( + cls.createDashboard(defs.Name, defs)) + + validation.extend( + cls.validateParameters(defs)) + + return validation + + @classmethod + def Clear(cls): + cls.testCases = {} + cls.extra = {} + cls.dashboards = {} + cls.parameters = {} + + @classmethod + def GetCurrentTestCases(cls): + return cls.testCases + + @classmethod + def GetCurrentTestCaseExtras(cls): + return cls.extra + + @classmethod + def GetCurrentDashboards(cls): + return cls.dashboards diff --git a/Facility/Loader/ue_loader.py b/Facility/Loader/ue_loader.py new file mode 100644 index 0000000..f83ee51 --- /dev/null +++ b/Facility/Loader/ue_loader.py @@ -0,0 +1,33 @@ +from Helper import Level +from .loader_base import Loader +from ..action_information import ActionInformation +from typing import Dict, List + + +class UeLoader(Loader): + ues: Dict[str, List[ActionInformation]] = {} + + @classmethod + def ProcessData(cls, data: Dict) -> [(Level, str)]: + validation = [] + keys = list(data.keys()) + + if len(keys) > 1: + validation.append((Level.WARNING, f'Multiple UEs defined on a single file: {keys}')) + + for key in keys: + if key in cls.ues.keys(): + validation.append((Level.WARNING, f'Redefining UE {key}')) + actions, v = cls.GetActionList(data[key]) + validation.extend(v) + cls.ues[key] = actions + + return validation + + @classmethod + def Clear(cls): + cls.ues = {} + + @classmethod + def GetCurrentUEs(cls): + return cls.ues diff --git a/Facility/facility.py b/Facility/facility.py index 85fcaac..1860745 100644 --- a/Facility/facility.py +++ b/Facility/facility.py @@ -1,12 +1,12 @@ -from os.path import abspath, join -import yaml +from os.path import abspath from .action_information import ActionInformation from .dashboard_panel import DashboardPanel from .resource import Resource from Helper import Log, Level -from typing import Dict, List, Tuple, Callable, Optional +from typing import Dict, List, Tuple, Optional from threading import Lock from Utils import synchronized +from .Loader import Loader, ResourceLoader, ScenarioLoader, UeLoader, TestCaseLoader class Facility: @@ -23,7 +23,7 @@ class Facility: ues: Dict[str, List[ActionInformation]] = {} testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} - dashboards: Dict[str, List[ActionInformation]] = {} + dashboards: Dict[str, List[DashboardPanel]] = {} resources: Dict[str, Resource] = {} scenarios: Dict[str, Dict] = {} @@ -31,194 +31,50 @@ class Facility: @classmethod def Reload(cls): - from Helper import IO - allParameters: Dict[str, Tuple[str, str]] = {} - - def _ensureFolder(path: str): - if not IO.EnsureFolder(path): - cls.Validation.append((Level.INFO, f'Auto-generated folder: {path}')) - - def _loadFolder(path: str, kind: str, callable: Callable): - ignored = [] - for file in IO.ListFiles(path): - if file.endswith('.yml'): - cls.Validation.append((Level.INFO, f'Loading {kind}: {file}')) - callable(join(path, file)) - else: - ignored.append(file) - if len(ignored) != 0: - cls.Validation.append((Level.WARNING, - f'Ignored the following files on the {kind}s folder: {(", ".join(ignored))}')) - - def _loadFile(path: str) -> Optional[Dict]: - try: - with open(path, 'r', encoding='utf-8') as file: - raw = yaml.safe_load(file) - return raw - except Exception as e: - cls.Validation.append((Level.ERROR, f"Unable to load file '{path}': {e}")) - return None - - def _get_ActionList(data: Dict) -> List[ActionInformation]: - actionList = [] - for action in data: - actionInfo = ActionInformation.FromMapping(action) - if actionInfo is not None: - actionList.append(actionInfo) - else: - cls.Validation.append((Level.ERROR, f'Action not correctly defined for element (data="{action}").')) - actionList.append(ActionInformation.MessageAction( - 'ERROR', f'Incorrect Action (data="{action}")' - )) - if len(actionList) == 0: - cls.Validation.append((Level.WARNING, 'No actions defined')) - else: - for action in actionList: - cls.Validation.append((Level.DEBUG, str(action))) - return actionList - - def _get_PanelList(data: Dict) -> List[DashboardPanel]: - panelList = [] - for panel in data: - try: - parsedPanel = DashboardPanel(panel) - valid, error = parsedPanel.Validate() - if not valid: - cls.Validation.append((Level.ERROR, f'Could not validate panel (data={panel}) - {error}')) - else: - panelList.append(parsedPanel) - except Exception as e: - cls.Validation.append((Level.ERROR, - f"Unable to parse Dashboard Panel (data={panel}), ignored. {e}")) - cls.Validation.append((Level.DEBUG, f'Defined {len(panelList)} dashboard panels')) - return panelList - - def _testcaseLoader(path: str): - try: - data = _loadFile(path) - - allKeys = list(data.keys()) - dashboard = data.pop('Dashboard', None) - standard = data.pop('Standard', None) - custom = data.pop('Custom', None) - distributed = data.pop('Distributed', False) - parameters = data.pop('Parameters', {}) - - if dashboard is None: - cls.Validation.append((Level.WARNING, f'Dashboard not defined. Keys: {allKeys}')) - - if standard is None: - standard = (custom is None) - cls.Validation.append((Level.WARNING, - f'Standard not defined, assuming {standard}. Keys: {allKeys}')) - keys = list(data.keys()) - - if len(keys) > 1: - cls.Validation.append((Level.ERROR, f'Multiple TestCases defined on a single file: {list(keys)}')) - for key in keys: - testCases[key] = _get_ActionList(data[key]) - - extra[key] = { - 'Standard': standard, - 'PublicCustom': (custom is not None and len(custom) == 0), - 'PrivateCustom': custom if custom is not None else [], - 'Parameters': parameters, - 'Distributed': distributed - } - - if dashboard is not None: - dashboards[key] = _get_PanelList(dashboard) - - for name, info in parameters.items(): - type, desc = (info['Type'], info['Description']) - if name not in allParameters.keys(): - allParameters[name] = (type, desc) - else: - oldType, oldDesc = allParameters[name] - if type != oldType or desc != oldDesc: - cls.Validation.append( - (Level.WARNING, f"Redefined parameter '{name}' with different settings: " - f"'{oldType}' - '{type}'; '{oldDesc}' - '{desc}'. " - f"Cannot guarantee consistency.")) - except Exception as e: - cls.Validation.append((Level.ERROR, f'Exception loading TestCase file {path}: {e}')) - - def _ueLoader(path: str): - try: - data = _loadFile(path) - keys = data.keys() - if len(keys) > 1: - cls.Validation.append((Level.WARNING, f'Multiple UEs defined on a single file: {list(keys)}')) - for key in keys: - if key in ues.keys(): - cls.Validation.append((Level.WARNING, f'Redefining UE {key}')) - actions = _get_ActionList(data[key]) - ues[key] = actions - except Exception as e: - cls.Validation.append((Level.ERROR, f'Exception loading UE file {path}: {e}')) - - def _resourceLoader(path: str): - try: - data = _loadFile(path) - resource = Resource(data) - if resource.Id in resources.keys(): - cls.Validation.append((Level.WARNING, f'Redefining Resource {resource.Id}')) - resources[resource.Id] = resource - except Exception as e: - cls.Validation.append((Level.ERROR, f'Exception loading Resource file {path}: {e}')) - - def _scenarioLoader(path: str): - try: - data = _loadFile(path) - if len(data.keys()) > 1: - cls.Validation.append((Level.WARNING, f'Multiple Scenarios defined on a single file: {list(keys)}')) - for key, value in data.items(): - if key in scenarios.keys(): - cls.Validation.append((Level.WARNING, f'Redefining Scenario {key}')) - scenarios[key] = value - cls.Validation.append((Level.DEBUG, f'{key}: {value}')) - except Exception as e: - cls.Validation.append((Level.ERROR, f'Exception loading Resource file {path}: {e}')) - cls.Validation.clear() - # Generate all folders + # Generate missing folders for folder in [cls.TESTCASE_FOLDER, cls.UE_FOLDER, cls.RESOURCE_FOLDER, cls.SCENARIO_FOLDER]: - _ensureFolder(folder) - - testCases = {} - ues = {} - dashboards = {} - extra = {} - scenarios = {} + v = Loader.EnsureFolder(folder) + cls.Validation.extend(v) + resources = cls.resources if len(cls.BusyResources()) != 0: - resources = cls.resources cls.Validation.append((Level.WARNING, "Resources in use, skipping reload")) else: - resources = {} - _loadFolder(cls.RESOURCE_FOLDER, "Resource", _resourceLoader) + ResourceLoader.Clear() + v = ResourceLoader.LoadFolder(cls.RESOURCE_FOLDER, "Resource") + cls.Validation.extend(v) + resources = ResourceLoader.GetCurrentResources() + + TestCaseLoader.Clear() + v = TestCaseLoader.LoadFolder(cls.TESTCASE_FOLDER, "TestCase") + cls.Validation.extend(v) - _loadFolder(cls.TESTCASE_FOLDER, "TestCase", _testcaseLoader) - _loadFolder(cls.UE_FOLDER, "UE", _ueLoader) - _loadFolder(cls.SCENARIO_FOLDER, "Scenario", _scenarioLoader) + UeLoader.Clear() + v = UeLoader.LoadFolder(cls.UE_FOLDER, "UE") + cls.Validation.extend(v) - for collection, name in [(testCases, "TestCases"), (ues, "UEs"), - (dashboards, "DashBoards"), (resources, "Resources"), - (scenarios, "Scenarios")]: + ScenarioLoader.Clear() + v = ScenarioLoader.LoadFolder(cls.SCENARIO_FOLDER, "Scenario") + cls.Validation.extend(v) + + cls.resources = resources + cls.testCases = TestCaseLoader.GetCurrentTestCases() + cls.extra = TestCaseLoader.GetCurrentTestCaseExtras() + cls.dashboards = TestCaseLoader.GetCurrentDashboards() + cls.ues = UeLoader.GetCurrentUEs() + cls.scenarios = ScenarioLoader.GetCurrentScenarios() + + for collection, name in [(cls.testCases, "TestCases"), (cls.ues, "UEs"), + (cls.dashboards, "DashBoards"), (cls.resources, "Resources"), + (cls.scenarios, "Scenarios")]: keys = collection.keys() if len(keys) == 0: cls.Validation.append((Level.WARNING, f'No {name} defined on the facility.')) else: cls.Validation.append((Level.INFO, f'{len(keys)} {name} defined on the facility: {(", ".join(keys))}.')) - cls.ues = ues - cls.testCases = testCases - cls.extra = extra - cls.dashboards = dashboards - cls.resources = resources - cls.scenarios = scenarios - @classmethod def GetUEActions(cls, id: str) -> List[ActionInformation]: return cls.ues.get(id, []) diff --git a/README.md b/README.md index e8600b2..7383236 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,39 @@ be seen in the `Available Tasks` section below. #### TestCases Similarly to the UEs, the files in the ´TestCases´ folder define the actions required in order to execute a certain -test case. The following is an example TestCase file: +test case. The exact format of the TestCase depends on the specific version used, though most of the fields are shared. +New TestCases should be defined using the latest format available, which would support the most complete feature set. +However, for convenience, compatibility with older versions will be retained for as long as possible. + +The version of the TestCase file is selected by setting the `Version` field to the corresponding value. If this field +is not present, the file will be processed as a V1 TestCase. + +###### - V2 TestCase (`Version: 2`) + +TestCases using this format explicitly specify all the information in separate fields. This means that all keys in the +root must have one of a set of specific values. This means that, instead of specifying the name of the TestCase as the +key value of the sequence of actions, it is set by using the `Name` field. It is not possible to define multiple +TestCases in a single file. The following is an example V2 TestCase: + +````yaml +Version: 2 +Name: Slice Creation +Sequence: + - Order: 5 + Task: Run.SingleSliceCreationTime + Config: + ExperimentId: "@{ExperimentId}" + WaitForRunning: True + Timeout: 60 + SliceId: "@{SliceId}" +Standard: True +Distributed: False +Dashboard: {} +```` + +###### - V1 TestCase (`Version: 1` or missing) + +TestCases using this format follow the same approach as for UE files. The following is an example V1 TestCase: ````yaml Slice Creation: