From 9bff60c851b5265f9016337b4e824394c8f492d2 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Tue, 1 Feb 2022 09:18:19 +0100 Subject: [PATCH 01/10] Quote database name when retrieving tags --- Helper/influx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helper/influx.py b/Helper/influx.py index 79f8dfe..7ae495a 100644 --- a/Helper/influx.py +++ b/Helper/influx.py @@ -193,7 +193,7 @@ def _getDateTime(value: str): cls.initialize() # Retrieve the list of tags from the server, to separate from fields - reply = cls.client.query(f"show tag keys on {cls.database} from {measurement}") + reply = cls.client.query(f'show tag keys on "{cls.database}" from "{measurement}"') tags = sorted([t['tagKey'] for t in reply.get_points()]) pointsPerTagSet = {} From 23afe71b42b822694b92fb7471f852f0bbfdf921 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Tue, 1 Feb 2022 12:06:16 +0100 Subject: [PATCH 02/10] Add notion of 'Verdict' to tasks, executors. Modify access to 'PreviousTaskLog' --- Executor/Tasks/Run/publish_from_source.py | 2 +- Executor/__init__.py | 2 +- Executor/{status.py => enums.py} | 9 +++++++++ Executor/executor.py | 7 +++++-- Executor/executor_base.py | 14 ++++++++++++-- Experiment/execution_tombstone.py | 4 ++-- Experiment/experiment_run.py | 9 +++++++-- README.md | 2 +- Task/task.py | 5 +++-- 9 files changed, 41 insertions(+), 13 deletions(-) rename Executor/{status.py => enums.py} (66%) diff --git a/Executor/Tasks/Run/publish_from_source.py b/Executor/Tasks/Run/publish_from_source.py index c57158c..4fec6cb 100644 --- a/Executor/Tasks/Run/publish_from_source.py +++ b/Executor/Tasks/Run/publish_from_source.py @@ -49,7 +49,7 @@ def __init__(self, logMethod, parent, params): super().__init__("Publish From Previous Task Log", parent, params, logMethod) def generator(self, params: Dict): - logMessages = self.parent.Params["PreviousTaskLog"] + logMessages = self.parent.PreviousTaskLog for message in logMessages: yield message diff --git a/Executor/__init__.py b/Executor/__init__.py index 00f8167..605fdd2 100644 --- a/Executor/__init__.py +++ b/Executor/__init__.py @@ -2,4 +2,4 @@ from .pre_runner import PreRunner from .executor import Executor from .post_runner import PostRunner -from .status import Status as ExecutorStatus +from .enums import Status as ExecutorStatus, Verdict diff --git a/Executor/status.py b/Executor/enums.py similarity index 66% rename from Executor/status.py rename to Executor/enums.py index 12b31fc..f95ae51 100644 --- a/Executor/status.py +++ b/Executor/enums.py @@ -10,3 +10,12 @@ def label(self): if self.name == 'Errored': return 'label-danger' if self.name == 'Finished': return 'label-success' return 'label-info' + + +@unique +class Verdict(Enum): + NotSet, Pass, Inconclusive, Fail, Cancel, Error = range(6) + + @staticmethod + def Max(a, b): + return a if a.value > b.value else b diff --git a/Executor/executor.py b/Executor/executor.py index feef6ee..6330760 100644 --- a/Executor/executor.py +++ b/Executor/executor.py @@ -3,7 +3,7 @@ from time import sleep from .executor_base import ExecutorBase from Task import Task -from .status import Status +from .enums import Status, Verdict from tempfile import TemporaryDirectory from math import floor @@ -19,6 +19,7 @@ def Run(self): tasks = self.Configuration.RunTasks self.params['PreviousTaskLog'] = [] + self.params['Verdict'] = Verdict.NotSet for i, task in enumerate(tasks, start=1): if self.stopRequested: self.LogAndMessage(Level.INFO, "Received stop request, exiting") @@ -31,8 +32,10 @@ def Run(self): # Add the values generated by the task to the global dictionary self.params.update(taskInstance.Vault) self.params['PreviousTaskLog'] = taskInstance.LogMessages + self.params['Verdict'] = Verdict.Max(self.Verdict, taskInstance.Verdict) - self.AddMessage(f'Task {taskInstance.name} finished', int(floor(10 + ((i / len(tasks)) * 80)))) + self.AddMessage(f"Task '{taskInstance.name}' finished with verdict '{taskInstance.Verdict.name}'", + int(floor(10 + ((i / len(tasks)) * 90)))) else: self.Status = Status.Finished diff --git a/Executor/executor_base.py b/Executor/executor_base.py index 69f593c..d0e3e45 100644 --- a/Executor/executor_base.py +++ b/Executor/executor_base.py @@ -5,7 +5,7 @@ from Composer import PlatformConfiguration from datetime import datetime, timezone from Helper import Serialize -from .status import Status +from .enums import Status, Verdict from tempfile import TemporaryDirectory from Interfaces import PortalApi @@ -53,6 +53,14 @@ def Configuration(self) -> Optional[PlatformConfiguration]: def DeployedSliceId(self) -> Optional[str]: return self.params.get('DeployedSliceId', None) + @property + def Verdict(self) -> Verdict: + return self.params.get('Verdict', Verdict.NotSet) + + @property + def PreviousTaskLog(self) -> List[str]: + return self.params.get('PreviousTaskLog', []) + def Run(self): raise NotImplementedError() @@ -123,7 +131,8 @@ def Serialize(self) -> Dict: 'Status': self.Status.name, 'Log': self.LogFile, 'Messages': self.Messages, - 'PerCent': self.PerCent + 'PerCent': self.PerCent, + 'Verdict': self.Verdict.name } return data @@ -156,5 +165,6 @@ def Load(cls, tag: str, id: str): res.Started = Serialize.StringToDate(data['Started']) res.Finished = Serialize.StringToDate(data['Finished']) res.Status = Status[data['Status']] + res.Params['Verdict'] = Verdict[data.get('Verdict', 'NotSet')] return res diff --git a/Experiment/execution_tombstone.py b/Experiment/execution_tombstone.py index e17095a..c276fb9 100644 --- a/Experiment/execution_tombstone.py +++ b/Experiment/execution_tombstone.py @@ -1,5 +1,5 @@ from Helper import Serialize -from Executor import Executor +from Executor import Executor, Verdict from .experiment_run import CoarseStatus @@ -17,4 +17,4 @@ def __init__(self, id: str): self.JsonDescriptor = data.get('JsonDescriptor', {}) self.Milestones = data.get('Milestones', []) self.RemoteId = data.get('RemoteId', None) - + self.Verdict = Verdict[data.get('Verdict', 'NotSet')] diff --git a/Experiment/experiment_run.py b/Experiment/experiment_run.py index 42cf490..9e523b0 100644 --- a/Experiment/experiment_run.py +++ b/Experiment/experiment_run.py @@ -1,4 +1,4 @@ -from Executor import PreRunner, Executor, PostRunner, ExecutorBase +from Executor import PreRunner, Executor, PostRunner, ExecutorBase, Verdict from Data import ExperimentDescriptor from typing import Dict, Optional, List from enum import Enum, unique @@ -105,6 +105,10 @@ def PerCent(self) -> int: current = self.CurrentChild return current.PerCent if current is not None else 0 + @property + def Verdict(self) -> Verdict: + return self.Executor.Verdict # Pre/PostRun always remain NotSet + @property def LastMessage(self) -> str: current = self.CurrentChild @@ -247,7 +251,8 @@ def Serialize(self) -> Dict: 'Cancelled': self.Cancelled, 'JsonDescriptor': self.Descriptor.Json, 'Milestones': self.Milestones, - 'RemoteId': self.RemoteId + 'RemoteId': self.RemoteId, + 'Verdict': self.Verdict.name } return data diff --git a/README.md b/README.md index c36a1fe..05b67ff 100644 --- a/README.md +++ b/README.md @@ -518,7 +518,7 @@ otherwise the file will be created as configured). Executes a TAP TestPlan, with the possibility of configuring external parameters. Configuration values: - `TestPlan`: Path (absolute) of the testplan file. - `GatherResults`: Indicates whether to compress the generated CSV files to a Zip file (see below) -- `External`: Dictionary of external parameters +- `Externals`: Dictionary of external parameters ###### Gathering generated results If selected, the task will attempt to retrieve all the results generated by the testplan, saving them to a Zip file diff --git a/Task/task.py b/Task/task.py index 9c8a682..da07fe1 100644 --- a/Task/task.py +++ b/Task/task.py @@ -6,7 +6,7 @@ class Task: def __init__(self, name: str, parent, params: Optional[Dict] = None, logMethod: Optional[Callable] = None, conditionMethod: Optional[Callable] = None): - from Executor import ExecutorBase + from Executor import ExecutorBase, Verdict self.name = name self.params = {} if params is None else params @@ -16,6 +16,7 @@ def __init__(self, name: str, parent, params: Optional[Dict] = None, self.condition = conditionMethod self.Vault = {} self.LogMessages = [] + self.Verdict = Verdict.NotSet def Start(self) -> Dict: if self.condition is None or self.condition(): @@ -23,7 +24,7 @@ def Start(self) -> Dict: self.Log(Level.DEBUG, f'Params: {self.params}') if self.SanitizeParams(): self.Run() - self.Log(Level.INFO, f"[Task '{self.name}' finished]") + self.Log(Level.INFO, f"[Task '{self.name}' finished (verdict: '{self.Verdict.name}')]") else: message = f"[Task '{self.name}' cancelled due to incorrect parameters ({self.params})]" self.Log(Level.ERROR, message) From b709eddd4619f04fdf156f2ff439ca775c3fae57 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 2 Feb 2022 09:38:04 +0100 Subject: [PATCH 03/10] Fix params on coordination tasks --- Executor/Tasks/PostRun/farewell.py | 2 +- Executor/Tasks/PreRun/coordinate.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Executor/Tasks/PostRun/farewell.py b/Executor/Tasks/PostRun/farewell.py index 39a9667..2008374 100644 --- a/Executor/Tasks/PostRun/farewell.py +++ b/Executor/Tasks/PostRun/farewell.py @@ -7,7 +7,7 @@ class Farewell(Task): def __init__(self, logMethod, parent): - super().__init__("Farewell", parent, logMethod, None) + super().__init__("Farewell", parent, None, logMethod, None) def Run(self): remote = self.parent.Descriptor.Remote diff --git a/Executor/Tasks/PreRun/coordinate.py b/Executor/Tasks/PreRun/coordinate.py index 0530372..a1b98b2 100644 --- a/Executor/Tasks/PreRun/coordinate.py +++ b/Executor/Tasks/PreRun/coordinate.py @@ -7,7 +7,7 @@ class Coordinate(Task): def __init__(self, logMethod, parent): - super().__init__("Coordinate", parent, logMethod, None) + super().__init__("Coordinate", parent, None, logMethod, None) def Run(self): remote = self.parent.Descriptor.Remote @@ -18,7 +18,6 @@ def Run(self): if host is not None: remoteApi = RemoteApi(host, port) self.parent.RemoteApi = remoteApi - # TODO: Why are these messages not visible in the logs? self.Log(Level.INFO, 'Remote connection configured. Waiting for remote Execution ID...') timeout = eastWest.Timeout or 120 From 3d817db5608537a17e209192fc87eb005df81027 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 2 Feb 2022 09:40:53 +0100 Subject: [PATCH 04/10] Handle VerdictOnError --- Executor/Tasks/Evolved5g/jenkins_api.py | 2 ++ Executor/Tasks/Evolved5g/nef_emulator.py | 1 + Executor/Tasks/Remote/getValue.py | 2 ++ Executor/Tasks/Remote/waitForMilestone.py | 3 +++ Executor/Tasks/Run/compress_files.py | 1 + Executor/Tasks/Run/csvToInflux.py | 2 ++ Executor/Tasks/Run/delay.py | 1 + Executor/Tasks/Run/publish.py | 2 -- Executor/Tasks/Run/publish_from_source.py | 2 ++ Executor/Tasks/Run/rest_api.py | 1 + Executor/Tasks/Run/slice_creation_time.py | 5 +++++ Executor/Tasks/Run/tap_execute.py | 2 ++ Executor/executor.py | 9 +++++++-- Task/task.py | 9 +++++++++ 14 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Executor/Tasks/Evolved5g/jenkins_api.py b/Executor/Tasks/Evolved5g/jenkins_api.py index 2ef3b15..21075b6 100644 --- a/Executor/Tasks/Evolved5g/jenkins_api.py +++ b/Executor/Tasks/Evolved5g/jenkins_api.py @@ -56,6 +56,7 @@ def Run(self): self.Publish(self.params["PublishKey"], jobId) except Exception as e: self.Log(Level.ERROR, f"Unable to trigger job: {e}") + self.MaybeSetErrorVerdict() class JenkinsStatus(JenkinsBase): @@ -79,3 +80,4 @@ def Run(self): self.Publish(self.params["PublishKey"], status) except Exception as e: self.Log(Level.ERROR, f"Unable to check job '{jobId}' status: {e}") + self.MaybeSetErrorVerdict() diff --git a/Executor/Tasks/Evolved5g/nef_emulator.py b/Executor/Tasks/Evolved5g/nef_emulator.py index 38bd6e0..3c29d43 100644 --- a/Executor/Tasks/Evolved5g/nef_emulator.py +++ b/Executor/Tasks/Evolved5g/nef_emulator.py @@ -50,3 +50,4 @@ def Run(self): self.Log(Level.INFO, f"Response: {msg}") except Exception as e: self.Log(Level.ERROR, str(e)) + self.MaybeSetErrorVerdict() diff --git a/Executor/Tasks/Remote/getValue.py b/Executor/Tasks/Remote/getValue.py index 2de09ca..507b492 100644 --- a/Executor/Tasks/Remote/getValue.py +++ b/Executor/Tasks/Remote/getValue.py @@ -13,6 +13,7 @@ def Run(self): if valueName is None: self.Log(Level.ERROR, "'Value' not defined, please review the Task configuration.") + self.MaybeSetErrorVerdict() return value = None @@ -22,6 +23,7 @@ def Run(self): value = self.remoteApi.GetValue(self.remoteId, valueName) if value is None: if self.timeout <= 0: + self.MaybeSetErrorVerdict() raise RuntimeError(f"Timeout reached while waiting for remote remote value '{valueName}'.") sleep(5) self.timeout -= 5 diff --git a/Executor/Tasks/Remote/waitForMilestone.py b/Executor/Tasks/Remote/waitForMilestone.py index 5b9d1b8..771ae87 100644 --- a/Executor/Tasks/Remote/waitForMilestone.py +++ b/Executor/Tasks/Remote/waitForMilestone.py @@ -13,6 +13,7 @@ def Run(self): if milestone is None: self.Log(Level.ERROR, "'Milestone' not defined, please review the Task configuration.") + self.MaybeSetErrorVerdict() return milestones = [] @@ -23,10 +24,12 @@ def Run(self): self.Log(Level.DEBUG, f"Status: '{status}'; Milestones: {milestones}") if status in [ExecutorStatus.Cancelled, ExecutorStatus.Errored]: + self.MaybeSetErrorVerdict() raise RuntimeError(f"Execution on remote side has been terminated with status: {status.name}") if milestone not in milestones: if self.timeout <= 0: + self.MaybeSetErrorVerdict() raise RuntimeError(f"Timeout reached while waiting for milestone '{milestone}'.") sleep(5) self.timeout -= 5 diff --git a/Executor/Tasks/Run/compress_files.py b/Executor/Tasks/Run/compress_files.py index fbdf04a..7d6fb3e 100644 --- a/Executor/Tasks/Run/compress_files.py +++ b/Executor/Tasks/Run/compress_files.py @@ -31,4 +31,5 @@ def Run(self): self.Log(Level.INFO, "File created") self.parent.GeneratedFiles.append(output) except Exception as e: + self.MaybeSetErrorVerdict() self.Log(Level.ERROR, f"Exception while creating zip file: {e}") diff --git a/Executor/Tasks/Run/csvToInflux.py b/Executor/Tasks/Run/csvToInflux.py index 61ff8f2..02758c3 100644 --- a/Executor/Tasks/Run/csvToInflux.py +++ b/Executor/Tasks/Run/csvToInflux.py @@ -34,6 +34,7 @@ def Run(self): self.Log(Level.DEBUG, f"Payload: {payload}") except Exception as e: self.Log(Level.ERROR, f"Exception while converting CSV: {e}") + self.MaybeSetErrorVerdict() return try: @@ -41,3 +42,4 @@ def Run(self): InfluxDb.Send(payload) except Exception as e: self.Log(Level.ERROR, f"Exception while sending CSV values to Influx: {e}") + self.MaybeSetErrorVerdict() diff --git a/Executor/Tasks/Run/delay.py b/Executor/Tasks/Run/delay.py index ab32d42..344f50e 100644 --- a/Executor/Tasks/Run/delay.py +++ b/Executor/Tasks/Run/delay.py @@ -16,6 +16,7 @@ def Run(self): raise ValueError except ValueError: self.Log(Level.ERROR, f"{value} is not a valid number of seconds") + self.MaybeSetErrorVerdict() return self.Log(Level.INFO, f'Waiting for {time} seconds') diff --git a/Executor/Tasks/Run/publish.py b/Executor/Tasks/Run/publish.py index 35edf02..eda424b 100644 --- a/Executor/Tasks/Run/publish.py +++ b/Executor/Tasks/Run/publish.py @@ -1,6 +1,4 @@ from Task import Task -from Helper import Level -from time import sleep class Publish(Task): diff --git a/Executor/Tasks/Run/publish_from_source.py b/Executor/Tasks/Run/publish_from_source.py index 4fec6cb..67c0bf2 100644 --- a/Executor/Tasks/Run/publish_from_source.py +++ b/Executor/Tasks/Run/publish_from_source.py @@ -26,6 +26,7 @@ def Run(self): for index, key in keys: self.Log(Level.DEBUG, f" {index}: {key}") except Exception as e: + self.MaybeSetErrorVerdict() raise RuntimeError(f"Invalid 'Keys' definition: {e}") regex = re.compile(pattern) @@ -41,6 +42,7 @@ def generator(self, params: Dict): raise NotImplementedError() def raiseConfigError(self, variable: str): + self.MaybeSetErrorVerdict() raise RuntimeError(f"'{variable}' not defined, please review the Task configuration.") diff --git a/Executor/Tasks/Run/rest_api.py b/Executor/Tasks/Run/rest_api.py index b8649f6..50f139f 100644 --- a/Executor/Tasks/Run/rest_api.py +++ b/Executor/Tasks/Run/rest_api.py @@ -75,4 +75,5 @@ def Run(self): if status not in statusCodes: message = f"Unexpected status code received: {status}" self.Log(Level.ERROR, message) + self.MaybeSetErrorVerdict() raise RuntimeError(message) diff --git a/Executor/Tasks/Run/slice_creation_time.py b/Executor/Tasks/Run/slice_creation_time.py index 7896180..040fd71 100644 --- a/Executor/Tasks/Run/slice_creation_time.py +++ b/Executor/Tasks/Run/slice_creation_time.py @@ -34,6 +34,7 @@ def Run(self): nestData = json.load(input) except Exception as e: self.Log(Level.ERROR, f"Exception while reading NEST file: {e}") + self.MaybeSetErrorVerdict() return from Helper import InfluxDb, InfluxPayload, InfluxPoint # Delayed to avoid cyclic imports @@ -51,6 +52,7 @@ def Run(self): sliceId = response except Exception as e: self.Log(Level.ERROR, f"Exception on instantiation, skipping iteration: {e}") + self.MaybeSetErrorVerdict() sleep(pollTime) continue @@ -86,6 +88,7 @@ def Run(self): f"Deployment time for slice {sliceId} (Iteration {iteration}): {ns_depl_time}") except Exception as e: self.Log(Level.ERROR, f"Exception while calculating deployment time, skipping iteration: {e}") + self.MaybeSetErrorVerdict() break point = InfluxPoint(datetime.now(timezone.utc)) @@ -113,6 +116,7 @@ def Run(self): self.Log(Level.DEBUG, f"Waiting for slice deletion.") except Exception as e: self.Log(Level.ERROR, f"Exception while deleting slice: {e}") + self.MaybeSetErrorVerdict() self.Log(Level.DEBUG, f"Payload: {payload}") self.Log(Level.INFO, f"Sending results to InfluxDb") @@ -120,6 +124,7 @@ def Run(self): InfluxDb.Send(payload) except Exception as e: self.Log(Level.ERROR, f"Exception while sending payload: {e}") + self.MaybeSetErrorVerdict() if csvFile is None: self.Log(Level.INFO, "Forcing creation of CSV file") csvFile = join(self.parent.TempFolder, f"SliceCreationTime.csv") diff --git a/Executor/Tasks/Run/tap_execute.py b/Executor/Tasks/Run/tap_execute.py index ba34a3e..d7abda6 100644 --- a/Executor/Tasks/Run/tap_execute.py +++ b/Executor/Tasks/Run/tap_execute.py @@ -20,6 +20,7 @@ def Run(self): if not config.Enabled: self.Log(Level.CRITICAL, "Trying to run TapExecute Task while TAP is not enabled") + self.MaybeSetErrorVerdict() else: tapPlan = self.params['TestPlan'] externals = self.params['Externals'] @@ -42,6 +43,7 @@ def Run(self): self.parent.GeneratedFiles.append(output) self.Log(Level.INFO, f"Saved {len(files)} files to {output}") except Exception as e: + self.MaybeSetErrorVerdict() self.Log(Level.ERROR, f"Exception while compressing results: {e}") else: self.Log(Level.WARNING, f"Results path ({path}) does not exist.") diff --git a/Executor/executor.py b/Executor/executor.py index 6330760..57c02d1 100644 --- a/Executor/executor.py +++ b/Executor/executor.py @@ -1,6 +1,5 @@ from Helper import Level from typing import Dict -from time import sleep from .executor_base import ExecutorBase from Task import Task from .enums import Status, Verdict @@ -24,10 +23,16 @@ def Run(self): if self.stopRequested: self.LogAndMessage(Level.INFO, "Received stop request, exiting") self.Status = Status.Cancelled + self.params['Verdict'] = Verdict.Cancel break taskInstance: Task = task.Task(self.Log, self, Expander.ExpandDict(task.Params, self)) self.AddMessage(f'Starting task {taskInstance.name}') - taskInstance.Start() + + try: + taskInstance.Start() + except Exception as e: + self.params['Verdict'] = Verdict.Error + raise e # Add the values generated by the task to the global dictionary self.params.update(taskInstance.Vault) diff --git a/Task/task.py b/Task/task.py index da07fe1..334b8b8 100644 --- a/Task/task.py +++ b/Task/task.py @@ -46,6 +46,10 @@ def Log(self, level: Union[Level, str], msg: str): self.LogMessages.append(msg) def SanitizeParams(self): + if 'VerdictOnError' not in self.paramRules: + from Executor import Verdict + self.paramRules['VerdictOnError'] = ("NotSet", False) + for key, value in self.paramRules.items(): default, mandatory = value if key not in self.params.keys(): @@ -56,3 +60,8 @@ def SanitizeParams(self): self.params[key] = default self.Log(Level.DEBUG, f"Parameter '{key}' set to default ({str(default)}).") return True + + def MaybeSetErrorVerdict(self): + from Executor import Verdict + if self.params['VerdictOnError'] != "NotSet": + self.Verdict = Verdict[self.params['VerdictOnError']] From 32f04f59b8d2bb893dcf9ce8a7d6da728d933a21 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 2 Feb 2022 11:41:31 +0100 Subject: [PATCH 05/10] Implement UpgradeVerdict task --- Executor/Tasks/Run/__init__.py | 1 + Executor/Tasks/Run/publish.py | 2 ++ Executor/Tasks/Run/upgrade_verdict.py | 52 +++++++++++++++++++++++++++ Executor/executor_base.py | 2 +- 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 Executor/Tasks/Run/upgrade_verdict.py diff --git a/Executor/Tasks/Run/__init__.py b/Executor/Tasks/Run/__init__.py index bed150a..2d3ebe2 100644 --- a/Executor/Tasks/Run/__init__.py +++ b/Executor/Tasks/Run/__init__.py @@ -11,3 +11,4 @@ from .add_milestone import AddMilestone from .publish_from_source import PublishFromPreviousTaskLog, PublishFromFile from .rest_api import RestApi +from .upgrade_verdict import UpgradeVerdict diff --git a/Executor/Tasks/Run/publish.py b/Executor/Tasks/Run/publish.py index eda424b..97051f5 100644 --- a/Executor/Tasks/Run/publish.py +++ b/Executor/Tasks/Run/publish.py @@ -7,4 +7,6 @@ def __init__(self, logMethod, parent, params): def Run(self): for key, value in self.params.items(): + if key is "VerdictOnError": + continue # This key is automatically added to all tasks self.Publish(key, value) diff --git a/Executor/Tasks/Run/upgrade_verdict.py b/Executor/Tasks/Run/upgrade_verdict.py new file mode 100644 index 0000000..713e1de --- /dev/null +++ b/Executor/Tasks/Run/upgrade_verdict.py @@ -0,0 +1,52 @@ +from Task import Task +from Helper import Level +import re + + +class UpgradeVerdict(Task): + def __init__(self, logMethod, parent, params): + super().__init__("Upgrade Verdict", parent, params, logMethod, None) + self.paramRules = { + 'Key': (None, True), + 'Pattern': (None, True), + 'VerdictOnMissing': ("NotSet", False), + 'VerdictOnMatch': ("NotSet", False), + 'VerdictOnNoMatch': ("NotSet", False), + } + + def Run(self): + from Executor import Verdict + + def _assignVerdict(name: str) -> Verdict | None: + try: + return Verdict[name] + except KeyError: + self.MaybeSetErrorVerdict() + self.Log(Level.ERROR, f"Unrecognized Verdict '{name}'") + return None + + onMiss = _assignVerdict(self.params["VerdictOnMissing"]) + onMatch = _assignVerdict(self.params["VerdictOnMatch"]) + onNoMatch = _assignVerdict(self.params["VerdictOnNoMatch"]) + if None in [onMiss, onMatch, onNoMatch]: return + + key = self.params["Key"] + regex = re.compile(self.params["Pattern"]) + collection = self.parent.Params + + if key not in collection.keys(): + self.Log(Level.WARNING, f"Key '{key}' not found. Setting Verdict to '{onMiss.name}'") + self.Log(Level.DEBUG, f"Available keys: {list(collection.keys())}") + self.Verdict = onMiss + else: + value = str(collection[key]) + if regex.match(value): + condition = "matches" + verdict = onMatch + else: + condition = "does not match" + verdict = onNoMatch + + self.Log(Level.INFO, f"'{key}'='{value}' {condition} pattern. Setting Verdict to '{verdict.name}'") + self.Verdict = verdict + diff --git a/Executor/executor_base.py b/Executor/executor_base.py index d0e3e45..1e88269 100644 --- a/Executor/executor_base.py +++ b/Executor/executor_base.py @@ -86,7 +86,7 @@ def SetFinished(self, status=Status.Finished, percent: int = None): self.Finished = datetime.now(timezone.utc) if self.Status.value < Status.Cancelled.value: self.Status = status - self.LogAndMessage(Level.INFO, f"Finished (status: {self.Status.name})", percent) + self.LogAndMessage(Level.INFO, f"Finished (status: {self.Status.name}, verdict: {self.Verdict.name})", percent) def findParent(self): # Only running experiments should be able to use this method from Status import ExecutionQueue From 3d557626653a955e148866ba597cd39ad8a70364 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 2 Feb 2022 13:14:04 +0100 Subject: [PATCH 06/10] Make default VerdictOnError configurable --- Executor/Tasks/Evolved5g/jenkins_api.py | 4 ++-- Executor/Tasks/Evolved5g/nef_emulator.py | 2 +- Executor/Tasks/Remote/getValue.py | 4 ++-- Executor/Tasks/Remote/waitForMilestone.py | 6 +++--- Executor/Tasks/Run/compress_files.py | 2 +- Executor/Tasks/Run/csvToInflux.py | 4 ++-- Executor/Tasks/Run/delay.py | 2 +- Executor/Tasks/Run/publish.py | 4 ++-- Executor/Tasks/Run/publish_from_source.py | 4 ++-- Executor/Tasks/Run/rest_api.py | 2 +- Executor/Tasks/Run/slice_creation_time.py | 10 +++++----- Executor/Tasks/Run/tap_execute.py | 4 ++-- Executor/Tasks/Run/upgrade_verdict.py | 2 +- Settings/config.py | 7 ++++++- Settings/default_config | 1 + Task/task.py | 16 +++++++++------- 16 files changed, 41 insertions(+), 33 deletions(-) diff --git a/Executor/Tasks/Evolved5g/jenkins_api.py b/Executor/Tasks/Evolved5g/jenkins_api.py index 21075b6..1a3741a 100644 --- a/Executor/Tasks/Evolved5g/jenkins_api.py +++ b/Executor/Tasks/Evolved5g/jenkins_api.py @@ -56,7 +56,7 @@ def Run(self): self.Publish(self.params["PublishKey"], jobId) except Exception as e: self.Log(Level.ERROR, f"Unable to trigger job: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() class JenkinsStatus(JenkinsBase): @@ -80,4 +80,4 @@ def Run(self): self.Publish(self.params["PublishKey"], status) except Exception as e: self.Log(Level.ERROR, f"Unable to check job '{jobId}' status: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() diff --git a/Executor/Tasks/Evolved5g/nef_emulator.py b/Executor/Tasks/Evolved5g/nef_emulator.py index 3c29d43..808cbaa 100644 --- a/Executor/Tasks/Evolved5g/nef_emulator.py +++ b/Executor/Tasks/Evolved5g/nef_emulator.py @@ -50,4 +50,4 @@ def Run(self): self.Log(Level.INFO, f"Response: {msg}") except Exception as e: self.Log(Level.ERROR, str(e)) - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() diff --git a/Executor/Tasks/Remote/getValue.py b/Executor/Tasks/Remote/getValue.py index 507b492..bbbc809 100644 --- a/Executor/Tasks/Remote/getValue.py +++ b/Executor/Tasks/Remote/getValue.py @@ -13,7 +13,7 @@ def Run(self): if valueName is None: self.Log(Level.ERROR, "'Value' not defined, please review the Task configuration.") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() return value = None @@ -23,7 +23,7 @@ def Run(self): value = self.remoteApi.GetValue(self.remoteId, valueName) if value is None: if self.timeout <= 0: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() raise RuntimeError(f"Timeout reached while waiting for remote remote value '{valueName}'.") sleep(5) self.timeout -= 5 diff --git a/Executor/Tasks/Remote/waitForMilestone.py b/Executor/Tasks/Remote/waitForMilestone.py index 771ae87..9c832d2 100644 --- a/Executor/Tasks/Remote/waitForMilestone.py +++ b/Executor/Tasks/Remote/waitForMilestone.py @@ -13,7 +13,7 @@ def Run(self): if milestone is None: self.Log(Level.ERROR, "'Milestone' not defined, please review the Task configuration.") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() return milestones = [] @@ -24,12 +24,12 @@ def Run(self): self.Log(Level.DEBUG, f"Status: '{status}'; Milestones: {milestones}") if status in [ExecutorStatus.Cancelled, ExecutorStatus.Errored]: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() raise RuntimeError(f"Execution on remote side has been terminated with status: {status.name}") if milestone not in milestones: if self.timeout <= 0: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() raise RuntimeError(f"Timeout reached while waiting for milestone '{milestone}'.") sleep(5) self.timeout -= 5 diff --git a/Executor/Tasks/Run/compress_files.py b/Executor/Tasks/Run/compress_files.py index 7d6fb3e..8a271bc 100644 --- a/Executor/Tasks/Run/compress_files.py +++ b/Executor/Tasks/Run/compress_files.py @@ -31,5 +31,5 @@ def Run(self): self.Log(Level.INFO, "File created") self.parent.GeneratedFiles.append(output) except Exception as e: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() self.Log(Level.ERROR, f"Exception while creating zip file: {e}") diff --git a/Executor/Tasks/Run/csvToInflux.py b/Executor/Tasks/Run/csvToInflux.py index 02758c3..c18b9c2 100644 --- a/Executor/Tasks/Run/csvToInflux.py +++ b/Executor/Tasks/Run/csvToInflux.py @@ -34,7 +34,7 @@ def Run(self): self.Log(Level.DEBUG, f"Payload: {payload}") except Exception as e: self.Log(Level.ERROR, f"Exception while converting CSV: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() return try: @@ -42,4 +42,4 @@ def Run(self): InfluxDb.Send(payload) except Exception as e: self.Log(Level.ERROR, f"Exception while sending CSV values to Influx: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() diff --git a/Executor/Tasks/Run/delay.py b/Executor/Tasks/Run/delay.py index 344f50e..1bcfb07 100644 --- a/Executor/Tasks/Run/delay.py +++ b/Executor/Tasks/Run/delay.py @@ -16,7 +16,7 @@ def Run(self): raise ValueError except ValueError: self.Log(Level.ERROR, f"{value} is not a valid number of seconds") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() return self.Log(Level.INFO, f'Waiting for {time} seconds') diff --git a/Executor/Tasks/Run/publish.py b/Executor/Tasks/Run/publish.py index 97051f5..e9b45a7 100644 --- a/Executor/Tasks/Run/publish.py +++ b/Executor/Tasks/Run/publish.py @@ -7,6 +7,6 @@ def __init__(self, logMethod, parent, params): def Run(self): for key, value in self.params.items(): - if key is "VerdictOnError": - continue # This key is automatically added to all tasks + if key in ["VerdictOnError"]: + continue # Keys common to all tasks are ignored self.Publish(key, value) diff --git a/Executor/Tasks/Run/publish_from_source.py b/Executor/Tasks/Run/publish_from_source.py index 67c0bf2..26c0e38 100644 --- a/Executor/Tasks/Run/publish_from_source.py +++ b/Executor/Tasks/Run/publish_from_source.py @@ -26,7 +26,7 @@ def Run(self): for index, key in keys: self.Log(Level.DEBUG, f" {index}: {key}") except Exception as e: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() raise RuntimeError(f"Invalid 'Keys' definition: {e}") regex = re.compile(pattern) @@ -42,7 +42,7 @@ def generator(self, params: Dict): raise NotImplementedError() def raiseConfigError(self, variable: str): - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() raise RuntimeError(f"'{variable}' not defined, please review the Task configuration.") diff --git a/Executor/Tasks/Run/rest_api.py b/Executor/Tasks/Run/rest_api.py index 50f139f..d0bc8c7 100644 --- a/Executor/Tasks/Run/rest_api.py +++ b/Executor/Tasks/Run/rest_api.py @@ -75,5 +75,5 @@ def Run(self): if status not in statusCodes: message = f"Unexpected status code received: {status}" self.Log(Level.ERROR, message) - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() raise RuntimeError(message) diff --git a/Executor/Tasks/Run/slice_creation_time.py b/Executor/Tasks/Run/slice_creation_time.py index 040fd71..5e04c10 100644 --- a/Executor/Tasks/Run/slice_creation_time.py +++ b/Executor/Tasks/Run/slice_creation_time.py @@ -34,7 +34,7 @@ def Run(self): nestData = json.load(input) except Exception as e: self.Log(Level.ERROR, f"Exception while reading NEST file: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() return from Helper import InfluxDb, InfluxPayload, InfluxPoint # Delayed to avoid cyclic imports @@ -52,7 +52,7 @@ def Run(self): sliceId = response except Exception as e: self.Log(Level.ERROR, f"Exception on instantiation, skipping iteration: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() sleep(pollTime) continue @@ -88,7 +88,7 @@ def Run(self): f"Deployment time for slice {sliceId} (Iteration {iteration}): {ns_depl_time}") except Exception as e: self.Log(Level.ERROR, f"Exception while calculating deployment time, skipping iteration: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() break point = InfluxPoint(datetime.now(timezone.utc)) @@ -116,7 +116,7 @@ def Run(self): self.Log(Level.DEBUG, f"Waiting for slice deletion.") except Exception as e: self.Log(Level.ERROR, f"Exception while deleting slice: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() self.Log(Level.DEBUG, f"Payload: {payload}") self.Log(Level.INFO, f"Sending results to InfluxDb") @@ -124,7 +124,7 @@ def Run(self): InfluxDb.Send(payload) except Exception as e: self.Log(Level.ERROR, f"Exception while sending payload: {e}") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() if csvFile is None: self.Log(Level.INFO, "Forcing creation of CSV file") csvFile = join(self.parent.TempFolder, f"SliceCreationTime.csv") diff --git a/Executor/Tasks/Run/tap_execute.py b/Executor/Tasks/Run/tap_execute.py index d7abda6..d590bd0 100644 --- a/Executor/Tasks/Run/tap_execute.py +++ b/Executor/Tasks/Run/tap_execute.py @@ -20,7 +20,7 @@ def Run(self): if not config.Enabled: self.Log(Level.CRITICAL, "Trying to run TapExecute Task while TAP is not enabled") - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() else: tapPlan = self.params['TestPlan'] externals = self.params['Externals'] @@ -43,7 +43,7 @@ def Run(self): self.parent.GeneratedFiles.append(output) self.Log(Level.INFO, f"Saved {len(files)} files to {output}") except Exception as e: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() self.Log(Level.ERROR, f"Exception while compressing results: {e}") else: self.Log(Level.WARNING, f"Results path ({path}) does not exist.") diff --git a/Executor/Tasks/Run/upgrade_verdict.py b/Executor/Tasks/Run/upgrade_verdict.py index 713e1de..4bdbb1d 100644 --- a/Executor/Tasks/Run/upgrade_verdict.py +++ b/Executor/Tasks/Run/upgrade_verdict.py @@ -21,7 +21,7 @@ def _assignVerdict(name: str) -> Verdict | None: try: return Verdict[name] except KeyError: - self.MaybeSetErrorVerdict() + self.SetVerdictOnError() self.Log(Level.ERROR, f"Unrecognized Verdict '{name}'") return None diff --git a/Settings/config.py b/Settings/config.py index 23e0da5..8c11e35 100644 --- a/Settings/config.py +++ b/Settings/config.py @@ -225,6 +225,10 @@ def TempFolder(self): def ResultsFolder(self): return Config.data.get('ResultsFolder', 'Results') + @property + def VerdictOnError(self): + return Config.data.get('VerdictOnError', 'Error') + @property def Tap(self): return TapConfig(Config.data.get('Tap', {})) @@ -259,12 +263,13 @@ def _validateSingle(key: str, default: str): keys.discard('Flask') keys.discard('TempFolder') keys.discard('ResultsFolder') + keys.discard('VerdictOnError') if getenv('SECRET_KEY') is None: Config.Validation.append((Level.CRITICAL, "SECRET_KEY not defined. Use environment variables or set a value in .flaskenv")) - for key, default in [('TempFolder', 'Temp'), ('ResultsFolder', 'Results')]: + for key, default in [('TempFolder', 'Temp'), ('ResultsFolder', 'Results'), ('VerdictOnError', 'Error')]: _validateSingle(key, default) for entry in [self.Logging, self.Portal, self.SliceManager, self.Tap, diff --git a/Settings/default_config b/Settings/default_config index cbeb582..6b9f5fd 100644 --- a/Settings/default_config +++ b/Settings/default_config @@ -1,5 +1,6 @@ TempFolder: 'Temp' ResultsFolder: 'Results' +VerdictOnError: 'Error' Logging: Folder: 'Logs' AppLevel: INFO diff --git a/Task/task.py b/Task/task.py index 334b8b8..6847785 100644 --- a/Task/task.py +++ b/Task/task.py @@ -1,5 +1,6 @@ from typing import Callable, Dict, Optional, Union, Tuple, Any from Helper import Log, Level +from Settings import Config class Task: @@ -46,10 +47,6 @@ def Log(self, level: Union[Level, str], msg: str): self.LogMessages.append(msg) def SanitizeParams(self): - if 'VerdictOnError' not in self.paramRules: - from Executor import Verdict - self.paramRules['VerdictOnError'] = ("NotSet", False) - for key, value in self.paramRules.items(): default, mandatory = value if key not in self.params.keys(): @@ -61,7 +58,12 @@ def SanitizeParams(self): self.Log(Level.DEBUG, f"Parameter '{key}' set to default ({str(default)}).") return True - def MaybeSetErrorVerdict(self): + def SetVerdictOnError(self): from Executor import Verdict - if self.params['VerdictOnError'] != "NotSet": - self.Verdict = Verdict[self.params['VerdictOnError']] + verdict = self.params.get('VerdictOnError', None) + if verdict is None: + verdict = Config().VerdictOnError + try: + self.Verdict = Verdict[verdict] + except KeyError as e: + raise RuntimeError(f"Unrecognized Verdict '{verdict}'") from e From 6c73d00ba1aa237f567037c964768a20a3e92d59 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 2 Feb 2022 13:46:51 +0100 Subject: [PATCH 07/10] Update documentation --- README.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 05b67ff..f05a183 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ The ELCM instance can be configured by editing the `config.yml` file. The values * TempFolder: Root folder where the temporal files for the Executors can be created. * ResultsFolder: Root folder where the files generated by each experiment execution will be saved. +* VerdictOnError: Verdict to set on errored tasks, unless overridden by the task parameters. For more information see +the 'Task and execution verdicts' section below. * Logging: * Folder: Root folder where the different log files will be saved. * AppLevel: Minimum log level that will be displayed in the console. @@ -133,6 +135,12 @@ configuration values in this file are: * Port: Port where the API is listening. If empty, the default ports (80 for http, 443 for https) will be used * User: Provided user name * Password: Provided password +* NefEmulator: Configuration values for the NEF Emulator. + * Enabled: Boolean value indicating if the NEF Emulator will be used. Defaults to `False`. + * Host: Address where the NEF Emulator is listening + * Port: Port where the API is listening. If empty, the default ports (80 for http, 443 for https) will be used + * User: User name + * Password: Password ## Facility Configuration (Platform registry) @@ -405,7 +413,13 @@ For more information about Network Slice deployment refer to the ## Available Tasks: The following is a list of the tasks that can be defined as part of a TestCase or UE list of actions, as well as -their configuration values: +their configuration values. + +#### Common values: +All tasks recognize the following configuration value: +- `VerdictOnError`: Name of the verdict to reach when the task encounters an error during execution (what is considered +an error varies from task to task). By default, the value in `config.yml` is used. See the 'Task and execution verdicts' +section below for more information. ### Run.CliExecute Executes a script or command through the command line. Configuration values: @@ -461,6 +475,8 @@ Will produce this message in the log: `- INFO - 1: Text; 2: 1; 3: <>; 4: NoProblem` +> Note that keys that are common to all tasks (for example, `VerdictOnError`) will be ignored. + ### Run.PublishFromFile / Run.PublishFromPreviousTaskLog Reads the contents of a file / the log of the previous task and looks for lines that match the specified regular expression pattern, publishing the groups found. Configuration values: @@ -520,6 +536,16 @@ Executes a TAP TestPlan, with the possibility of configuring external parameters - `GatherResults`: Indicates whether to compress the generated CSV files to a Zip file (see below) - `Externals`: Dictionary of external parameters +### Run.UpgradeVerdict +Sets a particular verdict for this task, which in turn upgrades the verdict of the experiment execution, based +on the value of a published variable. Configuration values: +- `Key`: Name of the key to compare +- `Pattern`: Regular expression to try to match against the value of `Key`, following +[Python's syntax](https://docs.python.org/3/library/re.html#regular-expression-syntax) +- `VerdictOnMissing`: Verdict to set if `Key` is not found. Defaults to `NotSet`. +- `VerdictOnMatch`: Verdict to set if the value matches the regular expression. Defaults to `NotSet`. +- `VerdictOnNoMatch`: Verdict to set if the value does not match the regular expression. Defaults to `NotSet`. + ###### Gathering generated results If selected, the task will attempt to retrieve all the results generated by the testplan, saving them to a Zip file that will be included along with the logs once the execution finishes. The task will look for the files in the TAP @@ -552,6 +578,38 @@ Separate values from the `Parameters` dictionary can also be expanded using the > a '.' the ELCM will fall back to looking at the Publish values (the default for Release A). If the collection > is not 'Publish' or 'Params', the expression will be replaced by `<>` +### Task and execution verdicts + +Each task that is part of an experiment is able to indicate a `Verdict` that defines in a concise way +the final status that was reached after its execution. This way, a task may reach a verdict of `Pass`, +if every action was completed successfully and the results were as expected, or an `Error` verdict if +it was impossible to complete the execution. + +Similarly, an experiment execution is also able to reach a certain verdict, which in this case is the +**verdict with the highest severity among all executed tasks**. The severity and description of the +available verdicts can be seen below: + +| Severity | Name | Description | +|----------|----------------|----------------------------------------------------------------------| +| 0 | `NotSet` | No verdict | +| 1 | `Pass` | Execution completed and all values within required limits | +| 2 | `Inconclusive` | Execution completed, but insufficient results or too close to limits | +| 3 | `Fail` | Execution completed, but results outside the limits | +| 4 | `Cancel` | Execution incomplete due to user request | +| 5 | `Error` | Execution incomplete due to an error | + +For example, if an experiment consists on two tasks, and they reach verdicts of `Inconclusive` and `Pass`, then the +whole execution is considered `Inconclusive`. + +The ELCM will automatically assign the `Cancel` verdict to any execution that receives a cancellation request +(however this verdict is free to use by any task that considers this necessary). In case of error, the selected +verdict is configurable: + - Globally, by setting the `VerdictOnError` value on `config.yml` (`Error` by default). + - Per task, by setting a particular `VerdictOnError` value in the parameters of the task. + +> Handling of verdicts in the ELCM is based on the same concept in +> [OpenTAP](https://docs.opentap.io/Developer%20Guide/Test%20Step/#verdict) + ## EVOLVED-5G specific tasks: The following is a list of tasks specifically tailored for use on the EVOLVED-5G H2020 project. Configuration From 3b6818d4be25e39a239946bec833ee05b57f9d48 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 2 Feb 2022 15:08:32 +0100 Subject: [PATCH 08/10] Return verdict on /json endpoint --- Executor/post_runner.py | 2 +- Experiment/experiment_run.py | 2 +- Scheduler/execution/routes.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Executor/post_runner.py b/Executor/post_runner.py index d6893c0..6ac5f2d 100644 --- a/Executor/post_runner.py +++ b/Executor/post_runner.py @@ -15,7 +15,7 @@ def Run(self): self.AddMessage('End coordination completed', 20) Decommission(self.Log, self, self.DeployedSliceId, self.Configuration.NetworkServices).Start() - self.AddMessage('Network services decommisioned', 50) + self.AddMessage('Network services decommissioned', 50) ReleaseResources(self.Log, self.ExecutionId, self.Configuration.Requirements, self).Start() self.AddMessage('Released resources', 90) diff --git a/Experiment/experiment_run.py b/Experiment/experiment_run.py index 9e523b0..1535254 100644 --- a/Experiment/experiment_run.py +++ b/Experiment/experiment_run.py @@ -107,7 +107,7 @@ def PerCent(self) -> int: @property def Verdict(self) -> Verdict: - return self.Executor.Verdict # Pre/PostRun always remain NotSet + return self.Executor.Verdict # Pre/PostRun do not modify the verdict @property def LastMessage(self) -> str: diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index 4994785..421390d 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -40,10 +40,12 @@ def view(executionId: int): def json(executionId: int): execution = executionOrTombstone(executionId) coarse = status = 'ERR' + verdict = 'NotSet' percent = 0 messages = [] if execution is not None: coarse = execution.CoarseStatus.name + verdict = execution.Verdict.name if isinstance(execution, Tombstone): status = "Not Running" else: @@ -52,7 +54,8 @@ def json(executionId: int): messages = execution.Messages return jsonify({ 'Coarse': coarse, 'Status': status, - 'PerCent': percent, 'Messages': messages + 'PerCent': percent, 'Messages': messages, + 'Verdict': verdict }) From 8f505e1991214e690078ad257bddaf3c4c8e3356 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Thu, 3 Feb 2022 13:36:15 +0100 Subject: [PATCH 09/10] Implement Evaluate task --- Executor/Tasks/Run/__init__.py | 1 + Executor/Tasks/Run/evaluate.py | 25 +++++++++++++++++++++++++ README.md | 31 +++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 Executor/Tasks/Run/evaluate.py diff --git a/Executor/Tasks/Run/__init__.py b/Executor/Tasks/Run/__init__.py index 2d3ebe2..fed00fc 100644 --- a/Executor/Tasks/Run/__init__.py +++ b/Executor/Tasks/Run/__init__.py @@ -12,3 +12,4 @@ from .publish_from_source import PublishFromPreviousTaskLog, PublishFromFile from .rest_api import RestApi from .upgrade_verdict import UpgradeVerdict +from .evaluate import Evaluate diff --git a/Executor/Tasks/Run/evaluate.py b/Executor/Tasks/Run/evaluate.py new file mode 100644 index 0000000..7c4cbdf --- /dev/null +++ b/Executor/Tasks/Run/evaluate.py @@ -0,0 +1,25 @@ +from Task import Task +from Helper import Level +import re + + +class Evaluate(Task): + def __init__(self, logMethod, parent, params): + super().__init__("Evaluate", parent, params, logMethod, None) + self.paramRules = { + 'Key': (None, True), + 'Expression': (None, True) + } + + def Run(self): + expression = self.params["Expression"] + key = self.params["Key"] + self.Log(Level.INFO, f"Evaluating '{key} = {expression}'") + + try: + result = eval(expression) + except Exception as e: + self.Log(Level.ERROR, f"Exception while evaluating expression '{expression}'") + raise e + + self.Publish(key, str(result)) diff --git a/README.md b/README.md index f05a183..a816be7 100644 --- a/README.md +++ b/README.md @@ -452,6 +452,37 @@ Adds a configurable time wait to an experiment execution. Has a single configura ### Run.Dummy Dummy action, will only display the values on the `Config` dictionary on the log +### Run.Evaluate + +Evaluates `Expression`, and publishes the generated result as the `Key` variable. Configuration values: +- `Key`: Name of the key used to save the generated value (as string). +- `Expression`: Python expression that will be evaluated (as string). Variable expansion can be used for specifying +runtime values. + +> ⚠ This task makes use of the [eval](https://docs.python.org/3/library/functions.html#eval) built-in function: +> - The `Expression` can execute arbitrary code. +> - Since the test cases are defined by the platform operators it is expected that no dangerous code will be executed, +> however, **excercise extreme caution, specially if variable expansion is used** as part of the expression. + +The following is an example of the use of this task: + +```yaml + - Order: 1 + Task: Run.Publish + Config: { VAR: 4 } + - Order: 2 + Task: Run.Evaluate + Config: + Key: VAR + Expression: +``` + +After the execution of both tasks, the value of `VAR` will be, depending on the expression: +- For `1+@[VAR]`: "5" +- For `1+@[VAR].0`: "5.0" +- For `1+@[VAR].0>3`: "True" +- For `self`: "" + ### Run.Message Displays a message on the log, with the configured severity. Configuration values: - `Severity`: Severity level (`DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`) From 845c7af230e3a5dc22699f71b839914695ceaf49 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Thu, 3 Feb 2022 13:40:47 +0100 Subject: [PATCH 10/10] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d33a17..a3d7122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +**03/02/2022** [Version 3.2.0] + + - Implement Verdict handling + - Add Evaluate, UpgradeVerdict tasks + **11/11/2021** [Version 3.1.0] - Add NEF Emulator handling