Allow jinja2 template task definitions

.vscode/launch.json

@@ -33,6 +33,16 @@
       "args": ["-t", "scp-basic", "-c", "test/cfg"],
       "justMyCode": false
     },
+    {
+      "name": "Python: Transfer - SSH Vars",
+      "type": "python",
+      "request": "launch",
+      "preLaunchTask": "Build Test containers",
+      "program": "src/opentaskpy/cli/task_run.py",
+      "console": "integratedTerminal",
+      "args": ["-t", "scp-basic-ssh-vars", "-c", "test/cfg", "-v", "10"],
+      "justMyCode": false
+    },
     {
       "name": "Python: Transfer - Basic - JSON Logging",
       "type": "python",

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+# v0.14.0
+
+- Add mode override docs to `--help`
+- Allow task definitions to use a whole templated object within them. e.g a protocol definition could be defined as a global variable to define SFTP connectivity to a commonly used source/destination. This is done using `.j2` task definition file instead of `.json`
+
 # v0.13.1
 
 - Fix protocol schemas to allow protocols other that SSH to actually be used for transfers

README.md

@@ -99,7 +99,7 @@ options:
 
 **-t, --taskId**
 
-This relates to the specific task that you want to run. It is the name of the configuration file to load (without the .json suffix), contained under the `CONFIGDIR`
+This relates to the specific task that you want to run. It is the name of the configuration file to load (without the `.json` or `.json.j2` suffix), contained under the `CONFIGDIR`
 
 **-r, --runId**
 
@@ -232,6 +232,8 @@ Task definitions are validated using the JSON Schemas defined within `src/openta
 
 At a later date, I plan to automate the creation of the JSON schema documentation.
 
+The task definitions themselves live under the `cfg` directory, and any number of sub directories under there to allow for grouping of tasks by whatever you like. Definitions can be either a plain JSON file, or a Jinja2 template. If a template is used, it must have the `.j2` suffix. JSON files may also use internal variables, this is not supported when using `.j2` task definitions.
+
 ## Transfers
 
 Transfers consist of a `source` definition, and an optional `destination`.

src/opentaskpy/cli/task_run.py

@@ -29,6 +29,18 @@ def main() -> None:
                                     OTF_LOG_DIRECTORY - Specify a particular log directory to write log files to
                                     OTF_LOG_LEVEL - Equivalent to using -v
                                     OTF_SSH_KEY - Specify a particular SSH key to use for SSH/SFTP related transfers
+
+                                Task Definition Overrides:
+
+                                To override task specific values, you can use the following format in the environment variable name:
+                                OTF_OVERRIDE_<TASK_TYPE>_<ATTRIBUTE>_<ATTRIBUTE>_<ATTRIBUTE>
+
+                                e.g. OTF_OVERRIDE_TRANSFER_SOURCE_HOSTNAME
+
+                                Case doesn't matter here. For attributes that are nested within an array, you can specify the array index
+
+                                e.g. OTF_OVERRIDE_TRANSFER_DESTINATION_0_PROTOCOL_CREDENTIALS_USERNAME
+
                                 """),
     )
     parser.add_argument(
@@ -106,15 +118,3 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()
-    main()

src/opentaskpy/config/loader.py

@@ -142,6 +142,10 @@ def load_task_definition(self, task_id: str) -> dict:
             dict: A dictionary representing the task definition
         """
         json_config = glob(f"{self.config_dir}/**/{task_id}.json", recursive=True)
+        json_config.extend(
+            glob(f"{self.config_dir}/**/{task_id}.json.j2", recursive=True)
+        )
+
         if not json_config or len(json_config) != 1:
             if len(json_config) > 1:
                 raise DuplicateConfigFileError(
@@ -223,15 +227,19 @@ def _enrich_variables(self, task_definition_file: str) -> dict:
             json_content = json_file.read()
             template = self.template_env.from_string(json_content)
 
-            # From this, convert it to JSON and pull out the variables key if there is one
-            task_definition = json.loads(json_content)
-            # Extend or replace any local variables for this task
-            if "variables" in task_definition:
-                self.global_variables = (
-                    self.global_variables | task_definition["variables"]
-                )
-
-            template = self.template_env.from_string(json_content)
+            # If the file is a Jinja2 template, then we do not allow additional
+            # variables to be defined in the task definition.
+            # Check the file extension
+            if not task_definition_file.endswith(".j2"):
+                # From this, convert it to JSON and pull out the variables key if there is one
+                task_definition = json.loads(json_content)
+                # Extend or replace any local variables for this task
+                if "variables" in task_definition:
+                    self.global_variables = (
+                        self.global_variables | task_definition["variables"]
+                    )
+
+                template = self.template_env.from_string(json_content)
 
             template.globals["now"] = datetime.datetime.utcnow
 

test/cfg/transfers/scp-basic-ssh-vars.json.j2

@@ -0,0 +1,21 @@
+{
+  "type": "transfer",
+  "source": {
+    "hostname": "{{ HOST_A }}",
+    "directory": "/tmp/testFiles/src",
+    "fileRegex": ".*\\.txt",
+    "protocol": {
+      "name": "ssh",
+      "credentials": {
+        "username": "{{ SSH_USERNAME }}"
+      }
+    }
+  },
+  "destination": [
+    {
+      "hostname": "{{ HOST_B }}",
+      "directory": "/tmp/testFiles/dest",
+      "protocol": {{ SSH_VARS| tojson }}
+    }
+  ]
+}

test/cfg/variables.json.j2

@@ -23,5 +23,6 @@
   "PREV_YYYY": "{{ (now()|delta_days(-1)).strftime('%Y') }}",
   "testLookup": "{{ lookup('file', path='/tmp/variable_lookup.txt') }}",
   "testLookup2": "{{ lookup('http_json', url='https://jsonplaceholder.typicode.com/posts/1', jsonpath='$.title') }}",
-  "global_protocol_vars": {"name": "email", "smtp_port": 587, "smtp_server": "smtp.gmail.com"}
+  "global_protocol_vars": {"name": "email", "smtp_port": 587, "smtp_server": "smtp.gmail.com"},
+  "SSH_VARS": {"name": "ssh", "credentials": {"username": "someuser"}}
 }

tests/test_config_loader.py

@@ -5,6 +5,7 @@
 from datetime import datetime, timedelta
 
 import pytest
+from jinja2.exceptions import UndefinedError
 from pytest_shell import fs
 
 from opentaskpy.config.loader import ConfigLoader
@@ -72,21 +73,36 @@ def test_load_new_variables_from_task_def(write_dummy_variables_file, tmpdir):
             {
                 f"{tmpdir}/task.json": {
                     "content": (
-                        '{"test_var": "{{ test }}", "variables": {"NEW_VARIABLE":'
-                        ' "NEW_VALUE"}}'
+                        '{"test_var": "{{ test }}", "my_new_var": "{{ NEW_VARIABLE }}",'
+                        ' "variables": {"NEW_VARIABLE": "NEW_VALUE"}}'
                     )
                 }
-            }
+            },
+            {
+                f"{tmpdir}/task1.json.j2": {
+                    "content": (
+                        '{"test_var": "{{ test }}","my_new_var": "{{ NEW_VARIABLE }}",'
+                        ' "variables": {"NEW_VARIABLE": "NEW_VALUE"}}'
+                    )
+                },
+            },
         ]
     )
 
     expected_task_definition = {
         "test_var": "test123456",
+        "my_new_var": "NEW_VALUE",
         "variables": {"NEW_VARIABLE": "NEW_VALUE"},
     }
 
     # Test that the task definition is loaded correctly
     assert config_loader.load_task_definition("task") == expected_task_definition
+    # task1 should fail because it's a jinja template and we don't support setting
+    # variables in jinja templates
+    config_loader = ConfigLoader(tmpdir)
+    # Expect a jinja2 UndefinedError
+    with pytest.raises(UndefinedError):
+        config_loader.load_task_definition("task1")
 
 
 def test_custom_plugin(tmpdir):