From 912e9ab7fb7270b453ec7ab924d29f0262dd1656 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 15 Jul 2024 21:14:32 -0400 Subject: [PATCH 01/12] fix: add handling for empty strings in key-value pairs of yaml Fix 149 --- seqerakit/seqeraplatform.py | 11 +++++++++++ tests/unit/test_seqeraplatform.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index e6a50d6..a08f816 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -67,8 +67,19 @@ def _construct_command(self, cmd, *args, **kwargs): if "params_file" in kwargs: command.append(f"--params-file={kwargs['params_file']}") + # Check for empty string arguments and handle them + self._check_empty_args(command) + return self._check_env_vars(command) + def _check_empty_args(self, command): + for current_arg, next_arg in zip(command, command[1:]): + if isinstance(next_arg, str) and next_arg.strip() == "": + raise ValueError( + f"Empty string argument found for parameter '{current_arg}'. " + "Please provide a valid value or remove the argument." + ) + # Checks environment variables to see that they are set accordingly def _check_env_vars(self, command): full_cmd_parts = [] diff --git a/tests/unit/test_seqeraplatform.py b/tests/unit/test_seqeraplatform.py index c95a25f..941f0ed 100644 --- a/tests/unit/test_seqeraplatform.py +++ b/tests/unit/test_seqeraplatform.py @@ -91,6 +91,22 @@ def test_resource_creation_error(self): with self.assertRaises(seqeraplatform.ResourceCreationError): command("import", "my_pipeline.json", "--name", "pipeline_name") + def test_empty_string_argument(self): + command = ["--profile", " ", "--config", "my_config"] + with self.assertRaises(ValueError) as context: + self.sp._check_empty_args(command) + self.assertIn( + "Empty string argument found for parameter '--profile'", + str(context.exception), + ) + + def test_no_empty_string_argument(self): + command = ["--profile", "test_profile", "--config", "my_config"] + try: + self.sp._check_empty_args(command) + except ValueError: + self.fail("_check_empty_args() raised ValueError unexpectedly!") + def test_json_parsing(self): with patch("subprocess.Popen") as mock_subprocess: # Mock the stdout of the Popen process to return JSON From a93a195df23a173fe932c45329c5877c4e52fbcd Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 15 Jul 2024 21:47:37 -0400 Subject: [PATCH 02/12] fix: remove defaults channel from install and build re #150 --- .github/workflows/teardown.yml | 2 +- README.md | 1 - environment.yaml | 9 +++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 environment.yaml diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml index a44b93c..12a8a9a 100644 --- a/.github/workflows/teardown.yml +++ b/.github/workflows/teardown.yml @@ -36,7 +36,7 @@ jobs: environment-file: environment.yml python-version: '3.12' mamba-version: '*' - channels: conda-forge,bioconda,defaults + channels: conda-forge,bioconda activate-environment: seqerakit use-mamba: true diff --git a/README.md b/README.md index ad31f84..b65ad07 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt You can install `seqerakit` and its dependencies via Conda. Ensure that you have the correct channels configured: ```console -conda config --add channels defaults conda config --add channels bioconda conda config --add channels conda-forge conda config --set channel_priority strict diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..374af80 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,9 @@ +name: seqerakitdev +channels: + - bioconda + - conda-forge + - bioconda +dependencies: + - conda-forge::python=3.10.9 + - conda-forge::pyyaml=6.0 + - bioconda::tower-cli=0.9.2 From b03b092fe5ef4fcd0ddf96030db8f76a299f0af1 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 15 Jul 2024 21:49:50 -0400 Subject: [PATCH 03/12] fix: remove duplicate bioconda in channels --- environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index 374af80..0de4e09 100644 --- a/environment.yaml +++ b/environment.yaml @@ -2,7 +2,6 @@ name: seqerakitdev channels: - bioconda - conda-forge - - bioconda dependencies: - conda-forge::python=3.10.9 - conda-forge::pyyaml=6.0 From 35972dbc45e012f435fb2effd41fe3fb0042e10a Mon Sep 17 00:00:00 2001 From: ejseqera Date: Wed, 31 Jul 2024 16:59:49 -0400 Subject: [PATCH 04/12] fix: improve logic to respect cli and dryrun flags with info fix #151 --- seqerakit/cli.py | 14 +++++++------- seqerakit/seqeraplatform.py | 6 +++--- tests/unit/test_seqeraplatform.py | 26 ++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index 0bb29e6..b0b7c43 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -145,10 +145,15 @@ def main(args=None): options = parse_args(args if args is not None else sys.argv[1:]) logging.basicConfig(level=options.log_level) + # Parse CLI arguments into a list and create a Seqera Platform instance + cli_args_list = options.cli_args.split() if options.cli_args else [] + sp = seqeraplatform.SeqeraPlatform(cli_args=cli_args_list, dryrun=options.dryrun) + # If the info flag is set, run 'tw info' if options.info: - sp = seqeraplatform.SeqeraPlatform() - print(sp.info()) + result = sp.info() + if not options.dryrun: + print(result) return if not options.yaml: @@ -161,11 +166,6 @@ def main(args=None): else: options.yaml = [sys.stdin] - # Parse CLI arguments into a list - cli_args_list = options.cli_args.split() if options.cli_args else [] - - sp = seqeraplatform.SeqeraPlatform(cli_args=cli_args_list, dryrun=options.dryrun) - block_manager = BlockParser( sp, [ diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index a08f816..4a62bb0 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -112,10 +112,10 @@ def _execute_command(self, full_cmd, to_json=False): return json.loads(stdout) if to_json else stdout - def _execute_info_command(self): + def _execute_info_command(self, *args, **kwargs): # Directly execute 'tw info' command - command = "tw info" - return self._execute_command(command) + command = ["info"] + return self._tw_run(command, *args, **kwargs) def _handle_command_errors(self, stdout): logging.error(stdout) diff --git a/tests/unit/test_seqeraplatform.py b/tests/unit/test_seqeraplatform.py index 941f0ed..2173afb 100644 --- a/tests/unit/test_seqeraplatform.py +++ b/tests/unit/test_seqeraplatform.py @@ -186,6 +186,32 @@ def test_cli_args_exclusion_of_verbose(self): # TODO: remove this test once fix "--verbose is not supported as a CLI argument to seqerakit.", ) + @patch("subprocess.Popen") + def test_info_command_construction(self, mock_subprocess): + # Mock the subprocess call to prevent actual execution + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = (b"", b"") + + self.sp.info() + called_command = mock_subprocess.call_args[0][0] + + # Check if the constructed command is correct + expected_command_part = "tw --url http://tower-api.com --insecure info" + self.assertIn(expected_command_part, called_command) + + # Check if the cli_args are included in the called command + for arg in self.cli_args: + self.assertIn(arg, called_command) + + @patch("subprocess.Popen") + def test_info_command_dryrun(self, mock_subprocess): + # Initialize SeqeraPlatform with dryrun enabled + self.sp.dryrun = True + self.sp.info() + + # Check that subprocess.Popen is not called + mock_subprocess.assert_not_called() + class TestKitOptions(unittest.TestCase): def setUp(self): From e34215ef6665c618758141a63ca3833fb8875674 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Wed, 31 Jul 2024 18:52:40 -0400 Subject: [PATCH 05/12] fix: improve logging and provide informative stdout cmd outputs re #152 --- seqerakit/cli.py | 15 +++++---------- seqerakit/seqeraplatform.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index b0b7c43..81798a0 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -143,7 +143,7 @@ def handle_block(self, block, args, destroy=False, dryrun=False): def main(args=None): options = parse_args(args if args is not None else sys.argv[1:]) - logging.basicConfig(level=options.log_level) + logging.basicConfig(level=getattr(logging, options.log_level.upper())) # Parse CLI arguments into a list and create a Seqera Platform instance cli_args_list = options.cli_args.split() if options.cli_args else [] @@ -186,15 +186,10 @@ def main(args=None): cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) for block, args_list in cmd_args_dict.items(): for args in args_list: - try: - # Run the 'tw' methods for each block - block_manager.handle_block( - block, args, destroy=options.delete, dryrun=options.dryrun - ) - except (ResourceExistsError, ResourceCreationError) as e: - logging.error(e) - sys.exit(1) - except ValueError as e: + block_manager.handle_block( + block, args, destroy=options.delete, dryrun=options.dryrun + ) + except (ResourceExistsError, ResourceCreationError, ValueError) as e: logging.error(e) sys.exit(1) diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index 4a62bb0..0a2e05b 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -19,8 +19,6 @@ import re import json -logging.basicConfig(level=logging.DEBUG) - class SeqeraPlatform: """ @@ -99,14 +97,17 @@ def _check_env_vars(self, command): return " ".join(full_cmd_parts) # Executes a 'tw' command in a subprocess and returns the output. - def _execute_command(self, full_cmd, to_json=False): - logging.debug(f" Running command: {full_cmd}") + def _execute_command(self, full_cmd, to_json=False, print_stdout=True): + logging.info(f" Running command: {full_cmd}") process = subprocess.Popen( full_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True ) stdout, _ = process.communicate() stdout = stdout.decode("utf-8").strip() + if print_stdout: + logging.info(f" Command output: {stdout}") + if "ERROR: " in stdout or process.returncode != 0: self._handle_command_errors(str(stdout)) @@ -115,11 +116,9 @@ def _execute_command(self, full_cmd, to_json=False): def _execute_info_command(self, *args, **kwargs): # Directly execute 'tw info' command command = ["info"] - return self._tw_run(command, *args, **kwargs) + return self._tw_run(command, *args, **kwargs, print_stdout=False) def _handle_command_errors(self, stdout): - logging.error(stdout) - # Check for specific tw cli error patterns and raise custom exceptions if re.search( r"ERROR: .*already (exists|a participant)", stdout, flags=re.IGNORECASE @@ -134,11 +133,12 @@ def _handle_command_errors(self, stdout): ) def _tw_run(self, cmd, *args, **kwargs): + print_stout = kwargs.pop("print_stdout", True) full_cmd = self._construct_command(cmd, *args, **kwargs) if not full_cmd or self.dryrun: logging.debug(f"DRYRUN: Running command {full_cmd}") return - return self._execute_command(full_cmd, kwargs.get("to_json")) + return self._execute_command(full_cmd, kwargs.get("to_json"), print_stout) # Allow any 'tw' subcommand to be called as a method. def __getattr__(self, cmd): From 7842033fb283c3766f80d75089971f8cbf136260 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 1 Aug 2024 21:04:23 -0400 Subject: [PATCH 06/12] fix: add context manager to handle supression of overwrite stdout --- seqerakit/overwrite.py | 34 +++++++++++++++++++--------------- seqerakit/seqeraplatform.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index 4be03a5..d98d916 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -117,13 +117,13 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False): if self.check_resource_exists(operation["name_key"], sp_args): # if resource exists and overwrite is true, delete if overwrite: - logging.debug( + logging.info( f" The attempted {block} resource already exists." " Overwriting.\n" ) self.delete_resource(block, operation, sp_args) elif destroy: - logging.debug(f" Deleting the {block} resource.") + logging.info(f" Deleting the {block} resource.") self.delete_resource(block, operation, sp_args) else: # return an error if resource exists, overwrite=False raise ResourceExistsError( @@ -147,7 +147,8 @@ def _get_team_args(self, args): if not jsondata: json_method = getattr(self.sp, "-o json") - json_out = json_method("teams", "list", "-o", args["organization"]) + with self.sp.suppress_output(): + json_out = json_method("teams", "list", "-o", args["organization"]) self.block_jsondata["teams"] = json_out else: json_out = jsondata @@ -244,27 +245,30 @@ def _get_json_data(self, block, args, keys_to_get): # Fetch the data if it does not exist if block == "teams": sp_args = self._get_values_from_cmd_args(args[0], keys_to_get) - self.cached_jsondata = json_method( - block, "list", "-o", sp_args["organization"] - ) + with self.sp.suppress_output(): + self.cached_jsondata = json_method( + block, "list", "-o", sp_args["organization"] + ) elif block in Overwrite.generic_deletion or block in { "participants", "labels", }: sp_args = self._get_values_from_cmd_args(args, keys_to_get) - self.cached_jsondata = json_method( - block, "list", "-w", sp_args["workspace"] - ) - elif block == "members" or block == "workspaces": # TODO + with self.sp.suppress_output(): + self.cached_jsondata = json_method( + block, "list", "-w", sp_args["workspace"] + ) + elif block == "members" or block == "workspaces": sp_args = self._get_values_from_cmd_args(args, keys_to_get) - self.cached_jsondata = json_method( - block, "list", "-o", sp_args["organization"] - ) + with self.sp.suppress_output(): + self.cached_jsondata = json_method( + block, "list", "-o", sp_args["organization"] + ) else: sp_args = self._get_values_from_cmd_args(args, keys_to_get) - self.cached_jsondata = json_method(block, "list") + with self.sp.suppress_output(): + self.cached_jsondata = json_method(block, "list") - # Store this data in the block_jsondata dict for later use self.block_jsondata[block] = self.cached_jsondata return self.cached_jsondata, sp_args diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index 0a2e05b..e12ad31 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from contextlib import contextmanager import os import shlex import logging @@ -42,13 +43,15 @@ def __call__(self, *args, **kwargs): return self.tw_instance._tw_run(command, **kwargs) # Constructs a new SeqeraPlatform instance - def __init__(self, cli_args=None, dryrun=False): + def __init__(self, cli_args=None, dryrun=False, print_stdout=True): if cli_args and "--verbose" in cli_args: raise ValueError( "--verbose is not supported as a CLI argument to seqerakit." ) self.cli_args = cli_args or [] self.dryrun = dryrun + self.print_stdout = print_stdout + self._suppress_output = False def _construct_command(self, cmd, *args, **kwargs): command = ["tw"] + self.cli_args @@ -105,11 +108,15 @@ def _execute_command(self, full_cmd, to_json=False, print_stdout=True): stdout, _ = process.communicate() stdout = stdout.decode("utf-8").strip() - if print_stdout: + should_print = ( + print_stdout if print_stdout is not None else self.print_stdout + ) and not self._suppress_output + + if should_print: logging.info(f" Command output: {stdout}") if "ERROR: " in stdout or process.returncode != 0: - self._handle_command_errors(str(stdout)) + self._handle_command_errors(stdout) return json.loads(stdout) if to_json else stdout @@ -133,19 +140,31 @@ def _handle_command_errors(self, stdout): ) def _tw_run(self, cmd, *args, **kwargs): - print_stout = kwargs.pop("print_stdout", True) + print_stdout = kwargs.pop("print_stdout", None) full_cmd = self._construct_command(cmd, *args, **kwargs) if not full_cmd or self.dryrun: - logging.debug(f"DRYRUN: Running command {full_cmd}") + logging.info(f"DRYRUN: Running command {full_cmd}") return - return self._execute_command(full_cmd, kwargs.get("to_json"), print_stout) + return self._execute_command(full_cmd, kwargs.get("to_json"), print_stdout) + + @contextmanager + def suppress_output(self): + original_suppress = self._suppress_output + self._suppress_output = True + try: + yield + finally: + self._suppress_output = original_suppress # Allow any 'tw' subcommand to be called as a method. def __getattr__(self, cmd): if cmd == "info": return self._execute_info_command - else: - return self.TwCommand(self, cmd.replace("_", "-")) + if cmd == "-o json": + return lambda *args, **kwargs: self._tw_run( + ["-o", "json"] + list(args), **kwargs + ) + return self.TwCommand(self, cmd.replace("_", "-")) class ResourceExistsError(Exception): From d3a208fb7d40ca457665ed21c59267d6268c866e Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 1 Aug 2024 21:07:42 -0400 Subject: [PATCH 07/12] refactor: remove and refactor separate info method --- seqerakit/seqeraplatform.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index e12ad31..5268a5f 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -120,11 +120,6 @@ def _execute_command(self, full_cmd, to_json=False, print_stdout=True): return json.loads(stdout) if to_json else stdout - def _execute_info_command(self, *args, **kwargs): - # Directly execute 'tw info' command - command = ["info"] - return self._tw_run(command, *args, **kwargs, print_stdout=False) - def _handle_command_errors(self, stdout): # Check for specific tw cli error patterns and raise custom exceptions if re.search( @@ -159,7 +154,9 @@ def suppress_output(self): # Allow any 'tw' subcommand to be called as a method. def __getattr__(self, cmd): if cmd == "info": - return self._execute_info_command + return lambda *args, **kwargs: self._tw_run( + ["info"], *args, **kwargs, print_stdout=False + ) if cmd == "-o json": return lambda *args, **kwargs: self._tw_run( ["-o", "json"] + list(args), **kwargs From 352b13b858ddc4a6e4cb2b5a330dec034ec69883 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 1 Aug 2024 21:17:28 -0400 Subject: [PATCH 08/12] test: add unit tests --- tests/unit/test_seqeraplatform.py | 104 ++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/unit/test_seqeraplatform.py b/tests/unit/test_seqeraplatform.py index 2173afb..ae1d34a 100644 --- a/tests/unit/test_seqeraplatform.py +++ b/tests/unit/test_seqeraplatform.py @@ -4,6 +4,8 @@ import json import subprocess import os +import logging +from io import StringIO class TestSeqeraPlatform(unittest.TestCase): @@ -268,5 +270,107 @@ def test_error_raised_for_unset_env_vars(self): ) +class TestSeqeraPlatformOutputHandling(unittest.TestCase): + def setUp(self): + self.sp = seqeraplatform.SeqeraPlatform() + # Set up logging to capture output + self.log_capture = StringIO() + self.log_handler = logging.StreamHandler(self.log_capture) + logging.getLogger().addHandler(self.log_handler) + logging.getLogger().setLevel(logging.INFO) + + def tearDown(self): + logging.getLogger().removeHandler(self.log_handler) + logging.getLogger().setLevel(logging.NOTSET) + + @patch("subprocess.Popen") + def test_suppress_output(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = ( + b'{"key": "value"}', + b"", + ) + + log_capture = StringIO() + logging.getLogger().addHandler(logging.StreamHandler(log_capture)) + + with self.sp.suppress_output(): + self.sp.pipelines("list") + + log_contents = log_capture.getvalue() + self.assertIn("Running command:", log_contents) + self.assertNotIn("Command output:", log_contents) + + @patch("subprocess.Popen") + def test_suppress_output_context(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = ( + b'{"key": "value"}', + b"", + ) + + # Test that stdout is suppressed within the context manager + with self.sp.suppress_output(): + result = self.sp._execute_command("tw pipelines list", to_json=True) + self.assertEqual(result, {"key": "value"}) + + # Test that stdout is not suppressed outside the context manager + result = self.sp._execute_command("tw pipelines list", to_json=True) + self.assertEqual(result, {"key": "value"}) + + @patch("subprocess.Popen") + def test_json_output_handling(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = ( + b'{"key": "value"}', + b"", + ) + + result = self.sp._execute_command("tw pipelines list", to_json=True) + self.assertEqual(result, {"key": "value"}) + + result = self.sp._execute_command("tw pipelines list", to_json=False) + self.assertEqual(result, '{"key": "value"}') + + @patch("subprocess.Popen") + def test_print_stdout_override(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = (b"output", b"") + + # Test with print_stdout=True + self.sp._execute_command("tw pipelines list", print_stdout=True) + log_output = self.log_capture.getvalue() + self.assertIn("Command output: output", log_output) + + # Clear the log capture + self.log_capture.truncate(0) + self.log_capture.seek(0) + + # Test with print_stdout=False + self.sp._execute_command("tw pipelines list", print_stdout=False) + log_output = self.log_capture.getvalue() + self.assertNotIn("Command output: output", log_output) + + @patch("subprocess.Popen") + def test_error_handling_with_suppressed_output(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=1) + mock_subprocess.return_value.communicate.return_value = ( + b"ERROR: Something went wrong", + b"", + ) + + with self.assertRaises(seqeraplatform.ResourceCreationError): + with self.sp.suppress_output(): + self.sp._execute_command("tw pipelines list") + + @patch("subprocess.Popen") + def test_json_parsing_error(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = (b"Invalid JSON", b"") + + with self.assertRaises(json.JSONDecodeError): + self.sp._execute_command("tw pipelines list", to_json=True) + + if __name__ == "__main__": unittest.main() From c34bcf2b3ca4d8646907e8c11f56b636b6ef265e Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 1 Aug 2024 21:32:20 -0400 Subject: [PATCH 09/12] feat: add cli flag to specify targets re #148 --- seqerakit/cli.py | 11 ++++++++++- seqerakit/helper.py | 7 ++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index 81798a0..e9bcbdf 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -80,6 +80,13 @@ def parse_args(args=None): help="Additional Seqera Platform CLI specific options to be passed," " enclosed in double quotes (e.g. '--cli=\"--insecure\"').", ) + yaml_processing.add_argument( + "--targets", + dest="targets", + type=str, + help="Specify the resources to be targeted for creation in a YAML file through " + "a comma-separated list (e.g. '--targets=teams,participants').", + ) return parser.parse_args(args) @@ -183,7 +190,9 @@ def main(args=None): # Parse the YAML file(s) by blocks # and get a dictionary of command line arguments try: - cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) + cmd_args_dict = helper.parse_all_yaml( + options.yaml, destroy=options.delete, targets=options.targets + ) for block, args_list in cmd_args_dict.items(): for args in args_list: block_manager.handle_block( diff --git a/seqerakit/helper.py b/seqerakit/helper.py index c4792af..86a71a1 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -55,7 +55,7 @@ def parse_yaml_block(yaml_data, block_name): return block_name, cmd_args_list -def parse_all_yaml(file_paths, destroy=False): +def parse_all_yaml(file_paths, destroy=False, targets=None): # If multiple yamls, merge them into one dictionary merged_data = {} @@ -108,6 +108,11 @@ def parse_all_yaml(file_paths, destroy=False): block_names = list(merged_data.keys()) + # Filter blocks based on targets if provided + if targets: + target_blocks = set(targets.split(",")) + block_names = [block for block in block_names if block in target_blocks] + # Define the order in which the resources should be created. resource_order = [ "organizations", From 3a87f4435bc1705abdb342497e96df432106fee6 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 1 Aug 2024 21:37:39 -0400 Subject: [PATCH 10/12] test: add unit tests --- tests/unit/test_helper.py | 70 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 76f5b55..73ce6d5 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -442,3 +442,73 @@ def test_error_duplicate_name_yaml_file(mock_yaml_file): "compute-envs: test_computeenv. Please specify " "a unique value." in str(e.value) ) + + +def test_targets_specified(): + # mock YAML data + yaml_data = """ +organizations: + - name: org1 + description: Organization 1 +workspaces: + - name: workspace1 + organization: org1 + description: Workspace 1 +pipelines: + - name: pipeline1 + workspace: workspace1 + description: Pipeline 1 +""" + with patch("builtins.open", lambda f, _: StringIO(yaml_data)): + result = helper.parse_all_yaml( + ["dummy_path.yaml"], targets="organizations,workspaces" + ) + + expected_organizations_output = [ + { + "cmd_args": ["--name", "org1", "--description", "Organization 1"], + "overwrite": False, + } + ] + expected_workspaces_output = [ + { + "cmd_args": [ + "--name", + "workspace1", + "--organization", + "org1", + "--description", + "Workspace 1", + ], + "overwrite": False, + } + ] + # Check that only 'organizations' and 'workspaces' are in the result + assert "organizations" in result + assert result["organizations"] == expected_organizations_output + assert "workspaces" in result + assert result["workspaces"] == expected_workspaces_output + assert "pipelines" not in result + + +def test_no_targets_specified(): + yaml_data = """ +organizations: + - name: org1 + description: Organization 1 +workspaces: + - name: workspace1 + organization: org1 + description: Workspace 1 +pipelines: + - name: pipeline1 + workspace: workspace1 + description: Pipeline 1 +""" + with patch("builtins.open", lambda f, _: StringIO(yaml_data)): + result = helper.parse_all_yaml(["dummy_path.yaml"]) + + # Check that all blocks are in the result + assert "organizations" in result + assert "workspaces" in result + assert "pipelines" in result From 2b95ab6c1355d2661565280d44f3420ebfd16584 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 2 Aug 2024 13:25:04 -0400 Subject: [PATCH 11/12] build: bump version in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84680f3..ee30640 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.8" +VERSION = "0.4.9" with open("README.md") as f: readme = f.read() From 2e2cc69a351639ea1e7c9b8bbf008a9dbbf5cd5f Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 2 Aug 2024 13:33:53 -0400 Subject: [PATCH 12/12] docs: add docs for targets flag --- README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/README.md b/README.md index b65ad07..866b020 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,43 @@ seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/pat Note: Use of `--verbose` option for the `tw` CLI is currently not supported by `seqerakit`. Supplying `--cli="--verbose"` will raise an error. +## Specify targets +When using a YAML file as input that defines multiple resources, you can use the `--targets` flag to specify which resources to create. This flag takes a comma-separated list of resource names. + +For example, given a YAML file that defines the following resources: + +```yaml +workspaces: + - name: 'showcase' + organization: 'seqerakit_automation' +... +compute-envs: + - name: 'compute-env' + type: 'aws-batch forge' + workspace: 'seqerakit/test' +... +pipelines: + - name: "hello-world-test-seqerakit" + url: "https://github.com/nextflow-io/hello" + workspace: 'seqerakit/test' + compute-env: "compute-env" +... +``` + +You can target the creation of `pipelines` only by running: + +```bash +seqerakit test.yml --targets pipelines +``` +This will process only the pipelines block from the YAML file and ignore other blocks such as `workspaces` and `compute-envs`. + +### Multiple Targets +You can also specify multiple resources to create by separating them with commas. For example, to create both workspaces and pipelines, run: + +```bash +seqerakit test.yml --targets workspaces,pipelines +``` + ## YAML Configuration Options There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not exposed as `tw` CLI options.