Skip to content

Commit

Permalink
Merge branch 'dev' into json_output_to_stdout
Browse files Browse the repository at this point in the history
  • Loading branch information
adamrtalbot committed Aug 29, 2024
2 parents 21d7bf0 + 130cf97 commit 0271c38
Show file tree
Hide file tree
Showing 10 changed files with 358 additions and 58 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/teardown.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
environment-file: environment.yml
python-version: '3.12'
mamba-version: '*'
channels: conda-forge,bioconda,defaults
channels: conda-forge,bioconda
activate-environment: seqerakit
use-mamba: true

Expand Down
38 changes: 37 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt
You can install `seqerakit` and its dependencies via Conda. Ensure that you have the correct channels configured:

```console
conda config --add channels defaults
conda config --add channels bioconda
conda config --add channels conda-forge
conda config --set channel_priority strict
Expand Down Expand Up @@ -170,6 +169,43 @@ seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/pat

<b>Note</b>: Use of `--verbose` option for the `tw` CLI is currently not supported by `seqerakit`. Supplying `--cli="--verbose"` will raise an error.

## Specify targets
When using a YAML file as input that defines multiple resources, you can use the `--targets` flag to specify which resources to create. This flag takes a comma-separated list of resource names.

For example, given a YAML file that defines the following resources:

```yaml
workspaces:
- name: 'showcase'
organization: 'seqerakit_automation'
...
compute-envs:
- name: 'compute-env'
type: 'aws-batch forge'
workspace: 'seqerakit/test'
...
pipelines:
- name: "hello-world-test-seqerakit"
url: "https://github.com/nextflow-io/hello"
workspace: 'seqerakit/test'
compute-env: "compute-env"
...
```

You can target the creation of `pipelines` only by running:

```bash
seqerakit test.yml --targets pipelines
```
This will process only the pipelines block from the YAML file and ignore other blocks such as `workspaces` and `compute-envs`.

### Multiple Targets
You can also specify multiple resources to create by separating them with commas. For example, to create both workspaces and pipelines, run:

```bash
seqerakit test.yml --targets workspaces,pipelines
```

## YAML Configuration Options

There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not exposed as `tw` CLI options.
Expand Down
8 changes: 8 additions & 0 deletions environment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
name: seqerakitdev
channels:
- bioconda
- conda-forge
dependencies:
- conda-forge::python=3.10.9
- conda-forge::pyyaml=6.0
- bioconda::tower-cli=0.9.2
45 changes: 25 additions & 20 deletions seqerakit/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,13 @@ def parse_args(args=None):
help="Additional Seqera Platform CLI specific options to be passed,"
" enclosed in double quotes (e.g. '--cli=\"--insecure\"').",
)
yaml_processing.add_argument(
"--targets",
dest="targets",
type=str,
help="Specify the resources to be targeted for creation in a YAML file through "
"a comma-separated list (e.g. '--targets=teams,participants').",
)
return parser.parse_args(args)


Expand Down Expand Up @@ -146,12 +153,20 @@ def handle_block(self, block, args, destroy=False, dryrun=False):

def main(args=None):
options = parse_args(args if args is not None else sys.argv[1:])
logging.basicConfig(level=options.log_level)
logging.basicConfig(level=getattr(logging, options.log_level.upper()))

# Parse CLI arguments into a list
cli_args_list = options.cli_args.split() if options.cli_args else []

sp = seqeraplatform.SeqeraPlatform(
cli_args=cli_args_list, dryrun=options.dryrun, json=options.json
)

# If the info flag is set, run 'tw info'
if options.info:
sp = seqeraplatform.SeqeraPlatform()
print(sp.info())
result = sp.info()
if not options.dryrun:
print(result)
return

if not options.yaml:
Expand All @@ -164,13 +179,6 @@ def main(args=None):
else:
options.yaml = [sys.stdin]

# Parse CLI arguments into a list
cli_args_list = options.cli_args.split() if options.cli_args else []

sp = seqeraplatform.SeqeraPlatform(
cli_args=cli_args_list, dryrun=options.dryrun, json=options.json
)

block_manager = BlockParser(
sp,
[
Expand All @@ -188,18 +196,15 @@ def main(args=None):
# Parse the YAML file(s) by blocks
# and get a dictionary of command line arguments
try:
cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete)
cmd_args_dict = helper.parse_all_yaml(
options.yaml, destroy=options.delete, targets=options.targets
)
for block, args_list in cmd_args_dict.items():
for args in args_list:
try:
# Run the 'tw' methods for each block
block_manager.handle_block(
block, args, destroy=options.delete, dryrun=options.dryrun
)
except (ResourceExistsError, ResourceCreationError) as e:
logging.error(e)
sys.exit(1)
except ValueError as e:
block_manager.handle_block(
block, args, destroy=options.delete, dryrun=options.dryrun
)
except (ResourceExistsError, ResourceCreationError, ValueError) as e:
logging.error(e)
sys.exit(1)

Expand Down
7 changes: 6 additions & 1 deletion seqerakit/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def parse_yaml_block(yaml_data, block_name):
return block_name, cmd_args_list


def parse_all_yaml(file_paths, destroy=False):
def parse_all_yaml(file_paths, destroy=False, targets=None):
# If multiple yamls, merge them into one dictionary
merged_data = {}

Expand Down Expand Up @@ -108,6 +108,11 @@ def parse_all_yaml(file_paths, destroy=False):

block_names = list(merged_data.keys())

# Filter blocks based on targets if provided
if targets:
target_blocks = set(targets.split(","))
block_names = [block for block in block_names if block in target_blocks]

# Define the order in which the resources should be created.
resource_order = [
"organizations",
Expand Down
34 changes: 19 additions & 15 deletions seqerakit/overwrite.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,13 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False):
if self.check_resource_exists(operation["name_key"], sp_args):
# if resource exists and overwrite is true, delete
if overwrite:
logging.debug(
logging.info(
f" The attempted {block} resource already exists."
" Overwriting.\n"
)
self.delete_resource(block, operation, sp_args)
elif destroy:
logging.debug(f" Deleting the {block} resource.")
logging.info(f" Deleting the {block} resource.")
self.delete_resource(block, operation, sp_args)
else: # return an error if resource exists, overwrite=False
raise ResourceExistsError(
Expand All @@ -147,7 +147,8 @@ def _get_team_args(self, args):

if not jsondata:
json_method = getattr(self.sp, "-o json")
json_out = json_method("teams", "list", "-o", args["organization"])
with self.sp.suppress_output():
json_out = json_method("teams", "list", "-o", args["organization"])
self.block_jsondata["teams"] = json_out
else:
json_out = jsondata
Expand Down Expand Up @@ -244,27 +245,30 @@ def _get_json_data(self, block, args, keys_to_get):
# Fetch the data if it does not exist
if block == "teams":
sp_args = self._get_values_from_cmd_args(args[0], keys_to_get)
self.cached_jsondata = json_method(
block, "list", "-o", sp_args["organization"]
)
with self.sp.suppress_output():
self.cached_jsondata = json_method(
block, "list", "-o", sp_args["organization"]
)
elif block in Overwrite.generic_deletion or block in {
"participants",
"labels",
}:
sp_args = self._get_values_from_cmd_args(args, keys_to_get)
self.cached_jsondata = json_method(
block, "list", "-w", sp_args["workspace"]
)
elif block == "members" or block == "workspaces": # TODO
with self.sp.suppress_output():
self.cached_jsondata = json_method(
block, "list", "-w", sp_args["workspace"]
)
elif block == "members" or block == "workspaces":
sp_args = self._get_values_from_cmd_args(args, keys_to_get)
self.cached_jsondata = json_method(
block, "list", "-o", sp_args["organization"]
)
with self.sp.suppress_output():
self.cached_jsondata = json_method(
block, "list", "-o", sp_args["organization"]
)
else:
sp_args = self._get_values_from_cmd_args(args, keys_to_get)
self.cached_jsondata = json_method(block, "list")
with self.sp.suppress_output():
self.cached_jsondata = json_method(block, "list")

# Store this data in the block_jsondata dict for later use
self.block_jsondata[block] = self.cached_jsondata
return self.cached_jsondata, sp_args

Expand Down
64 changes: 45 additions & 19 deletions seqerakit/seqeraplatform.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from contextlib import contextmanager
import os
import shlex
import logging
import subprocess
import re
import json

logging.basicConfig(level=logging.DEBUG)


class SeqeraPlatform:
"""
Expand All @@ -44,14 +43,16 @@ def __call__(self, *args, **kwargs):
return self.tw_instance._tw_run(command, **kwargs)

# Constructs a new SeqeraPlatform instance
def __init__(self, cli_args=None, dryrun=False, json=False):
def __init__(self, cli_args=None, dryrun=False, print_stdout=True, json=False):
if cli_args and "--verbose" in cli_args:
raise ValueError(
"--verbose is not supported as a CLI argument to seqerakit."
)
self.cli_args = cli_args or []
self.dryrun = dryrun
self.print_stdout = print_stdout
self.json = json
self._suppress_output = False

def _construct_command(self, cmd, *args, **kwargs):
command = ["tw"] + self.cli_args
Expand All @@ -68,8 +69,19 @@ def _construct_command(self, cmd, *args, **kwargs):
if "params_file" in kwargs:
command.append(f"--params-file={kwargs['params_file']}")

# Check for empty string arguments and handle them
self._check_empty_args(command)

return self._check_env_vars(command)

def _check_empty_args(self, command):
for current_arg, next_arg in zip(command, command[1:]):
if isinstance(next_arg, str) and next_arg.strip() == "":
raise ValueError(
f"Empty string argument found for parameter '{current_arg}'. "
"Please provide a valid value or remove the argument."
)

# Checks environment variables to see that they are set accordingly
def _check_env_vars(self, command):
full_cmd_parts = []
Expand All @@ -89,16 +101,23 @@ def _check_env_vars(self, command):
return " ".join(full_cmd_parts)

# Executes a 'tw' command in a subprocess and returns the output.
def _execute_command(self, full_cmd):
logging.debug(f" Running command: {full_cmd}")
def _execute_command(self, full_cmd, to_json=False, print_stdout=True):
logging.info(f" Running command: {full_cmd}")
process = subprocess.Popen(
full_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True
)
stdout, _ = process.communicate()
stdout = stdout.decode("utf-8").strip()

should_print = (
print_stdout if print_stdout is not None else self.print_stdout
) and not self._suppress_output

if should_print:
logging.info(f" Command output: {stdout}")

if "ERROR: " in stdout or process.returncode != 0:
self._handle_command_errors(str(stdout))
self._handle_command_errors(stdout)

if self.json:
out = json.loads(stdout)
Expand All @@ -109,14 +128,7 @@ def _execute_command(self, full_cmd):

return out

def _execute_info_command(self):
# Directly execute 'tw info' command
command = "tw info"
return self._execute_command(command)

def _handle_command_errors(self, stdout):
logging.error(stdout)

# Check for specific tw cli error patterns and raise custom exceptions
if re.search(
r"ERROR: .*already (exists|a participant)", stdout, flags=re.IGNORECASE
Expand All @@ -131,19 +143,33 @@ def _handle_command_errors(self, stdout):
)

def _tw_run(self, cmd, *args, **kwargs):
print_stdout = kwargs.pop("print_stdout", None)
full_cmd = self._construct_command(cmd, *args, **kwargs)
if not full_cmd or self.dryrun:
logging.debug(f"DRYRUN: Running command {full_cmd}")
logging.info(f"DRYRUN: Running command {full_cmd}")
return None
result = self._execute_command(full_cmd)
return result
return self._execute_command(full_cmd, kwargs.get("to_json"), print_stdout)

@contextmanager
def suppress_output(self):
original_suppress = self._suppress_output
self._suppress_output = True
try:
yield
finally:
self._suppress_output = original_suppress

# Allow any 'tw' subcommand to be called as a method.
def __getattr__(self, cmd):
if cmd == "info":
return self._execute_info_command
else:
return self.TwCommand(self, cmd.replace("_", "-"))
return lambda *args, **kwargs: self._tw_run(
["info"], *args, **kwargs, print_stdout=False
)
if cmd == "-o json":
return lambda *args, **kwargs: self._tw_run(
["-o", "json"] + list(args), **kwargs
)
return self.TwCommand(self, cmd.replace("_", "-"))


class ResourceExistsError(Exception):
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from setuptools import find_packages, setup

VERSION = "0.4.7"
VERSION = "0.4.9"

with open("README.md") as f:
readme = f.read()
Expand Down
Loading

0 comments on commit 0271c38

Please sign in to comment.