From 1e271b58f9fba87b1b684823507e1eb702825b99 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 6 Aug 2024 18:57:45 +1000 Subject: [PATCH 1/4] streamlining the syntax for nif workshop --- pydra2app/xnat/cli/__init__.py | 2 +- pydra2app/xnat/cli/release.py | 365 +++++++++++++++++++ pydra2app/xnat/cli/update_release.py | 147 -------- pydra2app/xnat/deploy.py | 256 +++++++++++++ pydra2app/xnat/image.py | 12 +- pydra2app/xnat/testing.py | 134 ------- pydra2app/xnat/tests/test_app.py | 4 +- pydra2app/xnat/tests/test_cli_xnat_images.py | 35 +- scripts/quick_test.py | 24 ++ 9 files changed, 671 insertions(+), 308 deletions(-) create mode 100644 pydra2app/xnat/cli/release.py delete mode 100644 pydra2app/xnat/cli/update_release.py create mode 100644 pydra2app/xnat/deploy.py delete mode 100644 pydra2app/xnat/testing.py create mode 100644 scripts/quick_test.py diff --git a/pydra2app/xnat/cli/__init__.py b/pydra2app/xnat/cli/__init__.py index a79fde9..84c36e8 100644 --- a/pydra2app/xnat/cli/__init__.py +++ b/pydra2app/xnat/cli/__init__.py @@ -1,3 +1,3 @@ from .base import xnat_group from .entrypoint import cs_entrypoint -from .update_release import pull_xnat_images, xnat_auth_refresh +from .release import pull_xnat_images, xnat_auth_refresh diff --git a/pydra2app/xnat/cli/release.py b/pydra2app/xnat/cli/release.py new file mode 100644 index 0000000..001ad57 --- /dev/null +++ b/pydra2app/xnat/cli/release.py @@ -0,0 +1,365 @@ +import re +import json +from pathlib import Path +import yaml +import logging +import click +import xnat +import os +from .base import xnat_group +from ..deploy import install_cs_command, launch_cs_command + + +# Configure basic logging +logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + +XNAT_HOST_KEY = "XNAT_HOST" +XNAT_USER_KEY = "XNAT_USER" +XNAT_PASS_KEY = "XNAT_PASS" +XNAT_AUTH_FILE_KEY = "XNAT_AUTH_FILE" +XNAT_AUTH_FILE_DEFAULT = Path("~/.pydra2app_xnat_auth.json").expanduser() + + +def load_auth(server, user, password, auth_file): + if server is None: + if user is None: + raise RuntimeError(f"A user must be provided if a server ({server}) is") + if password is None: + raise RuntimeError(f"a password must be provided if a server ({server}) is") + else: + click.echo(f"Reading existing alias/token pair from '{str(auth_file)}") + with open(auth_file) as fp: + auth = json.load(fp) + user = auth["alias"] + password = auth["secret"] + return server, user, password + + +@xnat_group.command( + name="install-command", + help="""Installs a container service pipelines command on an XNAT server + +IMAGE_OR_COMMAND_FILE the name of the Pydra2App container service pipeline Docker image or +the path to a command JSON file to install +""", +) +@click.argument("image_or_command_file", type=click.File()) +@click.option( + "--enable", + type=bool, + default=False, + help=("Whether to enable the command globally"), +) +@click.option( + "--enable-project", + "enable_projects", + type=str, + multiple=True, + help=("Enable the command for the given project"), +) +@click.option( + "--server", + envvar=XNAT_HOST_KEY, + default=None, + help=("The XNAT server to save install the command on"), +) +@click.option( + "--user", + envvar=XNAT_USER_KEY, + help=("the username used to authenticate with the XNAT instance to update"), +) +@click.option( + "--password", + envvar=XNAT_PASS_KEY, + help=("the password used to authenticate with the XNAT instance to update"), +) +@click.option( + "--auth-file", + type=click.Path(path_type=Path), + default=XNAT_AUTH_FILE_DEFAULT, + envvar=XNAT_AUTH_FILE_KEY, + help=("The path to save the alias/token pair to"), +) +def install_command( + image_or_command_file, enable, projects_to_enable, server, user, password, auth_file +): + server, user, password = load_auth(server, user, password, auth_file) + + if Path(image_or_command_file).exists(): + with open(image_or_command_file) as f: + image_or_command_file = json.load(f) + + with xnat.connect(server=server, user=user, password=password) as xlogin: + install_cs_command( + image_or_command_file, + xlogin=xlogin, + enable=enable, + projects_to_enable=projects_to_enable, + ) + + click.echo( + f"Successfully installed the '{image_or_command_file}' pipeline on '{server}'" + ) + + +@xnat_group.command( + name="launch-command", + help="""Launches a container service pipelines command on an XNAT server + +COMMAND_NAME the name of the command to launch + +PROJECT_ID of the project to launch the command on + +SESSION_ID of the session to launch the command on +""", +) +@click.argument("command_name", type=str) +@click.argument("project_id", type=str) +@click.argument("session_id", type=str) +@click.option( + "--input", + "inputs", + type=(str, str), + multiple=True, + help=("The input values to pass to the command"), +) +@click.option( + "--timeout", + type=int, + default=1000, + help=("The time to wait for the command to complete"), +) +@click.option( + "--poll-interval", + type=int, + default=10, + help=("The time to wait between polling the command status"), +) +@click.option( + "--server", + envvar=XNAT_HOST_KEY, + default=None, + help=("The XNAT server to save install the command on"), +) +@click.option( + "--user", + envvar=XNAT_USER_KEY, + help=("the username used to authenticate with the XNAT instance to update"), +) +@click.option( + "--password", + envvar=XNAT_PASS_KEY, + help=("the password used to authenticate with the XNAT instance to update"), +) +@click.option( + "--auth-file", + type=click.Path(path_type=Path), + default=XNAT_AUTH_FILE_DEFAULT, + envvar=XNAT_AUTH_FILE_KEY, + help=("The path to save the alias/token pair to"), +) +def launch_command( + command_name, + project_id, + session_id, + inputs, + timeout, + poll_interval, + server, + user, + password, + auth_file, +): + + server, user, password = load_auth(server, user, password, auth_file) + + with xnat.connect(server=server, user=user, password=password) as xlogin: + launch_cs_command( + command_name, + project_id=project_id, + session_id=session_id, + inputs=inputs, + timeout=timeout, + poll_interval=poll_interval, + xlogin=xlogin, + ) + + click.echo( + f"Successfully launched the '{command_name}' pipeline on '{session_id}' session " + f"in '{project_id}' project on '{server}'" + ) + + +@xnat_group.command( + name="save-token", + help="""Logs into the XNAT instance, generates a user access token and saves it in an +authentication file. If a username and password are not provided, then it is assumed that +a valid alias/token pair already exists in the authentication file, and they are used to +regenerate a new alias/token pair to prevent them expiring (2 days by default). + +CONFIG_YAML a YAML file contains the login details for the XNAT server to update + +AUTH_FILE the path at which to save the authentication file containing the alias/token +""", +) +@click.option( + "--auth-file", + type=click.Path(path_type=Path), + default=XNAT_AUTH_FILE_DEFAULT, + envvar=XNAT_AUTH_FILE_KEY, + help=("The path to save the alias/token pair to"), +) +@click.option( + "--server", + envvar=XNAT_HOST_KEY, + default=None, + help=("The XNAT server to save the credentials to"), +) +@click.option( + "--user", + envvar=XNAT_USER_KEY, + default=None, + help=("the username used to authenticate with the XNAT instance to update"), +) +@click.option( + "--password", + envvar=XNAT_PASS_KEY, + default=None, + help=("the password used to authenticate with the XNAT instance to update"), +) +def save_token(auth_file, server, user, password): + + server, user, password = load_auth(server, user, password, auth_file) + + with xnat.connect(server=server, user=user, password=password) as xlogin: + alias, secret = xlogin.services.issue_token() + + with open(auth_file, "w") as f: + json.dump( + { + "server": server, + "alias": alias, + "secret": secret, + }, + f, + ) + os.chmod(auth_file, 0o600) + + click.echo( + f"Saved alias/token to XNAT connection token to '{server}' at '{str(auth_file)}', " + "please ensure the file is secure" + ) + + +@xnat_group.command( + name="deploy-pipelines", + help=f"""Updates the installed pipelines on an XNAT instance from a manifest +JSON file using the XNAT instance's REST API. + +MANIFEST_FILE is a JSON file containing a list of container images built in a release +created by `pydra2app deploy xnat build` + +Authentication credentials can be passed through the {XNAT_USER_KEY} +and {XNAT_PASS_KEY} environment variables. Otherwise, tokens can be saved +in a JSON file passed to '--auth'. + +Which of available pipelines to install can be controlled by a YAML file passed to the +'--filters' option of the form + \b + include: + - tag: ghcr.io/Australian-Imaging-Service/mri.human.neuro.* + - tag: ghcr.io/Australian-Imaging-Service/pet.rodent.* + exclude: + - tag: ghcr.io/Australian-Imaging-Service/mri.human.neuro.bidsapps. +""", +) +@click.argument("manifest_file", type=click.File()) +@click.option( + "--server", + envvar=XNAT_HOST_KEY, + default=None, + help=("The XNAT server to save install the command on"), +) +@click.option( + "--user", + envvar=XNAT_USER_KEY, + help=("the username used to authenticate with the XNAT instance to update"), +) +@click.option( + "--password", + envvar=XNAT_PASS_KEY, + help=("the password used to authenticate with the XNAT instance to update"), +) +@click.option( + "--auth-file", + type=click.Path(path_type=Path), + default=XNAT_AUTH_FILE_DEFAULT, + envvar=XNAT_AUTH_FILE_KEY, + help=("The path to save the alias/token pair to"), +) +@click.option( + "--filters", + "filters_file", + default=None, + type=click.File(), + help=("a YAML file containing filter rules for the images to install"), +) +def deploy_pipelines(manifest_file, server, user, password, auth_file, filters_file): + + server, user, password = load_auth(server, user, password, auth_file) + + manifest = json.load(manifest_file) + filters = yaml.load(filters_file, Loader=yaml.Loader) if filters_file else {} + + def matches_entry(entry, match_exprs, default=True): + """Determines whether an entry meets the inclusion and exclusion criteria + + Parameters + ---------- + entry : ty.Dict[str, Any] + a image entry in the manifest + exprs : list[ty.Dict[str, str]] + match criteria + default : bool + the value if match_exprs are empty + """ + if not match_exprs: + return default + return re.match( + "|".join( + i["name"].replace(".", "\\.").replace("*", ".*") for i in match_exprs + ), + entry["name"], + ) + + with xnat.connect( + server=server, + user=user, + password=password, + ) as xlogin: + + for entry in manifest["images"]: + if matches_entry(entry, filters.get("include")) and not matches_entry( + entry, filters.get("exclude"), default=False + ): + tag = f"{entry['name']}:{entry['version']}" + xlogin.post( + "/xapi/docker/pull", query={"image": tag, "save-commands": True} + ) + + # Enable the commands in the built image + for cmd in xlogin.get("/xapi/commands").json(): + if cmd["image"] == tag: + for wrapper in cmd["xnat"]: + xlogin.put( + f"/xapi/commands/{cmd['id']}/" + f"wrappers/{wrapper['id']}/enabled" + ) + click.echo(f"Installed and enabled {tag}") + else: + click.echo(f"Skipping {tag} as it doesn't match filters") + + click.echo( + f"Successfully updated all container images from '{manifest['release']}' of " + f"'{manifest['package']}' package that match provided filters" + ) diff --git a/pydra2app/xnat/cli/update_release.py b/pydra2app/xnat/cli/update_release.py deleted file mode 100644 index ec61d87..0000000 --- a/pydra2app/xnat/cli/update_release.py +++ /dev/null @@ -1,147 +0,0 @@ -import re -import json -import yaml -import click -import xnat -from .base import xnat_group - - -PULL_IMAGES_XNAT_HOST_KEY = "XNAT_HOST" -PULL_IMAGES_XNAT_USER_KEY = "XNAT_USER" -PULL_IMAGES_XNAT_PASS_KEY = "XNAT_PASS" - - -@xnat_group.command( - name="pull-images", - help=f"""Updates the installed pipelines on an XNAT instance from a manifest -JSON file using the XNAT instance's REST API. - -MANIFEST_FILE is a JSON file containing a list of container images built in a release -created by `pydra2app deploy xnat build` - -Authentication credentials can be passed through the {PULL_IMAGES_XNAT_USER_KEY} -and {PULL_IMAGES_XNAT_PASS_KEY} environment variables. Otherwise, tokens can be saved -in a JSON file passed to '--auth'. - -Which of available pipelines to install can be controlled by a YAML file passed to the -'--filters' option of the form - \b - include: - - tag: ghcr.io/Australian-Imaging-Service/mri.human.neuro.* - - tag: ghcr.io/Australian-Imaging-Service/pet.rodent.* - exclude: - - tag: ghcr.io/Australian-Imaging-Service/mri.human.neuro.bidsapps. -""", -) -@click.argument("manifest_file", type=click.File()) -@click.option( - "--server", - type=str, - envvar=PULL_IMAGES_XNAT_HOST_KEY, - help=("the username used to authenticate with the XNAT instance to update"), -) -@click.option( - "--user", - envvar=PULL_IMAGES_XNAT_USER_KEY, - help=("the username used to authenticate with the XNAT instance to update"), -) -@click.option( - "--password", - envvar=PULL_IMAGES_XNAT_PASS_KEY, - help=("the password used to authenticate with the XNAT instance to update"), -) -@click.option( - "--filters", - "filters_file", - default=None, - type=click.File(), - help=("a YAML file containing filter rules for the images to install"), -) -def pull_xnat_images(manifest_file, server, user, password, filters_file): - manifest = json.load(manifest_file) - filters = yaml.load(filters_file, Loader=yaml.Loader) if filters_file else {} - - def matches_entry(entry, match_exprs, default=True): - """Determines whether an entry meets the inclusion and exclusion criteria - - Parameters - ---------- - entry : ty.Dict[str, Any] - a image entry in the manifest - exprs : list[ty.Dict[str, str]] - match criteria - default : bool - the value if match_exprs are empty - """ - if not match_exprs: - return default - return re.match( - "|".join( - i["name"].replace(".", "\\.").replace("*", ".*") for i in match_exprs - ), - entry["name"], - ) - - with xnat.connect( - server=server, - user=user, - password=password, - ) as xlogin: - - for entry in manifest["images"]: - if matches_entry(entry, filters.get("include")) and not matches_entry( - entry, filters.get("exclude"), default=False - ): - tag = f"{entry['name']}:{entry['version']}" - xlogin.post( - "/xapi/docker/pull", query={"image": tag, "save-commands": True} - ) - - # Enable the commands in the built image - for cmd in xlogin.get("/xapi/commands").json(): - if cmd["image"] == tag: - for wrapper in cmd["xnat"]: - xlogin.put( - f"/xapi/commands/{cmd['id']}/" - f"wrappers/{wrapper['id']}/enabled" - ) - click.echo(f"Installed and enabled {tag}") - else: - click.echo(f"Skipping {tag} as it doesn't match filters") - - click.echo( - f"Successfully updated all container images from '{manifest['release']}' of " - f"'{manifest['package']}' package that match provided filters" - ) - - -@xnat_group.command( - name="auth-refresh", - help="""Logs into the XNAT instance and regenerates a new authorisation token -to avoid them expiring (2 days by default) - -CONFIG_YAML a YAML file contains the login details for the XNAT server to update -""", -) -@click.argument("config_yaml_file", type=click.File()) -@click.argument("auth_file_path", type=click.Path(exists=True)) -def xnat_auth_refresh(config_yaml_file, auth_file_path): - config = yaml.load(config_yaml_file, Loader=yaml.Loader) - with open(auth_file_path) as fp: - auth = json.load(fp) - - with xnat.connect( - server=config["server"], user=auth["alias"], password=auth["secret"] - ) as xlogin: - alias, secret = xlogin.services.issue_token() - - with open(auth_file_path, "w") as f: - json.dump( - { - "alias": alias, - "secret": secret, - }, - f, - ) - - click.echo(f"Updated XNAT connection token to {config['server']} successfully") diff --git a/pydra2app/xnat/deploy.py b/pydra2app/xnat/deploy.py new file mode 100644 index 0000000..b278143 --- /dev/null +++ b/pydra2app/xnat/deploy.py @@ -0,0 +1,256 @@ +import typing as ty +import time +import logging +import json +import xnat +from pydra2app.core.exceptions import Pydra2AppError +from pydra2app.core.utils import extract_file_from_docker_image + + +logger = logging.getLogger("pydra2app-xnat") + + +def install_cs_command( + image_name_or_command_json: ty.Union[str, ty.Dict[str, ty.Any]], + xlogin: xnat.XNATSession, + enable: bool = False, + projects_to_enable: ty.Sequence[str] = (), +): + """Installs a new command for the XNAT container service and lanches it on + the specified session. + + Parameters + ---------- + command_json : ty.Dict[str, Any] + JSON that defines the XNAT command in the container service (see `generate_xnat_command`) + xlogin : xnat.XNATSession + XnatPy connection to the XNAT server + enable : bool + Enable the command globally + projects_to_enable : ty.Sequence[str] + ID of the project to enable the command for + + Returns + ------- + cmd_id : int + the ID of the installed command + """ + if isinstance(image_name_or_command_json, str): + commands = xlogin.get("/xapi/commands").json() + commands = [c for c in commands if c["image"] == image_name_or_command_json] + if not commands: + raise RuntimeError("Did not find ") + command_json_file = extract_file_from_docker_image( + image_name_or_command_json, "/xnat_command.json" + ) + with open(command_json_file) as f: + command_json = json.load(f) + elif isinstance(image_name_or_command_json, dict): + command_json = image_name_or_command_json + else: + raise RuntimeError( + f"Unrecognised type of 'image_name_or_command_json' arg: {type(image_name_or_command_json)} " + "expected str or dict" + ) + + wrapper_name = command_json["xnat"][0]["name"] + cmd_id = xlogin.post("/xapi/commands", json=command_json).json() + + # Enable the command globally and in the project + if enable: + xlogin.put(f"/xapi/commands/{cmd_id}/wrappers/{wrapper_name}/enabled") + for project_id in projects_to_enable: + xlogin.put( + f"/xapi/projects/{project_id}/commands/{cmd_id}/wrappers/{wrapper_name}/enabled" + ) + elif projects_to_enable: + raise RuntimeError( + "'enable' must be set to True for individual projects to be enabled " + f"({projects_to_enable})" + ) + return cmd_id + + +def launch_cs_command( + command_id_or_name: ty.Union[int, str], + project_id: str, + session_id: str, + inputs: ty.Dict[str, str], + xlogin: xnat.XNATSession, + timeout: int = 1000, # seconds + poll_interval: int = 10, # seconds +): + """Installs a new command for the XNAT container service and lanches it on + the specified session. + + Parameters + ---------- + command_id_or_name : ty.Union[int, str] + the ID (int) or name of the command to launch, or the name of the image containing + the command (assumed if name includes ':' or '/' characters) + project_id : str + ID of the project to enable the command for + session_id : str + ID of the session to launch the command on + inputs : ty.Dict[str, str] + Inputs passed to the pipeline at launch (i.e. typically through text fields in the CS launch UI) + xlogin : xnat.XNATSession + XnatPy connection to the XNAT server + timeout : int + the time to wait for the pipeline to complete (seconds) + poll_interval : int + the time interval between status polls (seconds) + + Returns + ------- + workflow_id : int + the auto-generated ID for the launched workflow + status : str + the status of the completed workflow + out_str : str + stdout and stderr from the workflow run + """ + if isinstance(command_id_or_name, int): + command_json = xlogin.get(f"/xapi/commands/{command_id_or_name}").json() + else: + assert isinstance(command_id_or_name, str) + commands = xlogin.get("/xapi/commands").json() + if ":" in command_id_or_name or "/" in command_id_or_name: + # Assume it is the docker image name + commands = [c for c in commands if c["image"] == command_id_or_name] + else: + commands = [c for c in commands if c["name"] == command_id_or_name] + if not commands: + raise RuntimeError( + f"Did not find command corresponding to name or image '{command_id_or_name}'" + ) + elif len(commands) > 1: + raise RuntimeError( + "Found multiple commands corresponding to name or image " + f"'{command_id_or_name}': {commands}" + ) + command_json = commands[0] + cmd_id = command_json["id"] + cmd_name = command_json["name"] + + launch_json = {"SESSION": f"/archive/experiments/{session_id}"} + + launch_json.update(inputs) + + launch_result = xlogin.post( + f"/xapi/projects/{project_id}/wrappers/{cmd_id}/root/SESSION/launch", + json=launch_json, + ).json() + + if launch_result["status"] != "success": + raise Pydra2AppError( + f"{cmd_name} workflow wasn't launched successfully ({launch_result['status']})" + ) + workflow_id = launch_result["workflow-id"] + assert workflow_id != "To be assigned" + + num_attempts = (timeout // poll_interval) + 1 + max_runtime = num_attempts * poll_interval + + for i in range(num_attempts): + wf_result = xlogin.get(f"/xapi/workflows/{workflow_id}").json() + if wf_result["status"] not in INCOMPLETE_CS_STATES: + break + time.sleep(poll_interval) + + launch_status = wf_result["status"] + if launch_status.lower() != "success": + raise ValueError( + f"Launching {cmd_name} in the XNAT CS failed with status {launch_status} " + f"for inputs: \n{launch_json}" + ) + + container_id = wf_result["comments"] + assert container_id != "" + + # Get workflow stdout/stderr for error messages if required + out_str = "" + stdout_result = xlogin.get( + f"/xapi/containers/{container_id}/logs/stdout", accepted_status=[200, 204] + ) + if stdout_result.status_code == 200: + out_str = f"stdout:\n{stdout_result.content.decode('utf-8')}\n" + + stderr_result = xlogin.get( + f"/xapi/containers/{container_id}/logs/stderr", accepted_status=[200, 204] + ) + if stderr_result.status_code == 200: + out_str += f"\nstderr:\n{stderr_result.content.decode('utf-8')}" + + if i == num_attempts - 1: + status = f"NotCompletedAfter{max_runtime}Seconds" + else: + status = wf_result["status"] + + return workflow_id, status, out_str + + +def install_and_launch_xnat_cs_command( + command_json: dict, + project_id: str, + session_id: str, + inputs: ty.Dict[str, str], + xlogin: xnat.XNATSession, + **kwargs, +): + """Installs a new command for the XNAT container service and lanches it on + the specified session. + + Parameters + ---------- + cmd_name : str + The name to install the command as + command_json : ty.Dict[str, Any] + JSON that defines the XNAT command in the container service (see `generate_xnat_command`) + project_id : str + ID of the project to enable the command for + session_id : str + ID of the session to launch the command on + inputs : ty.Dict[str, str] + Inputs passed to the pipeline at launch (i.e. typically through text fields in the CS launch UI) + xlogin : xnat.XNATSession + XnatPy connection to the XNAT server + **kwargs: + Keyword arguments passed directly through to 'launch_cs_command' + + Returns + ------- + workflow_id : int + the auto-generated ID for the launched workflow + status : str + the status of the completed workflow + out_str : str + stdout and stderr from the workflow run + """ + + cmd_id = install_cs_command( + command_json, xlogin=xlogin, enable=True, projects_to_enable=[project_id] + ) + + return launch_cs_command( + cmd_id, + project_id=project_id, + session_id=session_id, + inputs=inputs, + xlogin=xlogin, + **kwargs, + ) + + +# List of intermediatary states can pass through +# before completing successfully +INCOMPLETE_CS_STATES = ( + "Pending", + "Running", + "_Queued", + "Staging", + "Finalizing", + "Created", + "_die", + "die", +) diff --git a/pydra2app/xnat/image.py b/pydra2app/xnat/image.py index 821b843..bbaed3a 100644 --- a/pydra2app/xnat/image.py +++ b/pydra2app/xnat/image.py @@ -29,7 +29,7 @@ class XnatApp(App): def construct_dockerfile( self, build_dir: Path, - use_test_config: bool = False, + for_localhost: bool = False, **kwargs, ): """Creates a Docker image containing one or more XNAT commands ready @@ -40,7 +40,7 @@ def construct_dockerfile( build_dir : Path the directory to build the docker image within, i.e. where to write Dockerfile and supporting files to be copied within the image - use_test_config : bool + for_localhost : bool whether to create the container so that it will work with the test XNAT configuration (i.e. hard-coding the XNAT server IP) **kwargs: @@ -61,7 +61,7 @@ def construct_dockerfile( # Copy the generated XNAT commands inside the container for ease of reference self.copy_command_ref(dockerfile, xnat_command, build_dir) - self.save_store_config(dockerfile, build_dir, use_test_config=use_test_config) + self.save_store_config(dockerfile, build_dir, for_localhost=for_localhost) # Convert XNAT command label into string that can by placed inside the # Docker label @@ -97,7 +97,7 @@ def copy_command_ref(self, dockerfile: DockerRenderer, xnat_command, build_dir): ) def save_store_config( - self, dockerfile: DockerRenderer, build_dir: Path, use_test_config=False + self, dockerfile: DockerRenderer, build_dir: Path, for_localhost=False ): """Save a configuration for a XnatViaCS store. @@ -107,7 +107,7 @@ def save_store_config( Neurodocker renderer to build build_dir : Path the build directory to save supporting files - use_test_config : bool + for_localhost : bool whether the target XNAT is using the local test configuration, in which case the server location will be hard-coded rather than rely on the XNAT_HOST environment variable passed to the container by the XNAT CS @@ -115,7 +115,7 @@ def save_store_config( xnat_cs_store_entry = { "class": "<" + ClassResolver.tostr(XnatViaCS, strip_prefix=False) + ">" } - if use_test_config: + if for_localhost: if sys.platform == "linux": ip_address = "172.17.0.1" # Linux + GH Actions else: diff --git a/pydra2app/xnat/testing.py b/pydra2app/xnat/testing.py deleted file mode 100644 index 814fe5a..0000000 --- a/pydra2app/xnat/testing.py +++ /dev/null @@ -1,134 +0,0 @@ -import typing as ty -import time -import logging -import xnat -from pydra2app.core.exceptions import Pydra2AppError - - -logger = logging.getLogger("pydra2app-xnat") - - -def install_and_launch_xnat_cs_command( - command_json: dict, - project_id: str, - session_id: str, - inputs: ty.Dict[str, str], - xlogin: xnat.XNATSession, - timeout: int = 1000, # seconds - poll_interval: int = 10, # seconds -): - """Installs a new command for the XNAT container service and lanches it on - the specified session. - - Parameters - ---------- - cmd_name : str - The name to install the command as - command_json : ty.Dict[str, Any] - JSON that defines the XNAT command in the container service (see `generate_xnat_command`) - project_id : str - ID of the project to enable the command for - session_id : str - ID of the session to launch the command on - inputs : ty.Dict[str, str] - Inputs passed to the pipeline at launch (i.e. typically through text fields in the CS launch UI) - xlogin : xnat.XNATSession - XnatPy connection to the XNAT server - timeout : int - the time to wait for the pipeline to complete (seconds) - poll_interval : int - the time interval between status polls (seconds) - - Returns - ------- - workflow_id : int - the auto-generated ID for the launched workflow - status : str - the status of the completed workflow - out_str : str - stdout and stderr from the workflow run - """ - - cmd_name = command_json["name"] - wrapper_name = command_json["xnat"][0]["name"] - cmd_id = xlogin.post("/xapi/commands", json=command_json).json() - - # Enable the command globally and in the project - xlogin.put(f"/xapi/commands/{cmd_id}/wrappers/{wrapper_name}/enabled") - xlogin.put( - f"/xapi/projects/{project_id}/commands/{cmd_id}/wrappers/{wrapper_name}/enabled" - ) - - launch_json = {"SESSION": f"/archive/experiments/{session_id}"} - - launch_json.update(inputs) - - launch_result = xlogin.post( - f"/xapi/projects/{project_id}/wrappers/{cmd_id}/root/SESSION/launch", - json=launch_json, - ).json() - - if launch_result["status"] != "success": - raise Pydra2AppError( - f"{cmd_name} workflow wasn't launched successfully ({launch_result['status']})" - ) - workflow_id = launch_result["workflow-id"] - assert workflow_id != "To be assigned" - - num_attempts = (timeout // poll_interval) + 1 - max_runtime = num_attempts * poll_interval - - for i in range(num_attempts): - wf_result = xlogin.get(f"/xapi/workflows/{workflow_id}").json() - if wf_result["status"] not in INCOMPLETE_CS_STATES: - break - time.sleep(poll_interval) - - launch_status = wf_result["status"] - if launch_status == "success": - raise ValueError( - f"Launching {cmd_name} in the XNAT CS failed with status {launch_status} " - f"for inputs: \n{launch_json}" - ) - - container_id = wf_result["comments"] - if container_id == "": - raise RuntimeError( - f"Container failed to launch with status '{wf_result['status']}':\n" - f"{wf_result['details']}" - ) - - # Get workflow stdout/stderr for error messages if required - out_str = "" - stdout_result = xlogin.get( - f"/xapi/containers/{container_id}/logs/stdout", accepted_status=[200, 204] - ) - if stdout_result.status_code == 200: - out_str = f"stdout:\n{stdout_result.content.decode('utf-8')}\n" - - stderr_result = xlogin.get( - f"/xapi/containers/{container_id}/logs/stderr", accepted_status=[200, 204] - ) - if stderr_result.status_code == 200: - out_str += f"\nstderr:\n{stderr_result.content.decode('utf-8')}" - - if i == num_attempts - 1: - status = f"NotCompletedAfter{max_runtime}Seconds" - else: - status = wf_result["status"] - - return workflow_id, status, out_str - - -# List of intermediatary states can pass through -# before completing successfully -INCOMPLETE_CS_STATES = ( - "Pending", - "Running", - "_Queued", - "Staging", - "Finalizing", - "Created", - "_die", - "die", -) diff --git a/pydra2app/xnat/tests/test_app.py b/pydra2app/xnat/tests/test_app.py index 67b9e0d..266f520 100644 --- a/pydra2app/xnat/tests/test_app.py +++ b/pydra2app/xnat/tests/test_app.py @@ -9,7 +9,7 @@ ) from pydra2app.xnat.image import XnatApp from pydra2app.xnat.command import XnatCommand -from pydra2app.xnat.testing import ( +from pydra2app.xnat.deploy import ( install_and_launch_xnat_cs_command, ) from fileformats.medimage import NiftiGzX, NiftiGzXBvec @@ -176,7 +176,7 @@ def test_xnat_cs_pipeline(xnat_repository, run_spec, run_prefix, work_dir): build_dir=work_dir, pydra2app_install_extras=["test"], use_local_packages=True, - use_test_config=True, + for_localhost=True, ) # We manually set the command in the test XNAT instance as commands are diff --git a/pydra2app/xnat/tests/test_cli_xnat_images.py b/pydra2app/xnat/tests/test_cli_xnat_images.py index a693691..02030b2 100644 --- a/pydra2app/xnat/tests/test_cli_xnat_images.py +++ b/pydra2app/xnat/tests/test_cli_xnat_images.py @@ -1,4 +1,3 @@ -import sys import os from copy import copy import json @@ -7,13 +6,13 @@ import pytest import docker import xnat -from pydra2app.core.cli import make_app -from pydra2app.xnat.cli.update_release import ( - pull_xnat_images, - xnat_auth_refresh, - PULL_IMAGES_XNAT_HOST_KEY, - PULL_IMAGES_XNAT_USER_KEY, - PULL_IMAGES_XNAT_PASS_KEY, +from pydra2app.core.cli import make +from pydra2app.xnat.cli.release import ( + deploy_pipelines, + save_token, + XNAT_HOST_KEY, + XNAT_USER_KEY, + XNAT_PASS_KEY, ) from frametree.core.utils import show_cli_trace @@ -21,7 +20,7 @@ @pytest.mark.skip( reason=("Latest versions of XNAT have removed the pull images functionality"), ) -def test_pull_xnat_images( +def test_deploy_pipelines( xnat_repository, command_spec, work_dir, @@ -81,10 +80,10 @@ def test_pull_xnat_images( manifest_path = work_dir / "manifest.json" result = cli_runner( - make_app, + make, [ + "xnat", str(spec_dir), - "xnat:XnatApp", "--build-dir", str(build_dir), "--registry", @@ -100,7 +99,7 @@ def test_pull_xnat_images( run_prefix, "--save-manifest", str(manifest_path), - "--use-test-config", + "--for-localhost", "--push", ], ) @@ -139,13 +138,13 @@ def test_pull_xnat_images( with patch.dict( os.environ, { - PULL_IMAGES_XNAT_HOST_KEY: xnat4tests_config.xnat_uri, - PULL_IMAGES_XNAT_USER_KEY: xnat4tests_config.xnat_user, - PULL_IMAGES_XNAT_PASS_KEY: xnat4tests_config.xnat_password, + XNAT_HOST_KEY: xnat4tests_config.xnat_uri, + XNAT_USER_KEY: xnat4tests_config.xnat_user, + XNAT_PASS_KEY: xnat4tests_config.xnat_password, }, ): result = cli_runner( - pull_xnat_images, + deploy_pipelines, [str(manifest_path), "--filters", str(filters_file)], ) @@ -162,7 +161,7 @@ def test_pull_xnat_images( assert all(cmd in available_cmds for cmd in expected_commands) -def test_xnat_auth_refresh(xnat_repository, work_dir, cli_runner): +def test_save_token(xnat_repository, work_dir, cli_runner): config_path = work_dir / "config.yaml" with open(config_path, "w") as f: @@ -181,7 +180,7 @@ def test_xnat_auth_refresh(xnat_repository, work_dir, cli_runner): ) result = cli_runner( - xnat_auth_refresh, + save_token, [str(config_path), str(auth_path)], ) diff --git a/scripts/quick_test.py b/scripts/quick_test.py new file mode 100644 index 0000000..1027013 --- /dev/null +++ b/scripts/quick_test.py @@ -0,0 +1,24 @@ +from frametree.core.utils import show_cli_trace +from pydra2app.core.cli import make + + +def test_quickly(cli_runner): + + result = cli_runner( + make, + [ + "xnat", + ( + "/Users/tclose/git/pipelines-community/specs/" + "australian-imaging-service-community/examples/zip.yaml" + ), + "--spec-root", + "/Users/tclose/git/pipelines-community/specs", + "--for-localhost", + "--export-file", + "xnat_command.json", + "~/zip-xnat-command.json", + ], + ) + + assert result.exit_code == 0, show_cli_trace(result) From b69bc4e42abc9900161e57a88a20eddff480439b Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 6 Aug 2024 22:37:44 +1000 Subject: [PATCH 2/4] debugged launch and install command --- pydra2app/xnat/cli/__init__.py | 2 +- pydra2app/xnat/cli/release.py | 43 ++++++++++++++++++++----- pydra2app/xnat/deploy.py | 46 +++++++++++++++++++++++--- scripts/quick_test.py | 59 +++++++++++++++++++++++++++++++--- 4 files changed, 131 insertions(+), 19 deletions(-) diff --git a/pydra2app/xnat/cli/__init__.py b/pydra2app/xnat/cli/__init__.py index 84c36e8..12ca5c5 100644 --- a/pydra2app/xnat/cli/__init__.py +++ b/pydra2app/xnat/cli/__init__.py @@ -1,3 +1,3 @@ from .base import xnat_group from .entrypoint import cs_entrypoint -from .release import pull_xnat_images, xnat_auth_refresh +from .release import deploy_pipelines, save_token, install_command, launch_command diff --git a/pydra2app/xnat/cli/release.py b/pydra2app/xnat/cli/release.py index 001ad57..31f5643 100644 --- a/pydra2app/xnat/cli/release.py +++ b/pydra2app/xnat/cli/release.py @@ -17,19 +17,25 @@ XNAT_USER_KEY = "XNAT_USER" XNAT_PASS_KEY = "XNAT_PASS" XNAT_AUTH_FILE_KEY = "XNAT_AUTH_FILE" -XNAT_AUTH_FILE_DEFAULT = Path("~/.pydra2app_xnat_auth.json").expanduser() +XNAT_AUTH_FILE_DEFAULT = Path("~/.pydra2app_xnat_user_token.json").expanduser() def load_auth(server, user, password, auth_file): - if server is None: + if server is not None: if user is None: raise RuntimeError(f"A user must be provided if a server ({server}) is") if password is None: raise RuntimeError(f"a password must be provided if a server ({server}) is") else: + if auth_file == XNAT_AUTH_FILE_DEFAULT and not Path(auth_file).exists(): + raise RuntimeError( + "An auth file must be provided if no server is. " + "Use pydra2app ext xnat save-token to create one" + ) click.echo(f"Reading existing alias/token pair from '{str(auth_file)}") with open(auth_file) as fp: auth = json.load(fp) + server = auth["server"] user = auth["alias"] password = auth["secret"] return server, user, password @@ -43,20 +49,25 @@ def load_auth(server, user, password, auth_file): the path to a command JSON file to install """, ) -@click.argument("image_or_command_file", type=click.File()) +@click.argument("image_or_command_file", type=str) @click.option( - "--enable", + "--enable/--disable", type=bool, default=False, help=("Whether to enable the command globally"), ) @click.option( "--enable-project", - "enable_projects", + "projects_to_enable", type=str, multiple=True, help=("Enable the command for the given project"), ) +@click.option( + "--replace-existing/--no-replace-existing", + default=False, + help=("Whether to replace existing command with the same name"), +) @click.option( "--server", envvar=XNAT_HOST_KEY, @@ -81,7 +92,14 @@ def load_auth(server, user, password, auth_file): help=("The path to save the alias/token pair to"), ) def install_command( - image_or_command_file, enable, projects_to_enable, server, user, password, auth_file + image_or_command_file, + enable, + projects_to_enable, + replace_existing, + server, + user, + password, + auth_file, ): server, user, password = load_auth(server, user, password, auth_file) @@ -95,6 +113,7 @@ def install_command( xlogin=xlogin, enable=enable, projects_to_enable=projects_to_enable, + replace_existing=replace_existing, ) click.echo( @@ -173,12 +192,20 @@ def launch_command( server, user, password = load_auth(server, user, password, auth_file) + inputs_dict = {} + for name, val in inputs: + if name in inputs_dict: + raise KeyError( + f"Duplicate input name '{name}' (values: {inputs_dict[name]}, {val})" + ) + inputs_dict[name] = val + with xnat.connect(server=server, user=user, password=password) as xlogin: launch_cs_command( command_name, project_id=project_id, session_id=session_id, - inputs=inputs, + inputs=inputs_dict, timeout=timeout, poll_interval=poll_interval, xlogin=xlogin, @@ -246,7 +273,7 @@ def save_token(auth_file, server, user, password): os.chmod(auth_file, 0o600) click.echo( - f"Saved alias/token to XNAT connection token to '{server}' at '{str(auth_file)}', " + f"Saved alias/token for '{server}' XNAT in '{str(auth_file)}' file, " "please ensure the file is secure" ) diff --git a/pydra2app/xnat/deploy.py b/pydra2app/xnat/deploy.py index b278143..389f01e 100644 --- a/pydra2app/xnat/deploy.py +++ b/pydra2app/xnat/deploy.py @@ -9,12 +9,15 @@ logger = logging.getLogger("pydra2app-xnat") +INTERNAL_INPUTS = ("Pydra2App_flags", "PROJECT_ID", "SUBJECT_LABEL", "SESSION_LABEL") + def install_cs_command( image_name_or_command_json: ty.Union[str, ty.Dict[str, ty.Any]], xlogin: xnat.XNATSession, enable: bool = False, projects_to_enable: ty.Sequence[str] = (), + replace_existing: bool = False, ): """Installs a new command for the XNAT container service and lanches it on the specified session. @@ -29,6 +32,8 @@ def install_cs_command( Enable the command globally projects_to_enable : ty.Sequence[str] ID of the project to enable the command for + replace_existing : bool + Whether to replace existing command with the same name Returns ------- @@ -36,13 +41,13 @@ def install_cs_command( the ID of the installed command """ if isinstance(image_name_or_command_json, str): - commands = xlogin.get("/xapi/commands").json() - commands = [c for c in commands if c["image"] == image_name_or_command_json] - if not commands: - raise RuntimeError("Did not find ") command_json_file = extract_file_from_docker_image( image_name_or_command_json, "/xnat_command.json" ) + if command_json_file is None: + raise RuntimeError( + f"Could not find command JSON file in '{image_name_or_command_json}'" + ) with open(command_json_file) as f: command_json = json.load(f) elif isinstance(image_name_or_command_json, dict): @@ -53,7 +58,15 @@ def install_cs_command( "expected str or dict" ) + cmd_name = command_json["name"] wrapper_name = command_json["xnat"][0]["name"] + + if replace_existing: + for cmd in xlogin.get("/xapi/commands").json(): + if cmd["name"] == cmd_name: + xlogin.delete(f"/xapi/commands/{cmd['id']}", accepted_status=[200, 204]) + logger.info(f"Deleted existing command '{cmd_name}'") + cmd_id = xlogin.post("/xapi/commands", json=command_json).json() # Enable the command globally and in the project @@ -133,7 +146,30 @@ def launch_cs_command( cmd_id = command_json["id"] cmd_name = command_json["name"] - launch_json = {"SESSION": f"/archive/experiments/{session_id}"} + launch_json = { + "SESSION": f"/archive/projects/{project_id}/experiments/{session_id}" + } + + provided_inputs = list(inputs.keys()) + input_names = [ + i["name"] for i in command_json["inputs"] if i["name"] not in INTERNAL_INPUTS + ] + required_inputs = [ + i["name"] + for i in command_json["inputs"] + if i["required"] and i["name"] not in INTERNAL_INPUTS + ] + + missing_inputs = list(set(required_inputs) - set(provided_inputs)) + unexpected_inputs = list(set(provided_inputs) - set(input_names)) + if missing_inputs or unexpected_inputs: + raise ValueError( + f"Error launching '{cmd_name}' command:\n" + f" Valid inputs: {input_names}\n" + f" Provided inputs: {provided_inputs}\n" + f" Missing required inputs: {missing_inputs}\n" + f" Unexpected inputs: {unexpected_inputs}\n" + ) launch_json.update(inputs) diff --git a/scripts/quick_test.py b/scripts/quick_test.py index 1027013..e4f427f 100644 --- a/scripts/quick_test.py +++ b/scripts/quick_test.py @@ -1,8 +1,9 @@ from frametree.core.utils import show_cli_trace from pydra2app.core.cli import make +from pydra2app.xnat.cli import save_token, install_command, launch_command -def test_quickly(cli_runner): +def test_make(cli_runner): result = cli_runner( make, @@ -10,14 +11,62 @@ def test_quickly(cli_runner): "xnat", ( "/Users/tclose/git/pipelines-community/specs/" - "australian-imaging-service-community/examples/zip.yaml" + "australian-imaging-service-community/examples/bet.yaml" ), "--spec-root", "/Users/tclose/git/pipelines-community/specs", "--for-localhost", - "--export-file", - "xnat_command.json", - "~/zip-xnat-command.json", + "--use-local-packages", + ], + ) + + assert result.exit_code == 0, show_cli_trace(result) + + +def test_save_token(cli_runner): + + result = cli_runner( + save_token, + [ + "--server", + "http://localhost:8080", + "--user", + "admin", + "--password", + "admin", + ], + ) + + assert result.exit_code == 0, show_cli_trace(result) + + +def test_install_command(cli_runner): + + result = cli_runner( + install_command, + [ + "australian-imaging-service-community/examples.bet:6.0.6.4-1", + "--enable", + "--enable-project", + "OPENNEURO_T1W", + "--replace-existing", + ], + ) + + assert result.exit_code == 0, show_cli_trace(result) + + +def test_launch_command(cli_runner): + + result = cli_runner( + launch_command, + [ + "examples.bet", + "OPENNEURO_T1W", + "subject01_MR01", + "--input", + "T1w", + "t1w", ], ) From f26bff99abcc52edd1c8b0a946d969f1bfe0a6e7 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 6 Aug 2024 22:38:13 +1000 Subject: [PATCH 3/4] renamed test so it isn't included in unittests --- scripts/{quick_test.py => quick_tst.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{quick_test.py => quick_tst.py} (100%) diff --git a/scripts/quick_test.py b/scripts/quick_tst.py similarity index 100% rename from scripts/quick_test.py rename to scripts/quick_tst.py From 465618b87b71ec5a5420a9943240731bad4aefd5 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 6 Aug 2024 22:40:56 +1000 Subject: [PATCH 4/4] fixed up status check --- pydra2app/xnat/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra2app/xnat/deploy.py b/pydra2app/xnat/deploy.py index 389f01e..835804f 100644 --- a/pydra2app/xnat/deploy.py +++ b/pydra2app/xnat/deploy.py @@ -195,7 +195,7 @@ def launch_cs_command( time.sleep(poll_interval) launch_status = wf_result["status"] - if launch_status.lower() != "success": + if launch_status != "Complete": raise ValueError( f"Launching {cmd_name} in the XNAT CS failed with status {launch_status} " f"for inputs: \n{launch_json}"