diff --git a/.coveragerc b/.coveragerc index 84ad4b67..3c69f131 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,10 @@ [run] -omit = test/*, setup.py, *.adoc +source = em_workflows +omit = + # Omit test code + test/*, + setup.py, + # Coverage wants to check adoc files for some reason + *.adoc, + # Some *.py files show up in temp directories + */tmp/* diff --git a/coverage.svg b/coverage.svg index 7a18c7f4..1c7007cd 100644 --- a/coverage.svg +++ b/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 80% - 80% + 89% + 89% diff --git a/em_workflows/file_path.py b/em_workflows/file_path.py index 4d9c000a..b820a5e2 100644 --- a/em_workflows/file_path.py +++ b/em_workflows/file_path.py @@ -147,40 +147,40 @@ def copy_to_assets_dir(self, fp_to_cp: Path) -> Path: shutil.copyfile(fp_to_cp, dest) return dest - def add_assets_entry( - self, asset_path: Path, asset_type: str, metadata: Dict[str, str] = None - ) -> Dict: - """ - Suspect to be redundant - TODO - """ - # TODO move valid_types to Config.valid_callback_asset_types - valid_typs = [ - "averagedVolume", - "keyImage", - "thumbnail", - "keyThumbnail", - "recMovie", - "tiltMovie", - "volume", - "neuroglancerPrecomputed", - ] - if asset_type not in valid_typs: - raise ValueError( - f"Asset type: {asset_type} is not a valid type. {valid_typs}" - ) - fp_no_mount_point = asset_path.relative_to( - Config.assets_dir(env=self.environment) - ) - if metadata: - asset = { - "type": asset_type, - "path": fp_no_mount_point.as_posix(), - "metadata": metadata, - } - else: - asset = {"type": asset_type, "path": fp_no_mount_point.as_posix()} - self.prim_fp_elt["assets"].append(asset) - return asset + # def add_assets_entry( + # self, asset_path: Path, asset_type: str, metadata: Dict[str, str] = None + # ) -> Dict: + # """ + # Suspect to be redundant - TODO + # """ + # # TODO move valid_types to Config.valid_callback_asset_types + # valid_typs = [ + # "averagedVolume", + # "keyImage", + # "thumbnail", + # "keyThumbnail", + # "recMovie", + # "tiltMovie", + # "volume", + # "neuroglancerPrecomputed", + # ] + # if asset_type not in valid_typs: + # raise ValueError( + # f"Asset type: {asset_type} is not a valid type. {valid_typs}" + # ) + # fp_no_mount_point = asset_path.relative_to( + # Config.assets_dir(env=self.environment) + # ) + # if metadata: + # asset = { + # "type": asset_type, + # "path": fp_no_mount_point.as_posix(), + # "metadata": metadata, + # } + # else: + # asset = {"type": asset_type, "path": fp_no_mount_point.as_posix()} + # self.prim_fp_elt["assets"].append(asset) + # return asset def gen_output_fp(self, output_ext: str = None, out_fname: str = None) -> Path: """ @@ -197,13 +197,13 @@ def gen_output_fp(self, output_ext: str = None, out_fname: str = None) -> Path: output_fp = f"{self.working_dir.as_posix()}/{f_name}" return Path(output_fp) - @staticmethod - def filter_by_suffix(fp: Path, suffixes: List[str]) -> bool: - """This method currently isn't used""" - for ext in suffixes: - if fp.suffix.lower() == ext: - return True - return False + # @staticmethod + # def filter_by_suffix(fp: Path, suffixes: List[str]) -> bool: + # """This method currently isn't used""" + # for ext in suffixes: + # if fp.suffix.lower() == ext: + # return True + # return False def gen_asset(self, asset_type: str, asset_fp) -> Dict: """ diff --git a/em_workflows/utils/utils.py b/em_workflows/utils/utils.py index beb70503..404cba39 100644 --- a/em_workflows/utils/utils.py +++ b/em_workflows/utils/utils.py @@ -1,7 +1,6 @@ from em_workflows.file_path import FilePath from jinja2 import Environment, FileSystemLoader import subprocess -import re import requests import os import shutil @@ -18,7 +17,6 @@ from prefect.engine.signals import SKIP, TRIGGERFAIL from em_workflows.config import Config from prefect.tasks.control_flow.filter import FilterTask - from collections import namedtuple # used for keeping outputs of imod's header command (dimensions of image). @@ -219,6 +217,7 @@ def update_adoc( | Uses jinja templating to update the adoc file with input params. | dual_p is calculated by inputs_paired() and is used to define `dual` | Some of these parameters are derived programatically. + :todo: Remove references to ``dual_p`` in comments? """ file_loader = FileSystemLoader(str(adoc_fp.parent)) @@ -274,6 +273,8 @@ def copy_tg_to_working_dir(fname: Path, working_dir: Path) -> Path: """ copies files (tomograms/mrc files) into working_dir returns Path of copied file + :todo: Determine if the 'a' & 'b' files still exist and if these files need + to be copied. (See comment in ``run_brt`` before this call is made) """ new_loc = Path(f"{working_dir}/{fname.name}") if fname.exists(): @@ -291,7 +292,11 @@ def copy_tg_to_working_dir(fname: Path, working_dir: Path) -> Path: def copy_template(working_dir: Path, template_name: str) -> Path: """ - copies the template adoc file to the working_dir, + :param working_dir: libpath.Path of temporary working directory + :param template_name: base str name of the ADOC template + :return: libpath.Path of the copied file + + copies the template adoc file to the working_dir """ adoc_fp = f"{working_dir}/{template_name}.adoc" template_fp = f"{Config.template_dir}/{template_name}.adoc" @@ -411,12 +416,12 @@ def run_brt( # return "\\" + match.group(0) -def _tr_str(name): - """ - :todo: Consider removing as this looks to be dead code - """ - _to_esc = re.compile(r"\s|[]()[]") - return _to_esc.sub("_", name) +# def _tr_str(name): +# """ +# :todo: Consider removing as this looks to be dead code +# """ +# _to_esc = re.compile(r"\s|[]()[]") +# return _to_esc.sub("_", name) # def _escape_str(name): @@ -469,8 +474,13 @@ def _tr_str(name): def log(msg): - # log_name is defined by the dir_name (all wfs are associated with an input_dir - # Verify that we are in a flow and have perameters defined; not true if testing + """ + Convenience function to print an INFO message to both the "input_dir" context log and + the "root" prefect log. + + :param msg: string to output + :return: None + """ if hasattr(context, "parameters"): logger = logging.getLogger(context.parameters["input_dir"]) logger.info(msg) @@ -479,6 +489,18 @@ def log(msg): @task(max_retries=1, retry_delay=datetime.timedelta(seconds=10), trigger=always_run) def copy_workdirs(file_path: FilePath) -> Path: + """ + :param file_path: The FilePath of the file whose workdir is to be copied + :returns: Resulting Assets FilePath + + This test copies the workdir, in it's entirety, to the Assets path. This can + be a very large number of files and storage space. This work is delgated to + FilePath. + + :param file_path: FilePath of the current imagefile + :return: pathlib.Path of copied directory + + """ return file_path.copy_workdir_to_assets() @@ -491,24 +513,32 @@ def copy_workdirs(file_path: FilePath) -> Path: # shutil.copytree(working_dir.as_posix(), dest) -@task -def cp_logs_to_assets(working_dir: Path, assets_dir: Path) -> None: - print(f"looking in {working_dir}") - print(f"copying to {assets_dir}") - for _log in working_dir.glob("*.log"): - print(f"found {_log}") - print(f"going to copy to {assets_dir}") - shutil.copy(_log, assets_dir) +# @task +# def cp_logs_to_assets(working_dir: Path, assets_dir: Path) -> None: +# """ +# :todo: Consider removing as this function isn't used (according to PyCharm) +# """ +# print(f"looking in {working_dir}") +# print(f"copying to {assets_dir}") +# for _log in working_dir.glob("*.log"): +# print(f"found {_log}") +# print(f"going to copy to {assets_dir}") +# shutil.copy(_log, assets_dir) @task def list_files(input_dir: Path, exts: List[str], single_file: str = None) -> List[Path]: """ - List all files within input_dir with spefified extension. - if a specific file is requested that file is returned only. - This allows workflows run on single files rather than entire dirs (default). - Note, if no files are found does NOT raise exception. Function can be called - multiple times, sometimes there will be no files of that extension. + :param input_dir: libpath.Path of the input directory + :param exts: List of str extensions to be checked + :param single_file: if present, only that file returned + :return: List of pathlib.Paths of matching files + + - List all files within input_dir with specified extension. + - if a specific file is requested that file is returned only. + - This allows workflows to run on single files rather than entire dirs (default). + - Note, if no files are found does NOT raise exception. Function can be called + multiple times, sometimes there will be no files of that extension. """ _files = list() log(f"Looking for *.{exts} in {input_dir}") @@ -547,52 +577,69 @@ def list_dirs(input_dir_fp: Path) -> List[Path]: return dirs -@task -def gen_output_fp(input_fp: Path, output_ext: str, working_dir: Path = None) -> Path: - """ - | cat working_dir to input_fp.name, but swap the extension to output_ext - - the reason for having a working_dir default to None is sometimes the output - dir is not the same as the input dir, and working_dir is used to define output - in this case. - - :todo: Consider removing since this function isn't currently called - - """ - stem_name = _tr_str(input_fp.stem) - if working_dir: - output_fp = f"{working_dir.as_posix()}/{stem_name}{output_ext}" - else: - output_fp = f"{input_fp.parent}/{stem_name}{output_ext}" - - msg = f"Using dir: {working_dir}, file: {input_fp}, ext: {output_ext} creating output_fp {output_fp}" - log(msg=msg) - return Path(output_fp) +# @task +# def gen_output_fp(input_fp: Path, output_ext: str, working_dir: Path = None) -> Path: +# """ +# :param input_fp: +# :param output_ext: +# :param working_dir: +# :returns: +# +# | cat working_dir to input_fp.name, but swap the extension to output_ext +# +# the reason for having a working_dir default to None is sometimes the output +# dir is not the same as the input dir, and working_dir is used to define output +# in this case. +# +# :todo: Consider removing since this function isn't currently called (according to PyCharm) +# """ +# stem_name = _tr_str(input_fp.stem) +# if working_dir: +# output_fp = f"{working_dir.as_posix()}/{stem_name}{output_ext}" +# else: +# output_fp = f"{input_fp.parent}/{stem_name}{output_ext}" +# +# msg = f"Using dir: {working_dir}, file: {input_fp}, ext: {output_ext} creating output_fp {output_fp}" +# log(msg=msg) +# return Path(output_fp) -@task -def gen_output_fname(input_fp: Path, output_ext) -> Path: - """ - Each file is generated using the input file name, without extension, - with a new extension.""" - output_fp = Path(f"{input_fp.stem}{output_ext}") - log(f"input: {input_fp} output: {output_fp}") - return output_fp +# @task +# def gen_output_fname(input_fp: Path, output_ext) -> Path: +# """ +# :param input_fp: +# :param output_ext: +# :return: +# +# :todo: Consider removing since this function isn't currently called (according to PyCharm) +# +# Each file is generated using the input file name, without extension, +# with a new extension. +# """ +# output_fp = Path(f"{input_fp.stem}{output_ext}") +# log(f"input: {input_fp} output: {output_fp}") +# return output_fp -@task -def run_single_file(input_fps: List[Path], fp_to_check: str) -> List[Path]: - """ - Workflows can be run on single files, if the file_name param is used. - This function will limit the list of inputs to only that file_name (if - provided), and check the file exists, if so will return as Path, else - raise exception.""" - if fp_to_check is None: - return input_fps - for _fp in input_fps: - if _fp.name == fp_to_check: - return [Path(fp_to_check)] - raise signals.FAIL(f"Expecting file: {fp_to_check}, not found in input_dir") +# @task +# def run_single_file(input_fps: List[Path], fp_to_check: str) -> List[Path]: +# """ +# :param input_fps: List of Paths +# :param fp_to_check: the filename to check as a str +# :returns: the input filename in a single-element Path list +# +# :todo: Consider removing since this function isn't currently called (according to PyCharm) +# +# Workflows can be run on single files, if the file_name param is used. +# This function will limit the list of inputs to only that file_name (if +# provided), and check the file exists, if so will return as Path, else +# raise exception.""" +# if fp_to_check is None: +# return input_fps +# for _fp in input_fps: +# if _fp.name == fp_to_check: +# return [Path(fp_to_check)] +# raise signals.FAIL(f"Expecting file: {fp_to_check}, not found in input_dir") def notify_api_running(flow: Flow, old_state, new_state) -> State: @@ -722,10 +769,14 @@ def get_environment() -> str: @task def get_input_dir(input_dir: str) -> Path: """ - Concat the POSTed input file path to the mount point. - create working dir - set up logger - returns Path obj + :param input_dir: + :return: + + | Concat the POSTed input file path to the mount point. + | create working dir + | set up logger + | returns Path obj + """ if not input_dir.endswith("/"): input_dir = input_dir + "/" @@ -768,103 +819,105 @@ def gen_fps(input_dir: Path, fps_in: List[Path]) -> List[FilePath]: # log(t) -@task -def add_assets_entry( - base_elt: Dict, path: Path, asset_type: str, metadata: Dict[str, str] = None -) -> Dict: - """ - asset type can be one of: - - averagedVolume - keyImage - keyThumbnail - recMovie - tiltMovie - volume - neuroglancerPrecomputed - - used to build the callback for API - metadata is used in conjunction with neuroglancer only - Used in FilePath obj - """ - valid_typs = [ - "averagedVolume", - "keyImage", - "thumbnail", - "keyThumbnail", - "recMovie", - "tiltMovie", - "volume", - "neuroglancerPrecomputed", - ] - if asset_type not in valid_typs: - raise signals.FAIL( - f"Asset type: {asset_type} is not a valid type. {valid_typs}" - ) - fp_no_mount_point = path.relative_to(Config.assets_dir(env=get_environment())) - if metadata: - asset = { - "type": asset_type, - "path": fp_no_mount_point.as_posix(), - "metadata": metadata, - } - else: - asset = {"type": asset_type, "path": fp_no_mount_point.as_posix()} - base_elt["assets"].append(asset) - return base_elt +# @task +# def add_assets_entry( +# base_elt: Dict, path: Path, asset_type: str, metadata: Dict[str, str] = None +# ) -> Dict: +# """ +# asset type can be one of: +# +# averagedVolume +# keyImage +# keyThumbnail +# recMovie +# tiltMovie +# volume +# neuroglancerPrecomputed +# +# used to build the callback for API +# metadata is used in conjunction with neuroglancer only +# Used in FilePath obj +# """ +# valid_typs = [ +# "averagedVolume", +# "keyImage", +# "thumbnail", +# "keyThumbnail", +# "recMovie", +# "tiltMovie", +# "volume", +# "neuroglancerPrecomputed", +# ] +# if asset_type not in valid_typs: +# raise signals.FAIL( +# f"Asset type: {asset_type} is not a valid type. {valid_typs}" +# ) +# fp_no_mount_point = path.relative_to(Config.assets_dir(env=get_environment())) +# if metadata: +# asset = { +# "type": asset_type, +# "path": fp_no_mount_point.as_posix(), +# "metadata": metadata, +# } +# else: +# asset = {"type": asset_type, "path": fp_no_mount_point.as_posix()} +# base_elt["assets"].append(asset) +# return base_elt -@task -def make_assets_dir(input_dir: Path, subdir_name: Path = None) -> Path: - """ - input_dir comes in the form {mount_point}/RMLEMHedwigQA/Projects/Lab/PI/ - want to create: {mount_point}/RMLEMHedwigQA/Assets/Lab/PI/ - Sometimes you don't want to create a subdir based on a file name. (eg fibsem) - Used in FilePath obj - """ - if "Projects" not in input_dir.as_posix(): - raise signals.FAIL( - f"Input directory {input_dir} does not look correct, it must contain the string 'Projects'." - ) - assets_dir_as_str = input_dir.as_posix().replace("/Projects/", "/Assets/") - if subdir_name: - assets_dir = Path(f"{assets_dir_as_str}/{subdir_name.stem}") - else: - assets_dir = Path(assets_dir_as_str) - log(f"making assets dir for {input_dir} at {assets_dir.as_posix()}") - assets_dir.mkdir(parents=True, exist_ok=True) - return assets_dir +# @task +# def make_assets_dir(input_dir: Path, subdir_name: Path = None) -> Path: +# """ +# input_dir comes in the form {mount_point}/RMLEMHedwigQA/Projects/Lab/PI/ +# want to create: {mount_point}/RMLEMHedwigQA/Assets/Lab/PI/ +# Sometimes you don't want to create a subdir based on a file name. (eg fibsem) +# Used in FilePath obj +# :todo: Consider removing since this function is only used in test/callbacks.py +# """ +# if "Projects" not in input_dir.as_posix(): +# raise signals.FAIL( +# f"Input directory {input_dir} does not look correct, it must contain the string 'Projects'." +# ) +# assets_dir_as_str = input_dir.as_posix().replace("/Projects/", "/Assets/") +# if subdir_name: +# assets_dir = Path(f"{assets_dir_as_str}/{subdir_name.stem}") +# else: +# assets_dir = Path(assets_dir_as_str) +# log(f"making assets dir for {input_dir} at {assets_dir.as_posix()}") +# assets_dir.mkdir(parents=True, exist_ok=True) +# return assets_dir -@task -def copy_to_assets_dir(fp: Path, assets_dir: Path, prim_fp: Path = None) -> Path: - """ - Copy fp to the assets (reported output) dir - fp is the Path to be copied. - assets_dir is the root dir (the input_dir with s/Projects/Assets/) - If prim_fp is passed, assets will be copied to a subdir defined by the input - file name, eg: - copy /gs1/home/macmenaminpe/tmp/tmp7gcsl4on/keyMov_SARsCoV2_1.mp4 - to - /mnt/ai-fas12/RMLEMHedwigQA/Assets/Lab/Pi/SARsCoV2_1/keyMov_SARsCoV2_1.mp4 - {mount_point}/{dname}/keyMov_SARsCoV2_1.mp4 - (note "SARsCoV2_1" in assets_dir) - If prim_fp is not used, no such subdir is created. - """ - if prim_fp is not None: - assets_sub_dir = Path(f"{assets_dir}/{prim_fp.stem}") - else: - assets_sub_dir = assets_dir - assets_sub_dir.mkdir(exist_ok=True) - dest = Path(f"{assets_sub_dir}/{fp.name}") - log(f"Trying to copy {fp} to {dest}") - if fp.is_dir(): - if dest.exists(): - shutil.rmtree(dest) - shutil.copytree(fp, dest) - else: - shutil.copyfile(fp, dest) - return dest +# @task +# def copy_to_assets_dir(fp: Path, assets_dir: Path, prim_fp: Path = None) -> Path: +# """ +# Copy fp to the assets (reported output) dir +# fp is the Path to be copied. +# assets_dir is the root dir (the input_dir with s/Projects/Assets/) +# If prim_fp is passed, assets will be copied to a subdir defined by the input +# file name, eg: +# copy /gs1/home/macmenaminpe/tmp/tmp7gcsl4on/keyMov_SARsCoV2_1.mp4 +# to +# /mnt/ai-fas12/RMLEMHedwigQA/Assets/Lab/Pi/SARsCoV2_1/keyMov_SARsCoV2_1.mp4 +# {mount_point}/{dname}/keyMov_SARsCoV2_1.mp4 +# (note "SARsCoV2_1" in assets_dir) +# If prim_fp is not used, no such subdir is created. +# :todo: Consider removing since this function is only used in test/callbacks.py +# """ +# if prim_fp is not None: +# assets_sub_dir = Path(f"{assets_dir}/{prim_fp.stem}") +# else: +# assets_sub_dir = assets_dir +# assets_sub_dir.mkdir(exist_ok=True) +# dest = Path(f"{assets_sub_dir}/{fp.name}") +# log(f"Trying to copy {fp} to {dest}") +# if fp.is_dir(): +# if dest.exists(): +# shutil.rmtree(dest) +# shutil.copytree(fp, dest) +# else: +# shutil.copyfile(fp, dest) +# return dest @task(max_retries=3, retry_delay=datetime.timedelta(minutes=1), trigger=any_successful) diff --git a/test/test_utils.py b/test/test_utils.py index 985f2948..cbfe801b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,3 +1,5 @@ +import glob + from em_workflows.utils import utils from em_workflows.file_path import FilePath from em_workflows.config import Config @@ -6,6 +8,7 @@ import shutil from pathlib import Path from prefect.engine import signals +import tempfile def test_hedwig_env() -> None: @@ -155,31 +158,31 @@ def test_update_adoc(mock_nfs_mount): LocalAlignments = 0 THICKNESS = 30 - adoc_tmplt = Path(os.path.join(Config.template_dir, f"{adoc_file}.adoc")) - local_tmplt = Path.cwd() / f"{adoc_file}.adoc" - shutil.copy(adoc_tmplt, local_tmplt) + with tempfile.TemporaryDirectory() as tmp_dir: + adoc_tmplt = Path(os.path.join(Config.template_dir, f"{adoc_file}.adoc")) + copied_tmplt = Path(tmp_dir) / f"{adoc_file}.adoc" + shutil.copy(adoc_tmplt, copied_tmplt) - env = utils.get_environment() - mrc_image = "test/input_files/brt_inputs/2013-1220-dA30_5-BSC-1_10.mrc" - mrc_file = Path(os.path.join(Config.proj_dir(env), mrc_image)) - - updated_adoc = utils.update_adoc( - adoc_fp=local_tmplt, - tg_fp=mrc_file, - montage=montage, - gold=gold, - focus=focus, - fiducialless=fiducialless, - trackingMethod=trackingMethod, - TwoSurfaces=TwoSurfaces, - TargetNumberOfBeads=TargetNumberOfBeads, - LocalAlignments=LocalAlignments, - THICKNESS=THICKNESS, - ) + env = utils.get_environment() + mrc_image = "test/input_files/brt_inputs/2013-1220-dA30_5-BSC-1_10.mrc" + mrc_file = Path(os.path.join(Config.proj_dir(env), mrc_image)) + + updated_adoc = utils.update_adoc( + adoc_fp=copied_tmplt, + tg_fp=mrc_file, + montage=montage, + gold=gold, + focus=focus, + fiducialless=fiducialless, + trackingMethod=trackingMethod, + TwoSurfaces=TwoSurfaces, + TargetNumberOfBeads=TargetNumberOfBeads, + LocalAlignments=LocalAlignments, + THICKNESS=THICKNESS, + ) - # These will raise FileNotFoundException if the file is missing - updated_adoc.unlink(missing_ok=False) - local_tmplt.unlink(missing_ok=False) + assert updated_adoc.exists() + assert copied_tmplt.exists() def test_update_adoc_bad_surfaces(mock_nfs_mount): @@ -195,30 +198,30 @@ def test_update_adoc_bad_surfaces(mock_nfs_mount): LocalAlignments = 0 THICKNESS = 30 - adoc_tmplt = Path(os.path.join(Config.template_dir, f"{adoc_file}.adoc")) - local_tmplt = Path.cwd() / f"{adoc_file}.adoc" - shutil.copy(adoc_tmplt, local_tmplt) + with tempfile.TemporaryDirectory() as tmp_dir: + adoc_tmplt = Path(os.path.join(Config.template_dir, f"{adoc_file}.adoc")) + copied_tmplt = Path(tmp_dir) / f"{adoc_file}.adoc" + shutil.copy(adoc_tmplt, copied_tmplt) - env = utils.get_environment() - mrc_image = "test/input_files/brt_inputs/2013-1220-dA30_5-BSC-1_10.mrc" - mrc_file = Path(os.path.join(Config.proj_dir(env), mrc_image)) - - with pytest.raises(signals.FAIL) as fail_msg: - updated_adoc = utils.update_adoc( - adoc_fp=local_tmplt, - tg_fp=mrc_file, - montage=montage, - gold=gold, - focus=focus, - fiducialless=fiducialless, - trackingMethod=trackingMethod, - TwoSurfaces=TwoSurfaces, - TargetNumberOfBeads=TargetNumberOfBeads, - LocalAlignments=LocalAlignments, - THICKNESS=THICKNESS, - ) - assert "Unable to resolve SurfacesToAnalyze" in str(fail_msg.value) - local_tmplt.unlink(missing_ok=False) + env = utils.get_environment() + mrc_image = "test/input_files/brt_inputs/2013-1220-dA30_5-BSC-1_10.mrc" + mrc_file = Path(os.path.join(Config.proj_dir(env), mrc_image)) + + with pytest.raises(signals.FAIL) as fail_msg: + updated_adoc = utils.update_adoc( + adoc_fp=copied_tmplt, + tg_fp=mrc_file, + montage=montage, + gold=gold, + focus=focus, + fiducialless=fiducialless, + trackingMethod=trackingMethod, + TwoSurfaces=TwoSurfaces, + TargetNumberOfBeads=TargetNumberOfBeads, + LocalAlignments=LocalAlignments, + THICKNESS=THICKNESS, + ) + assert "Unable to resolve SurfacesToAnalyze" in str(fail_msg.value) # Longer-running tests - comment-out following line to run @@ -243,6 +246,90 @@ def test_mrc_to_movie(mock_nfs_mount): # mrc_list = utils.gen_fps.__wrapped__(input_path, [image_path]) asset = utils.mrc_to_movie.__wrapped__(mrc_filepath, "adjusted", "recMovie") - print(f"asset= {asset}") assert type(asset) == dict assert "adjusted_recMovie.mp4" in asset["path"] + + +def test_copy_template(mock_nfs_mount): + """ + Tests that adoc template get copied to working directory + """ + with tempfile.TemporaryDirectory() as tmp_dir: + adoc_fp = utils.copy_template(working_dir=tmp_dir, template_name="plastic_brt") + adoc_fp = utils.copy_template(working_dir=tmp_dir, template_name="cryo_brt") + tmp_path = Path(tmp_dir) + assert tmp_path.exists() + assert Path(tmp_path / "plastic_brt.adoc").exists() + assert Path(tmp_path / "cryo_brt.adoc").exists() + + +def test_copy_template_missing(mock_nfs_mount): + """ + Tests that adoc template get copied to working directory + """ + with tempfile.TemporaryDirectory() as tmp_dir: + with pytest.raises(FileNotFoundError) as fnfe: + adoc_fp = utils.copy_template( + working_dir=tmp_dir, template_name="no_such_tmplt" + ) + assert "no_such_tmplt" in str(fnfe.value) + + +def test_copy_workdirs_small(mock_nfs_mount): + """ + Tests that the workdir is copied to the assets dir. This uses toy data only. + """ + proj_dir = Config.proj_dir(utils.get_environment()) + input_dir = "test/input_files/dm_inputs/Projects/Lab/PI" + image_name = "P6_J130_fsc_iteration_001.png" + input_path = Path(proj_dir) / input_dir + image_path = Path(proj_dir) / input_dir / image_name + assert image_path.exists() + image_filepath = FilePath(input_dir=input_path, fp_in=image_path) + + # copy image to working dir to verify test + shutil.copy(image_path, image_filepath.working_dir) + + workdest = utils.copy_workdirs.__wrapped__(image_filepath) + assert workdest.exists() + assert (workdest / image_name).exists() + # Clean up + assert "work_dir" in str(workdest.parent) + shutil.rmtree(workdest.parent) + + +# @pytest.mark.skip(reason="This test takes a long time to run") +def test_copy_workdirs_large(mock_nfs_mount): + """ + Tests that the entire workdir is copied to the assets dir + NOTE: This test assumes the availability of workdir in Assets after a + run of test_brt. + """ + proj_dir = Config.proj_dir(utils.get_environment()) + test_dir = "test/input_files/brt_inputs/Projects/test/" + test_assets_dir = "test/input_files/brt_inputs/Assets/test/" + image_name = "fake_image.mrc" + input_path = Path(proj_dir) / test_dir + assets_path = Path(proj_dir) / test_assets_dir + input_path.mkdir() + image_path = Path(proj_dir) / test_dir / image_name + test_filepath = FilePath(input_dir=input_path, fp_in=image_path) + + # copy entire Asset working dir from previous brt test run to test work_dir + asset_path = ( + Path(proj_dir) / "test/input_files/brt_inputs/Assets/2013-1220-dA30_5-BSC-1_10" + ) + filelist = glob.glob(f"{asset_path.as_posix()}/work_dir_*/*/*") + for item in filelist: + if os.path.isdir(item): + shutil.copytree(item, test_filepath.working_dir, dirs_exist_ok=True) + else: + shutil.copy(item, test_filepath.working_dir) + + workdest = utils.copy_workdirs.__wrapped__(test_filepath) # Actual test + + assert workdest.exists() + # Clean up + assert "work_dir" in str(workdest.parent) + shutil.rmtree(input_path) + shutil.rmtree(assets_path)