Merge pull request #6 from ArcanaFramework/develop

Updates to match upstream packages

arcana/xnat/cli/tests/test_cli_add_columns.py

@@ -17,7 +17,7 @@ def test_add_source_xnat(dataset, cli_runner, arcana_home, work_dir):
         [
             dataset_locator,
             "a_source",
-            "fileformats.text:Plain",
+            "text/text-file",
             "--path",
             "file1",
             "--row-frequency",
@@ -47,7 +47,7 @@ def test_add_sink_xnat(dataset, work_dir, arcana_home, cli_runner):
         [
             dataset_locator,
             "a_sink",
-            "fileformats.text:Plain",
+            "text/text-file",
             "--path",
             "deriv",
             "--row-frequency",

arcana/xnat/data/api.py

@@ -29,7 +29,7 @@
 from arcana.core.utils.serialize import asdict
 from arcana.core.data.tree import DataTree
 from arcana.core.data.entry import DataEntry
-from arcana.stdlib import Clinical
+from arcana.common import Clinical
 
 
 logger = logging.getLogger("arcana")
@@ -358,14 +358,14 @@ def download_files(self, entry: DataEntry, download_dir: Path) -> Path:
                 self.connection.download_stream(
                     entry.uri + "/files", f, format="zip", verbose=True
                 )
-            # Extract downloaded zip file
-            expanded_dir = download_dir / "expanded"
-            try:
-                with ZipFile(zip_path) as zip_file:
-                    zip_file.extractall(expanded_dir)
-            except BadZipfile as e:
-                raise ArcanaError(f"Could not unzip file '{zip_path}' ({e})") from e
-            data_path = glob(str(expanded_dir) + "/**/files", recursive=True)[0]
+        # Extract downloaded zip file
+        expanded_dir = download_dir / "expanded"
+        try:
+            with ZipFile(zip_path) as zip_file:
+                zip_file.extractall(expanded_dir)
+        except BadZipfile as e:
+            raise ArcanaError(f"Could not unzip file '{zip_path}' ({e})") from e
+        data_path = next(expanded_dir.glob("**/files"))
         return data_path
 
     def upload_files(self, cache_path: Path, entry: DataEntry):
@@ -564,6 +564,9 @@ def get_xrow(self, row: DataRow):
             elif row.frequency == Clinical.session:
                 xrow = xproject.experiments[row.frequency_id("session")]
             else:
+                # For rows that don't have a place within the standard XNAT hierarchy,
+                # e.g. groups, we create a dummy subject with an escaped name to hold
+                # the associated data
                 xrow = self.connection.classes.SubjectData(
                     label=self.make_row_name(row), parent=xproject
                 )

arcana/xnat/data/cs.py

@@ -7,8 +7,8 @@
 import logging
 from pathlib import Path
 import attrs
-from fileformats.core.base import FileSet
-from arcana.stdlib import Clinical
+from fileformats.core import FileSet
+from arcana.common import Clinical
 from arcana.core.data.space import DataSpace
 from arcana.core.data.row import DataRow
 from arcana.core.data.entry import DataEntry
@@ -92,8 +92,7 @@ def get_fileset(self, entry: DataEntry, datatype: type) -> FileSet:
         )
         if entry.is_derivative:
             # entry is in input mount
-            stem_path = self.entry_fspath(entry)
-            fspaths = list(stem_path.iterdir())
+            resource_path = self.entry_fspath(entry)
         else:
             path = re.match(
                 r"/data/(?:archive/)?projects/[a-zA-Z0-9\-_]+/"
@@ -105,16 +104,16 @@ def get_fileset(self, entry: DataEntry, datatype: type) -> FileSet:
                 path = path.replace("scans", "SCANS").replace("resources/", "")
             path = path.replace("resources", "RESOURCES")
             resource_path = input_mount / path
-            fspaths = list(resource_path.iterdir())
+        fspaths = list(p for p in resource_path.iterdir() if not p.name.endswith("_catalog.xml"))
         return datatype(fspaths)
 
     def put_fileset(self, fileset: FileSet, entry: DataEntry) -> FileSet:
         if not entry.is_derivative:
             super().put_fileset(fileset, entry)  # Fallback to API access
-        cached = fileset.copy_to(
+        cached = fileset.copy(
             dest_dir=self.output_mount,
             make_dirs=True,
-            stem=entry.path.split("/")[-1].split("@")[0],
+            new_stem=entry.path.split("/")[-1].split("@")[0],
             trim=False,
             overwrite=True,
         )

arcana/xnat/data/tests/test_store.py

@@ -8,9 +8,10 @@
 from functools import reduce
 import itertools
 import pytest
+from pydra.utils.hash import hash_single, Cache
 from fileformats.generic import File
 from fileformats.field import Text as TextField
-from arcana.stdlib import Clinical
+from arcana.common import Clinical
 from arcana.core.data.set import Dataset
 from arcana.xnat.data import XnatViaCS
 from arcana.core.utils.serialize import asdict
@@ -178,3 +179,10 @@ def test_provenance_roundtrip(datatype: type, value: str, simple_dataset: Datase
         data_store.put_provenance(provenance, entry)  # Save the provenance
         reloaded_provenance = data_store.get_provenance(entry)  # reload the provenance
         assert provenance == reloaded_provenance
+
+
+def test_dataset_bytes_hash(static_dataset):
+
+    hsh = hash_single(static_dataset, Cache({}))
+    # Check hashing is stable
+    assert hash_single(static_dataset, Cache({})) == hsh

arcana/xnat/deploy/command.py

@@ -5,7 +5,7 @@
 from fileformats.core import FileSet
 from arcana.core.deploy.command.base import ContainerCommand
 from arcana.xnat.data import XnatViaCS
-from arcana.stdlib import Clinical
+from arcana.common import Clinical
 from arcana.core.utils.serialize import ClassResolver
 
 if ty.TYPE_CHECKING:
@@ -15,7 +15,7 @@
 @attrs.define(kw_only=True)
 class XnatCommand(ContainerCommand):
 
-    image: XnatApp = None
+    image: ty.Optional[XnatApp] = None
 
     # Hard-code the data_space of XNAT commands to be clinical
     DATA_SPACE = Clinical

arcana/xnat/deploy/tests/test_app.py

@@ -47,7 +47,6 @@ def run_spec(
                     "arcana",
                     "arcana-xnat",
                     "fileformats",
-                    "fileformats-testing",
                     "fileformats-medimage",
                     "pydra",
                 ],
@@ -82,6 +81,7 @@ def run_spec(
                     "arcana-bids",
                     "fileformats",
                     "fileformats-medimage",
+                    "fileformats-medimage-extras",
                     "pydra",
                 ],
             },

arcana/xnat/utils/testing.py

@@ -5,7 +5,7 @@
 import tempfile
 import attrs
 import xnat
-from arcana.stdlib import Clinical
+from arcana.common import Clinical
 from arcana.core.data.space import DataSpace
 from arcana.core.data.row import DataRow
 from arcana.testing.data.blueprint import TestDatasetBlueprint, FileSetEntryBlueprint
@@ -30,9 +30,9 @@ class TestXnatDatasetBlueprint(TestDatasetBlueprint):
     # Overwrite attributes in core blueprint class
     space: type = Clinical
     hierarchy: list[DataSpace] = ["subject", "session"]
-    filesets: list[str] = None
+    filesets: ty.Optional[list[str]] = None
 
-    def make_entries(self, row: DataRow, source_data: Path = None):
+    def make_entries(self, row: DataRow, source_data: ty.Optional[Path] = None):
         logger.debug("Making entries in %s row: %s", row, self.scans)
         xrow = row.dataset.store.get_xrow(row)
         xclasses = xrow.xnat_session.classes
@@ -50,7 +50,7 @@ def make_entries(self, row: DataRow, source_data: Path = None):
                     source_fallback=True,
                     escape_source_name=False,
                 )
-                item.copy_to(tmp_dir)
+                item.copy(tmp_dir)
                 xresource.upload_dir(tmp_dir)
 
 

conftest.py

@@ -20,7 +20,7 @@
 import medimages4tests.dummy.nifti
 import medimages4tests.dummy.dicom.mri.fmap.siemens.skyra.syngo_d13c
 from arcana.core.deploy.image.base import BaseImage
-from arcana.stdlib import Clinical
+from arcana.common import Clinical
 from arcana.core.data.set import Dataset
 from fileformats.medimage import NiftiGzX, NiftiGz, DicomSet, NiftiX
 from fileformats.text import Plain as Text
@@ -34,9 +34,13 @@
     ScanBlueprint as ScanBP,
 )
 from arcana.xnat.data.cs import XnatViaCS
-from pydra import set_input_validator
 
-set_input_validator(True)
+try:
+    from pydra import set_input_validator
+except ImportError:
+    pass
+else:
+    set_input_validator(True)
 
 # For debugging in IDE's don't catch raised exceptions and let the IDE
 # break at it
@@ -503,15 +507,15 @@ def command_spec():
         "task": "arcana.testing.tasks:concatenate",
         "inputs": {
             "first_file": {
-                "datatype": "fileformats.text:Plain",
+                "datatype": "text/text-file",
                 "field": "in_file1",
                 "default_column": {
                     "row_frequency": "session",
                 },
                 "help_string": "the first file to pass as an input",
             },
             "second_file": {
-                "datatype": "fileformats.text:Plain",
+                "datatype": "text/text-file",
                 "field": "in_file2",
                 "default_column": {
                     "row_frequency": "session",
@@ -521,7 +525,7 @@ def command_spec():
         },
         "outputs": {
             "concatenated": {
-                "datatype": "fileformats.text:Plain",
+                "datatype": "text/text-file",
                 "field": "out_file",
                 "help_string": "an output file",
             }
@@ -552,21 +556,21 @@ def bids_command_spec(mock_bids_app_executable):
             "configuration": {
                 "path": "anat/T1w",
             },
-            "datatype": "fileformats.medimage:NiftiGzX",
+            "datatype": "medimage/nifti-gz-x",
             "help_string": "T1-weighted image",
         },
         "T2w": {
             "configuration": {
                 "path": "anat/T2w",
             },
-            "datatype": "fileformats.medimage:NiftiGzX",
+            "datatype": "medimage/nifti-gz-x",
             "help_string": "T2-weighted image",
         },
         "DWI": {
             "configuration": {
                 "path": "dwi/dwi",
             },
-            "datatype": "fileformats.medimage:NiftiGzXBvec",
+            "datatype": "medimage/nifti-gz-x-bvec",
             "help_string": "DWI-weighted image",
         },
     }
@@ -576,14 +580,14 @@ def bids_command_spec(mock_bids_app_executable):
             "configuration": {
                 "path": "file1",
             },
-            "datatype": "fileformats.text:Plain",
+            "datatype": "text/text-file",
             "help_string": "an output file",
         },
         "file2": {
             "configuration": {
                 "path": "file2",
             },
-            "datatype": "fileformats.text:Plain",
+            "datatype": "text/text-file",
             "help_string": "another output file",
         },
     }

pyproject.toml

@@ -9,8 +9,9 @@ readme = "README.rst"
 requires-python = ">=3.8"
 dependencies = [
     "arcana",
-    "fileformats[extended] >=0.3.3",
-    "fileformats-medimage[extended] >=0.2.1",
+    "fileformats >=0.3.3",
+    "fileformats-medimage >=0.2.1",
+    "fileformats-medimage-extras >=0.1.3",
     "xnat==0.4.3",
 ]
 license = {file = "LICENSE"}