diff --git a/README.md b/README.md
index 3b6442e9..20ecc463 100644
--- a/README.md
+++ b/README.md
@@ -36,32 +36,32 @@ See the SWC-BIDS [specification](https://swc-bids.neuroinformatics.dev/) for mor
└── raw_data/
├── sub-001/
│ └── ses-001/
- │ │ ├── ephys/
- │ │ └── behav/
- │ └── histology/
+ │ ├── ephys/
+ │ └── behav/
+ │ └── anat/
└── sub-002/
└── ses-001/
│ ├── behav/
│ └── imaging/
└── ses-002/
- │ └── behav/
- └── histology/
+ └── behav/
+ └── anat/
```
```+
└── project_name/
- └── rawdata/
+ └── rawdata/[test_utils.py](tests%2Ftest_utils.py)
├── sub-001 /
- │ ├── ses-001/
- │ │ ├── ephys
- │ │ └── behav
- │ └── histology
+ │ └── ses-001/
+ │ ├── ephys
+ │ └── behav
+ │ └── anat
└── sub-002/
├── ses-001/
│ ├── behav
│ └── imaging
- ├── ses-002/
- │ └── behav
- └── histology
+ └── ses-002/
+ └── behav
+ └── anat
```
diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py
index 6bb395a1..212890e8 100644
--- a/datashuttle/configs/canonical_configs.py
+++ b/datashuttle/configs/canonical_configs.py
@@ -46,7 +46,7 @@ def get_datatypes() -> List[str]:
Canonical list of datatype flags based on
NeuroBlueprint.
"""
- return ["ephys", "behav", "funcimg", "histology"]
+ return ["ephys", "behav", "funcimg", "anat"]
def get_flags() -> List[str]:
diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py
index 27a505c6..83e78625 100644
--- a/datashuttle/configs/canonical_folders.py
+++ b/datashuttle/configs/canonical_folders.py
@@ -51,9 +51,9 @@ def get_datatype_folders(cfg: Configs) -> dict:
name="funcimg",
level="ses",
),
- "histology": Folder(
- name="histology",
- level="sub",
+ "anat": Folder(
+ name="anat",
+ level="ses",
),
}
diff --git a/datashuttle/datashuttle.py b/datashuttle/datashuttle.py
index c7055e79..2bae4a96 100644
--- a/datashuttle/datashuttle.py
+++ b/datashuttle/datashuttle.py
@@ -180,7 +180,7 @@ def make_folders(
folders are made.
datatype :
The datatype to make in the sub / ses folders.
- (e.g. "ephys", "behav", "histology"). If "all"
+ (e.g. "ephys", "behav", "anat"). If "all"
is selected, all datatypes permitted in
NeuroBlueprint will be created. If "" is passed
no datatype will be created.
diff --git a/docs/source/pages/documentation.md b/docs/source/pages/documentation.md
index bb208b5a..d65f3b59 100644
--- a/docs/source/pages/documentation.md
+++ b/docs/source/pages/documentation.md
@@ -93,7 +93,6 @@ ssh \
--overwrite_old_files
```
-
Now setup is complete! _Configuration_ settings can be edited at any time with the `update-config` command. Alternatively, custom *configuration* files can be supplied using the `supply-config` command (this simplifies setting up projects across multiple *local* machines).
Next, we can start setting up the project by automatically creating standardised project folder trees.
@@ -132,33 +131,31 @@ Another example call, which creates a range of subject and session folders, is s
```
datashuttle \
my_first_project \
+make-folders -sub 001@TO@003 -ses 010_@TIME@ -dt behav funcimg anat
+```
-make-sub-folders -sub 001@TO@003 -ses 010_@TIME@ -dt behav funcimg histology
```
├── sub-001/
-│ ├── ses-010_time-160248/
-│ │ ├── behav
-│ │ └── funcimg
-│ └── histology
+│ └── ses-010_time-160248/
+│ ├── behav
+│ └── funcimg
+│ └── anat
├── sub-002/
-│ ├── ses-010_time-160248/
-│ │ ├── behav
-│ │ └── funcimg
-│ └── histology
+│ └── ses-010_time-160248/
+│ └── behav
+│ └── funcimg
+│ └── anat
└── sub-003/
- ├── ses-010_time-160248/
- │ ├── behav
- │ └── funcimg
- └── histology
+ └── ses-010_time-160248/
+ ├── behav
+ ├── funcimg
+ └── anat
```
-
### Datatype Folders
-In [SWC-Blueprint](https://swc-blueprint.neuroinformatics.dev/specification.html), *datatypes* specify where acquired experimental data of currently supported types (`behav`, `ephys`, `funcimg` and `histology`) is stored. See the [*datatypes* section of the SWC-Blueprint for more details](https://swc-blueprint.neuroinformatics.dev/specification.html#datatype).
-
-At present, `histology` is saved to the `sub-` level, as it is assumed `histology` is conducted *ex vivo* and so session will be possible. Please don't hesitate to get into contact if you have an alternative use case.
+In [SWC-Blueprint](https://swc-blueprint.neuroinformatics.dev/specification.html), *datatypes* specify where acquired experimental data of currently supported types (`behav`, `ephys`, `funcimg` and `anat`) is stored. See the [*datatypes* section of the SWC-Blueprint for more details](https://swc-blueprint.neuroinformatics.dev/specification.html#datatype).
## Data Transfer
@@ -498,7 +495,7 @@ DataShuttle provides a number of keyword arguments to allow separate handling of
`all` : All *session* and non-*session* files and folders within a *subject* level folder (e.g. `sub-001`) will be transferred.
-`all_ses` : *Session* folders only (i.e. prefixed with `-ses`) will be transferred. Note that the only exception is the `histology` folder, the transfer of which is determined by the `-dt` flag (below).
+`all_ses` : *Session* folders only (i.e. prefixed with `-ses`) will be transferred.
`all_non_ses` : All files and folders that are not prefixed with `-sub` will be transferred. Any folders prefixed with `-ses` will not be transferred.
@@ -506,7 +503,7 @@ DataShuttle provides a number of keyword arguments to allow separate handling of
`all` : All *datatype* folders at the *subject* or *session* folder level will be transferred, as well as all files and folders within selected *session* folders.
-`all_datatype` : All *datatype* folders (i.e. folders with the pre-determined name: `behav`, `ephys`, `funcimg`, `histology`) residing at either the *subject* or *session* level will be
+`all_datatype` : All *datatype* folders (i.e. folders with the pre-determined name: `behav`, `ephys`, `funcimg`, `anat`) residing at the *session* level will be
transferred. Non-*datatype* folders at the *session* level will not be transferred
`all_ses_level_non_datatype` : Non *datatype* folders at the *session* level will not be transferred
@@ -519,7 +516,6 @@ Below, a number of examples are given to exemplify how these arguments effect da
├── a_project_related_file.json
├── sub-001/
│ ├── sub-001_extra-file.json
- │ ├── histology
│ └── ses-001/
│ ├── ses-001_extra-file.json
│ ├── behav/
@@ -528,7 +524,6 @@ Below, a number of examples are given to exemplify how these arguments effect da
│ └── ...
└── sub-002/
├── sub-002_extra-file.json
- ├── histology
└── ses-001/
├── behav/
│ └── ...
@@ -569,7 +564,7 @@ Would upload:
- Contents residing in the `sub-001` folder only.
- The file `sub-001_extra-file.json`
-- All *datatype* folder contents (`histology`, `behav`, `ephys`)
+- All *datatype* folder contents (`anat`, `behav`, `ephys`)
The command:
@@ -585,7 +580,7 @@ upload
Would upload:
- Contents residing in the `sub-001` folder only.
-- All *datatype* folder contents (`histology`, `behav`, `ephys`)
+- All *datatype* folder contents (`anat`, `behav`, `ephys`)
3) The final example shows the effect of transferring `all_non_sub` files only. The command:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index b5fdcb2a..143b71d8 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -321,7 +321,7 @@ def check_datatype_sub_ses_uploaded_correctly(
sub_names = glob_basenames(join(base_path_to_check, "*"))
assert sub_names == sorted(subs_to_upload)
- # Check ses are all uploaded + histology if transferred
+ # Check ses are all uploaded
if ses_to_upload:
for sub in subs_to_upload:
ses_names = glob_basenames(
@@ -331,42 +331,15 @@ def check_datatype_sub_ses_uploaded_correctly(
"*",
)
)
- if datatype_to_transfer == ["histology"]:
- assert ses_names == ["histology"]
- return # handle the case in which histology
- # only is transferred,
- # and there are no sessions to transfer.
-
- copy_datatype_to_transfer = (
- check_and_strip_within_sub_data_folders(
- ses_names, datatype_to_transfer
- )
- )
assert ses_names == sorted(ses_to_upload)
# check datatype folders in session folder
- if copy_datatype_to_transfer:
+ if datatype_to_transfer:
for ses in ses_names:
data_names = glob_basenames(
join(base_path_to_check, sub, ses, "*")
)
- assert data_names == sorted(copy_datatype_to_transfer)
-
-
-def check_and_strip_within_sub_data_folders(ses_names, datatype_to_transfer):
- """
- Check if datatype folders at the sub level are picked
- up when sessions are searched for with wildcard. Remove
- so that sessions can be explicitly tested next.
- """
- if "histology" in datatype_to_transfer:
- assert "histology" in ses_names
-
- ses_names.remove("histology")
- copy_ = copy.deepcopy(datatype_to_transfer)
- copy_.remove("histology")
- return copy_
- return datatype_to_transfer
+ assert data_names == sorted(datatype_to_transfer)
def make_and_check_local_project_folders(
diff --git a/tests/tests_integration/file_conflicts_pathtable.py b/tests/tests_integration/file_conflicts_pathtable.py
index 3227e8b4..56b63dd8 100644
--- a/tests/tests_integration/file_conflicts_pathtable.py
+++ b/tests/tests_integration/file_conflicts_pathtable.py
@@ -16,19 +16,19 @@ def get_pathtable(base_folder):
[base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "behav" / "behav.csv", False, False, False, "sub-001", "ses-003_date-20231901", "behav"],
[base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "ephys" / "ephys.bin", False, False, False, "sub-001", "ses-003_date-20231901", "ephys"],
[base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "non_data" / "non_data.mp4", False, False, True, "sub-001", "ses-003_date-20231901", None],
- [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "nondatatype_level_file.csv", False, False, True, "sub-001", "ses-003_date-20231901", None],
+ [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "nondatatype_level_file.csv", False, False, True, "sub-001", "ses-003_date-20231901", None],
[base_folder, Path("rawdata") / "sub-001" / "random-ses_level_file.mp4", False, True, False, "sub-001", None, None],
- [base_folder, Path("rawdata") / "sub-001" / "histology" / "sub-001_histology.file", False, False, False, "sub-001", None, "histology"],
+ [base_folder, Path("rawdata") / "sub-001" / "ses-004" / "anat" / "sub-001_anat.file", False, False, False, "sub-001", "ses-004", "anat"],
[base_folder, Path("rawdata") / "sub-002_random-value" / "sub-002_random-value.file", False, True, False, "sub-002_random-value", None, None],
- [base_folder, Path("rawdata") / "sub-002_random-value" / "ses-001" / "non_datatype_level_folder" / "file.csv", False, False, True, "sub-002_random-value", "ses-001", None],
+ [base_folder, Path("rawdata") / "sub-002_random-value" / "ses-001" / "non_datatype_level_folder" / "file.csv", False, False, True, "sub-002_random-value", "ses-001", None],
[base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-001" / "funcimg" / ".myfile.xlsx", False, False, False, "sub-003_date-20231901", "ses-001", "funcimg"],
- [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-003_date-20231901" / "nondatatype_level_file.csv", False, False, True, "sub-003_date-20231901", "ses-003_date-20231901", None],
+ [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-003_date-20231901" / "nondatatype_level_file.csv", False, False, True, "sub-003_date-20231901", "ses-003_date-20231901", None],
[base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-003_date-20231901" / "funcimg" / "funcimg.nii", False, False, False, "sub-003_date-20231901", "ses-003_date-20231901", "funcimg"],
- [base_folder, Path("rawdata") / "sub-003_date-20231901" / "seslevel_non-prefix_folder" / "nonlevel.mat", False, True, False, "sub-003_date-20231901", "seslevel_non-prefix_folder", None],
+ [base_folder, Path("rawdata") / "sub-003_date-20231901" / "seslevel_non-prefix_folder" / "nonlevel.mat", False, True, False, "sub-003_date-20231901", "seslevel_non-prefix_folder", None],
[base_folder, Path("rawdata") / "sub-003_date-20231901" / "sub-ses-level_file.txt", False, True, False, "sub-003_date-20231901", None, None],
- [base_folder, Path("rawdata") / "sub-003_date-20231901" / "histology" / ".histology.file", False, False, False, "sub-003_date-20231901", None, "histology"],
+ [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-004" / "anat" / ".anat.file", False, False, False, "sub-003_date-20231901", "ses-004", "anat"],
[base_folder, Path("rawdata") / "project_level_file.txt", True, False, False, None, None, None],
- [base_folder, Path("rawdata") / "sublevel_non_sub-prefix_folder" / "ses_non_folder.file", True, False, False, None, None, None],
+ [base_folder, Path("rawdata") / "sublevel_non_sub-prefix_folder" / "ses_non_folder.file", True, False, False, None, None, None],
]
diff --git a/tests/tests_integration/test_configs.py b/tests/tests_integration/test_configs.py
index 1ede531c..276e4132 100644
--- a/tests/tests_integration/test_configs.py
+++ b/tests/tests_integration/test_configs.py
@@ -332,6 +332,7 @@ def test_supplied_config_file_changes_wrong_order(self, project, tmp_path):
any order and just converted to dict?
"""
bad_order_configs_path = project._datashuttle_path / "new_configs.yaml"
+
good_order_configs = test_utils.get_test_config_arguments_dict(
tmp_path, project.project_name
)
diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py
index 98f80c35..14ec77da 100644
--- a/tests/tests_integration/test_filesystem_transfer.py
+++ b/tests/tests_integration/test_filesystem_transfer.py
@@ -137,12 +137,12 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download):
["behav"],
["ephys"],
["funcimg"],
- ["histology"],
+ ["anat"],
["behav", "ephys"],
- ["ephys", "histology"],
- ["behav", "ephys", "histology"],
- ["funcimg", "histology", "behav"],
- ["behav", "ephys", "funcimg", "histology"],
+ ["ephys", "anat"],
+ ["behav", "ephys", "anat"],
+ ["funcimg", "anat", "behav"],
+ ["behav", "ephys", "funcimg", "anat"],
],
)
@pytest.mark.parametrize("upload_or_download", ["upload", "download"])
@@ -179,10 +179,10 @@ def test_transfer_empty_folder_specific_dataal_data(
@pytest.mark.parametrize(
"datatype_to_transfer",
[
- ["histology"],
+ ["anat"],
["behav", "ephys"],
- ["funcimg", "histology", "behav"],
- ["behav", "ephys", "funcimg", "histology"],
+ ["funcimg", "anat", "behav"],
+ ["behav", "ephys", "funcimg", "anat"],
],
)
@pytest.mark.parametrize("upload_or_download", ["upload" "download"])
@@ -223,7 +223,7 @@ def test_transfer_empty_folder_specific_subs(
@pytest.mark.parametrize("sub_idx_to_upload", [[0], [1, 2], [0, 1, 2]])
@pytest.mark.parametrize(
"datatype_to_transfer",
- [["ephys"], ["funcimg", "histology", "behav"]],
+ [["ephys"], ["funcimg", "anat", "behav"]],
)
@pytest.mark.parametrize("upload_or_download", ["upload", "download"])
def test_transfer_empty_folder_specific_ses(
@@ -429,10 +429,10 @@ def test_rclone_overwrite_modified_file(
the version in source is newer than target.
"""
path_to_test_file = (
- Path("rawdata") / "sub-001" / "histology" / "test_file.txt"
+ Path("rawdata") / "sub-001" / "ses-001" / "anat" / "test_file.txt"
)
- project.make_folders("sub-001", datatype="histology")
+ project.make_folders("sub-001", "ses-001", datatype="anat")
local_test_file_path = project.cfg["local_path"] / path_to_test_file
central_test_file_path = (
diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py
index 88167bbb..42951f2f 100644
--- a/tests/tests_integration/test_logging.py
+++ b/tests/tests_integration/test_logging.py
@@ -271,8 +271,9 @@ def test_logs_upload_and_download(
assert "Using config file from" in log
assert "Local file system at" in log
- assert """ "--include" "sub-11/histology/**" """ in log
- assert """/central/test_project/rawdata""" in log
+ assert "--include" in log
+ assert "sub-11/ses-123/anat/**" in log
+ assert "/central/test_project/rawdata" in log
assert "Waiting for checks to finish" in log
@pytest.mark.parametrize("upload_or_download", ["upload", "download"])
diff --git a/tests/tests_integration/test_make_folders.py b/tests/tests_integration/test_make_folders.py
index 3011c048..dde4a6e4 100644
--- a/tests/tests_integration/test_make_folders.py
+++ b/tests/tests_integration/test_make_folders.py
@@ -134,16 +134,14 @@ def test_explicitly_session_list(self, project):
join(base_folder, sub, ses, "funcimg")
)
test_utils.check_and_cd_folder(
- join(base_folder, sub, "histology")
+ join(base_folder, sub, ses, "anat")
)
@pytest.mark.parametrize("behav", [True, False])
@pytest.mark.parametrize("ephys", [True, False])
@pytest.mark.parametrize("funcimg", [True, False])
- @pytest.mark.parametrize("histology", [True, False])
- def test_every_datatype_passed(
- self, project, behav, ephys, funcimg, histology
- ):
+ @pytest.mark.parametrize("anat", [True, False])
+ def test_every_datatype_passed(self, project, behav, ephys, funcimg, anat):
"""
Check every combination of data type used and ensure only the
correct ones are made.
@@ -157,8 +155,8 @@ def test_every_datatype_passed(
datatypes_to_make.append("ephys")
if funcimg:
datatypes_to_make.append("funcimg")
- if histology:
- datatypes_to_make.append("histology")
+ if anat:
+ datatypes_to_make.append("anat")
# Make folder tree
subs = ["sub-001", "sub-002"]
@@ -176,7 +174,7 @@ def test_every_datatype_passed(
"behav": behav,
"ephys": ephys,
"funcimg": funcimg,
- "histology": histology,
+ "anat": anat,
},
)
@@ -188,7 +186,7 @@ def test_custom_folder_names(self, project):
# Change folder names to custom names
project.cfg.datatype_folders["ephys"].name = "change_ephys"
project.cfg.datatype_folders["behav"].name = "change_behav"
- project.cfg.datatype_folders["histology"].name = "change_histology"
+ project.cfg.datatype_folders["anat"].name = "change_anat"
project.cfg.datatype_folders["funcimg"].name = "change_funcimg"
# Make the folders
@@ -215,7 +213,7 @@ def test_custom_folder_names(self, project):
)
test_utils.check_and_cd_folder(
- join(base_folder, sub, "change_histology")
+ join(base_folder, sub, ses, "change_anat")
)
@pytest.mark.parametrize(
@@ -223,8 +221,8 @@ def test_custom_folder_names(self, project):
[
["all"],
["ephys", "behav"],
- ["ephys", "behav", "histology"],
- ["ephys", "behav", "histology", "funcimg"],
+ ["ephys", "behav", "anat"],
+ ["ephys", "behav", "anat", "funcimg"],
["funcimg", "ephys"],
["funcimg"],
],
@@ -244,13 +242,10 @@ def test_datatypes_subsection(self, project, files_to_test):
base_folder = test_utils.get_top_level_folder_path(project)
# Check at the subject level
- sub_file_names = test_utils.glob_basenames(
+ test_utils.glob_basenames(
join(base_folder, sub, "*"),
exclude=ses,
)
- if "histology" in files_to_test:
- assert "histology" in sub_file_names
- files_to_test.remove("histology")
# Check at the session level
ses_file_names = test_utils.glob_basenames(
@@ -259,7 +254,9 @@ def test_datatypes_subsection(self, project, files_to_test):
)
if files_to_test == ["all"]:
- assert ses_file_names == sorted(["ephys", "behav", "funcimg"])
+ assert ses_file_names == sorted(
+ ["ephys", "behav", "funcimg", "anat"]
+ )
else:
assert ses_file_names == sorted(files_to_test)
diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py
index 73070bd3..989dbfbf 100644
--- a/tests/tests_integration/test_ssh_file_transfer.py
+++ b/tests/tests_integration/test_ssh_file_transfer.py
@@ -151,9 +151,9 @@ def central_from_local(self, path_):
["all_datatype"],
["behav"],
["ephys"],
- ["histology"],
+ ["anat"],
["funcimg"],
- ["histology", "behav", "all_ses_level_non_datatype"],
+ ["anat", "behav", "all_ses_level_non_datatype"],
],
)
@pytest.mark.parametrize("upload_or_download", ["upload", "download"])
@@ -296,11 +296,6 @@ def make_pathtable_search_filter(self, sub_names, ses_names, datatype):
if sub == "all_non_sub":
extra_arguments += ["is_non_sub == True"]
else:
- if "histology" in datatype:
- sub_ses_dtype_arguments += [
- f"(parent_sub == '{sub}' & (parent_datatype == 'histology' | parent_datatype == 'histology'))"
- ]
-
for ses in ses_names:
if ses == "all_non_ses":
extra_arguments += [