Merge pull request #70 from catalystneuro/final-huszar-conversion

Final Huszar Conversion

buzsaki_lab_to_nwb/huszar_hippocampus_dynamics/behaviorinterface.py

@@ -9,6 +9,7 @@
 from pynwb.file import NWBFile, TimeIntervals, TimeSeries
 from scipy.io import loadmat as loadmat_scipy
 from pymatreader import read_mat
+import warnings
 
 from ndx_events import LabeledEvents
 
@@ -22,6 +23,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = Fal
         self.session_id = self.session_path.stem
 
         file_path = self.session_path / f"{self.session_id}.Behavior.mat"
+
         mat_file = read_mat(file_path)
 
         events_data = mat_file["behavior"]["events"]
@@ -83,8 +85,10 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = Fal
         data = []
         for sleep_state in state_label_names:
             values = sleep_state_dic[sleep_state]
-            if len(values) != 0 and isinstance(values[0], int):
+
+            if len(values) != 0 and type(values[0]) is not np.ndarray:
                 values = [values]
+
             for start_time, stop_time in values:
                 data.append(
                     dict(
@@ -152,6 +156,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = Fal
         self.session_id = self.session_path.stem
 
         file_path = self.session_path / f"{self.session_id}.Behavior.mat"
+
         mat_file = loadmat_scipy(file_path, simplify_cells=True)
 
         timestamps = mat_file["behavior"]["timestamps"]

buzsaki_lab_to_nwb/huszar_hippocampus_dynamics/convert_all_sessions.py

@@ -3,19 +3,25 @@
 
 import concurrent.futures
 import psutil
+import json
+import shutil
+
 
 if __name__ == "__main__":
     # Parameters for conversion
-    stub_test = True  # Converts a only a stub of the data for quick iteration and testing
-    verbose = True
+    stub_test = False  # Converts a only a stub of the data for quick iteration and testing
+    verbose = False
     write_electrical_series = True  # Write the electrical series to the NWB file
     iterator_opts = dict(buffer_gb=1.0, display_progress=verbose)
 
-    output_dir_path = Path.home() / "conversion_nwb"
+    output_dir_path = Path.home() / "final_conversion" / "HuszarR"  # "conversion_nwb"
+
     project_root_path = Path("/shared/catalystneuro/HuszarR/optotagCA1")
 
     all_condition_paths = (path for path in project_root_path.iterdir() if path.is_dir())
 
+    excluded = dict()
+
     session_dir_path_list = []
 
     for condition in all_condition_paths:
@@ -36,7 +42,7 @@ def worker(session_dir_path):
                 verbose=verbose,
             )
         except Exception as e:
-            print(e)
+            print(f"ERROR ({str(session_dir_path.relative_to(project_root_path))}): {str(e)}")
 
     # Create a pool of worker processes
     num_physical_cores = psutil.cpu_count(logical=False)

buzsaki_lab_to_nwb/huszar_hippocampus_dynamics/convert_session.py

@@ -3,6 +3,7 @@
 from neuroconv.utils import load_dict_from_file, dict_deep_update
 from buzsaki_lab_to_nwb.huszar_hippocampus_dynamics import HuzsarNWBConverter
 from pathlib import Path
+import warnings
 
 
 def session_to_nwbfile(session_dir_path, output_dir_path, stub_test=False, write_electrical_series=True, verbose=False):
@@ -37,10 +38,10 @@ def session_to_nwbfile(session_dir_path, output_dir_path, stub_test=False, write
             conversion_options.update(
                 Recording=dict(stub_test=stub_test, write_electrical_series=write_electrical_series)
             )
-        else:
+        elif verbose:
             print(f"Skipping recording interface for {session_id} because the file {file_path} does not have any data.")
 
-    else:
+    elif verbose:
         print(f"Skipping recording interface for {session_id} because the file {file_path} does not exist.")
 
     # Add LFP
@@ -58,36 +59,44 @@ def session_to_nwbfile(session_dir_path, output_dir_path, stub_test=False, write
             source_data.update(LFP=dict(file_path=str(file_path), xml_file_path=str(xml_file_path)))
             conversion_options.update(LFP=dict(stub_test=stub_test, write_electrical_series=write_electrical_series))
 
-        else:
-            print(f"Skipping LFP interface for {session_id} because the file {file_path} does not have any data.")
+        elif verbose:
+            warnings.warn(
+                f"Skipping LFP interface for {session_id} because the file {file_path} does not have any data."
+            )
 
-    else:
-        print(f"Skipping LFP interface for {session_id} because the file {file_path} does not exist.")
+    elif verbose:
+        warnings.warn(f"Skipping LFP interface for {session_id} because the file {file_path} does not exist.")
 
-    write_ecephys_metadata = (not raw_recording_file_available) and (not lfp_file_available)
+    # Add behavior data
+    file_path = session_dir_path / f"{session_id}.Behavior.mat"
+    if file_path.is_file():
+        write_ecephys_metadata = (not raw_recording_file_available) and (not lfp_file_available)
 
-    # Add sorter
-    file_path = session_dir_path / f"{session_id}.spikes.cellinfo.mat"
-    source_data.update(Sorting=dict(file_path=str(file_path), verbose=verbose))
-    conversion_options.update(Sorting=dict(write_ecephys_metadata=write_ecephys_metadata))
+        # Add sorter
+        file_path = session_dir_path / f"{session_id}.spikes.cellinfo.mat"
+        source_data.update(Sorting=dict(file_path=str(file_path), verbose=verbose))
+        conversion_options.update(Sorting=dict(write_ecephys_metadata=write_ecephys_metadata))
 
-    # Add behavior data
-    source_data.update(Behavior8Maze=dict(folder_path=str(session_dir_path)))
-    conversion_options.update(Behavior8Maze=dict(stub_test=stub_test))
+        # Add linear track behavior
+        source_data.update(Behavior8Maze=dict(folder_path=str(session_dir_path)))
+        conversion_options.update(Behavior8Maze=dict(stub_test=stub_test))
 
-    source_data.update(BehaviorSleep=dict(folder_path=str(session_dir_path)))
+        # Add reward events in linear track
+        source_data.update(BehaviorRewards=dict(folder_path=str(session_dir_path)))
 
-    # Add epochs
-    source_data.update(Epochs=dict(folder_path=str(session_dir_path)))
+        # Add trials
+        source_data.update(Trials=dict(folder_path=str(session_dir_path)))
 
-    # Add trials
-    source_data.update(Trials=dict(folder_path=str(session_dir_path)))
+    elif verbose:
+        warnings.warn(
+            f"Behavior file not found: {file_path}. Skipping sorting, rewards, maze, and trials interfaces. \n"
+        )
 
-    # Add linear track behavior
-    source_data.update(Behavior8Maze=dict(folder_path=str(session_dir_path)))
+    # Add sleep data
+    source_data.update(BehaviorSleep=dict(folder_path=str(session_dir_path)))
 
-    # Add reward events in linear track
-    source_data.update(BehaviorRewards=dict(folder_path=str(session_dir_path)))
+    # Add epochs
+    source_data.update(Epochs=dict(folder_path=str(session_dir_path)))
 
     # Add ripple events
     source_data.update(RippleEvents=dict(folder_path=str(session_dir_path)))
@@ -120,6 +129,13 @@ def session_to_nwbfile(session_dir_path, output_dir_path, stub_test=False, write
     verbose = True
     output_dir_path = Path.home() / "conversion_nwb"
     project_root = Path("/shared/catalystneuro/HuszarR/optotagCA1")
-    session_dir_path = project_root / "e13" / "e13_26m1" / "e13_26m1_211119"
+
+    # # Still session error
+    session_dir_path = project_root / "e14" / "e14_2m3" / "e14_2m3_201208"
+
+    # # strptime() argument 1 must be str, not numpy.ndarray
+    # session_dir_path = project_root / "e13" / "e13_26m1" / "e13_26m1_211116"
+
+    # session_dir_path = project_root / "e13" / "e13_26m1" / "e13_26m1_211119"
     assert session_dir_path.is_dir()
     nwbfile = session_to_nwbfile(session_dir_path, output_dir_path, stub_test=stub_test, verbose=verbose)

buzsaki_lab_to_nwb/huszar_hippocampus_dynamics/converter.py

@@ -43,7 +43,7 @@ class HuzsarNWBConverter(NWBConverter):
     def __init__(self, source_data: dict, verbose: bool = True):
         super().__init__(source_data=source_data, verbose=verbose)
 
-        self.session_folder_path = Path(self.data_interface_objects["Behavior8Maze"].source_data["folder_path"])
+        self.session_folder_path = Path(self.data_interface_objects["RippleEvents"].source_data["folder_path"])
         self.session_id = self.session_folder_path.stem
 
     def get_metadata(self):
@@ -52,9 +52,11 @@ def get_metadata(self):
         assert session_file.is_file(), f"Session file not found: {session_file}"
 
         session_mat = loadmat_scipy(session_file, simplify_cells=True)
-        date = session_mat["session"]["general"]["date"]  # This does not contain the time
-        # Conver date str to date object
-        date = datetime.strptime(date, "%Y-%m-%d")
+        date = self.session_id.split("_")[2]  # This does not contain the time
+
+        # Convert date str to date object
+
+        date = datetime.strptime("20" + date, "%Y%m%d")
         # Build a datetime object and add the timezone from NY
         date = datetime.combine(date, datetime.min.time())
         tzinfo = ZoneInfo("America/New_York")  # This is the standard library
@@ -75,6 +77,9 @@ def get_metadata(self):
         # Add Subject metadata
         subject_metadata = session_mat["session"]["animal"]
         metadata["Subject"]["subject_id"] = subject_metadata["name"]
+
+        if metadata["Subject"]["subject_id"] == "DATA":
+            metadata["Subject"]["subject_id"] = "_".join(metadata["NWBFile"]["session_id"].split("_")[:2])
         metadata["Subject"]["sex"] = subject_metadata["sex"][0]
         metadata["Subject"]["strain"] = subject_metadata["strain"]
         metadata["Subject"]["genotype"] = subject_metadata["geneticLine"]

buzsaki_lab_to_nwb/huszar_hippocampus_dynamics/epochsinterface.py

@@ -4,6 +4,7 @@
 from neuroconv.utils.json_schema import FolderPathType
 from pynwb.file import NWBFile
 from pymatreader import read_mat
+import numpy as np
 
 
 class HuszarEpochsInterface(BaseDataInterface):
@@ -18,10 +19,14 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = Fal
         assert session_file_path.is_file(), session_file_path
         mat_file = read_mat(session_file_path)
 
-        epoch_list = mat_file["session"]["epochs"]
+        epoch_list = mat_file["session"].get("epochs", [])  # Include epochs if present
+
+        if type(epoch_list) is not np.ndarray and not isinstance(epoch_list, list):
+            epoch_list = [epoch_list]
 
         for epoch in epoch_list:
+            epoch_name = epoch["name"]
             start_time = float(epoch["startTime"])
-            stop_time = float(epoch["stopTime"])
-
-            nwbfile.add_epoch(start_time=start_time, stop_time=stop_time)
+            if epoch.get("stopTime"):
+                stop_time = float(epoch["stopTime"])
+                nwbfile.add_epoch(start_time=start_time, stop_time=stop_time)