Merge pull request #19 from catalystneuro/add_processed_trials_to_sep…

…arate_table

Add processed trials to separate table

src/constantinople_lab_to_nwb/mah_2024/interfaces/mah_2024_processedbehaviorinterface.py

@@ -7,6 +7,8 @@
 import pandas as pd
 from ndx_structured_behavior.utils import loadmat
 from neuroconv import BaseDataInterface
+from neuroconv.utils import get_base_schema, DeepDict
+from pynwb.epoch import TimeIntervals
 from pynwb.file import NWBFile
 
 
@@ -35,6 +37,8 @@ def __init__(
 
         self.default_struct_name = default_struct_name
         self.date_index = date_index
+        self._side_name_mapping = {"L": "Left", "R": "Right"}
+        self._block_name_mapping = {1: "Mixed", 2: "High", 3: "Low"}
         super().__init__(file_path=file_path, verbose=verbose)
 
     def _read_file(self, file_path: Union[str, Path]) -> pd.DataFrame:
@@ -80,24 +84,55 @@ def _transform_data(self, data: dict) -> pd.DataFrame:
             dataframe["side"] = side_to_add
 
         if "wait_thresh" in data:
-            dataframe["wait_thresh"] = data["wait_thresh"] * len(dataframe)
+            dataframe["wait_thresh"] = [data["wait_thresh"]] * len(dataframe)
 
         return dataframe
 
+    def get_metadata_schema(self) -> dict:
+        metadata_schema = super().get_metadata_schema()
+        metadata_schema["properties"]["Behavior"] = get_base_schema(tag="Behavior")
+        metadata_schema["properties"]["Behavior"].update(
+            required=["TimeIntervals"],
+            properties=dict(
+                TimeIntervals=dict(
+                    type="object",
+                    properties=dict(name=dict(type="string"), description=dict(type="string")),
+                )
+            ),
+        )
+        return metadata_schema
+
+    def get_metadata(self) -> dict:
+        metadata = super().get_metadata()
+        metadata["Behavior"] = dict(
+            TimeIntervals=dict(
+                name="processed_trials",
+                description="Contains the processed Bpod trials.",
+            )
+        )
+        return metadata
+
     def add_to_nwbfile(
         self,
         nwbfile: NWBFile,
         metadata: dict,
         column_name_mapping: Optional[dict] = None,
         column_descriptions: Optional[dict] = None,
+        trial_start_times: Optional[list] = None,
+        trial_stop_times: Optional[list] = None,
     ) -> None:
         dataframe = self._read_file(file_path=self.source_data["file_path"])
 
+        time_intervals_metadata = metadata["Behavior"]["TimeIntervals"]
+        trials_table = TimeIntervals(**time_intervals_metadata)
+
         if "side" in dataframe.columns:
-            side_mapping = {"L": "Left", "R": "Right"}
-            dataframe["side"] = dataframe["side"].map(side_mapping)
+            dataframe["side"] = dataframe["side"].map(self._side_name_mapping)
 
-        columns_with_boolean = ["hits", "vios", "optout"]
+        if "block" in dataframe.columns:
+            dataframe["block"] = dataframe["block"].map(self._block_name_mapping)
+
+        columns_with_boolean = ["catch", "hits", "vios", "optout"]
         for column in columns_with_boolean:
             if column in dataframe.columns:
                 dataframe[column] = dataframe[column].astype(bool)
@@ -106,9 +141,30 @@ def add_to_nwbfile(
         if column_name_mapping is not None:
             columns_to_add = [column for column in column_name_mapping.keys() if column in dataframe.columns]
 
-        trials = nwbfile.trials
-        if trials is None:
-            raise ValueError("Trials table not found in NWB file.")
+        num_trials = len(dataframe)
+        if nwbfile.trials is None:
+            assert (
+                trial_start_times is not None
+            ), "'trial_start_times' must be provided if trials table is not in acquisition."
+            assert (
+                trial_stop_times is not None
+            ), "'trial_stop_times' must be provided if trials table is not in acquisition."
+            assert (
+                len(trial_start_times) == num_trials
+            ), f"Length of 'trial_start_times' ({len(trial_start_times)}) must match the number of trials ({num_trials})."
+            assert (
+                len(trial_stop_times) == num_trials
+            ), f"Length of 'trial_stop_times' ({len(trial_stop_times)}) must match the number of trials ({num_trials})."
+        else:
+            trial_start_times = nwbfile.trials["start_time"][:]
+            trial_stop_times = nwbfile.trials["stop_time"][:]
+
+        for start_time, stop_time in zip(trial_start_times, trial_stop_times):
+            trials_table.add_row(
+                start_time=start_time,
+                stop_time=stop_time,
+                check_ragged=False,
+            )
 
         for column_name in columns_to_add:
             name = column_name_mapping.get(column_name, column_name) if column_name_mapping is not None else column_name
@@ -117,8 +173,10 @@ def add_to_nwbfile(
                 if column_descriptions is not None
                 else "no description"
             )
-            trials.add_column(
+            trials_table.add_column(
                 name=name,
                 description=description,
                 data=dataframe[column_name].values.tolist(),
             )
+
+        nwbfile.add_time_intervals(trials_table)

src/constantinople_lab_to_nwb/mah_2024/mah_2024_convert_all_sessions.py

@@ -218,7 +218,17 @@ def sessions_to_nwb(
     raw_behavior_folder_path = Path(r"/Volumes/T9/Constantinople/raw_Bpod")
     rat_info_folder_path = Path(r"/Volumes/T9/Constantinople/Rat_info")
 
+    # The column name mapping is used to rename the columns in the processed data to more descriptive column names.
     column_name_mapping = dict(
+        trainingstage="training_stage",
+        nic="nose_in_center",
+        catch="is_catch",
+        prob_cacth="catch_percentage",
+        adapt_block="num_trials_in_adaptation_blocks",
+        test_block="num_trials_in_mixed_blocks",
+        reward="reward_volume_ul",
+        reward_delay="reward_delay",
+        block="block_type",
         hits="is_rewarded",
         vios="is_violation",
         optout="is_opt_out",
@@ -227,6 +237,7 @@ def sessions_to_nwb(
         wait_thresh="wait_time_threshold",
         wait_for_cpoke="wait_for_center_poke",
         zwait_for_cpoke="z_scored_wait_for_center_poke",
+        # timeout="timeout",
         side="rewarded_port",
         lpoke="num_left_pokes",
         rpoke="num_right_pokes",
@@ -235,11 +246,20 @@ def sessions_to_nwb(
         rpokedur="duration_of_right_pokes",
         cpokedur="duration_of_center_pokes",
         rt="reaction_time",
-        slrt="side_poke_reaction_time",  # side led on = side poke
+        slrt="side_poke_reaction_time",
         ITI="inter_trial_interval",
     )
-    # The column descriptions are used to add descriptions to the columns in the processed data. (optional)
+    # The column descriptions are used to add descriptions to the columns in the processed data.
     column_descriptions = dict(
+        trainingstage="The stage of the training.",
+        nic="The time in seconds when the animal is required to maintain center port to initiate the trial (uniformly drawn from 0.8 - 1.2 seconds).",
+        catch="Whether the trial is a catch trial.",
+        prob_cacth="The percentage of catch trials.",
+        adapt_block="The number of trials in each high reward (20, 40, or 80μL) or low reward (5, 10, or 20μL) blocks.",
+        test_block="The number of trials in each mixed blocks.",
+        reward="The volume of reward in microliters.",
+        reward_delay="The delay in seconds to receive reward, drawn from exponential distribution with mean = 2.5 seconds.",
+        block="The block type (High, Low or Mixed). High and Low blocks are high reward (20, 40, or 80μL) or low reward (5, 10, or 20μL) blocks. The mixed blocks offered all volumes.",
         hits="Whether the subject received reward for each trial.",
         vios="Whether the subject violated the trial by not maintaining center poke for the time required by 'nose_in_center'.",
         optout="Whether the subject opted out for each trial.",
@@ -261,7 +281,6 @@ def sessions_to_nwb(
         wait_time_unthresholded="The wait time for the subject for each trial in seconds without removing outliers.",
         wait_thresh="The threshold in seconds to remove wait-times (mean + 1*std of all cumulative wait-times).",
     )
-
     nwbfile_folder_path = Path("/Users/weian/data/001169")
 
     overwrite = False

src/constantinople_lab_to_nwb/mah_2024/mah_2024_convert_session.py

@@ -179,8 +179,17 @@ def session_to_nwb(
     processed_behavior_file_path = Path("/Volumes/T9/Constantinople/A_Structs/ratTrial_C005.mat")
     # The row index of the date in the processed behavior file
     date_index = 0
-    # The column name mapping is used to rename the columns in the processed data to more descriptive column names. (optional)
+    # The column name mapping is used to rename the columns in the processed data to more descriptive column names.
     column_name_mapping = dict(
+        trainingstage="training_stage",
+        nic="nose_in_center",
+        catch="is_catch",
+        prob_cacth="catch_percentage",
+        adapt_block="num_trials_in_adaptation_blocks",
+        test_block="num_trials_in_mixed_blocks",
+        reward="reward_volume_ul",
+        reward_delay="reward_delay",
+        block="block_type",
         hits="is_rewarded",
         vios="is_violation",
         optout="is_opt_out",
@@ -189,6 +198,7 @@ def session_to_nwb(
         wait_thresh="wait_time_threshold",
         wait_for_cpoke="wait_for_center_poke",
         zwait_for_cpoke="z_scored_wait_for_center_poke",
+        # timeout="timeout",
         side="rewarded_port",
         lpoke="num_left_pokes",
         rpoke="num_right_pokes",
@@ -200,8 +210,17 @@ def session_to_nwb(
         slrt="side_poke_reaction_time",
         ITI="inter_trial_interval",
     )
-    # The column descriptions are used to add descriptions to the columns in the processed data. (optional)
+    # The column descriptions are used to add descriptions to the columns in the processed data.
     column_descriptions = dict(
+        trainingstage="The stage of the training.",
+        nic="The time in seconds when the animal is required to maintain center port to initiate the trial (uniformly drawn from 0.8 - 1.2 seconds).",
+        catch="Whether the trial is a catch trial.",
+        prob_cacth="The percentage of catch trials.",
+        adapt_block="The number of trials in each high reward (20, 40, or 80μL) or low reward (5, 10, or 20μL) blocks.",
+        test_block="The number of trials in each mixed blocks.",
+        reward="The volume of reward in microliters.",
+        reward_delay="The delay in seconds to receive reward, drawn from exponential distribution with mean = 2.5 seconds.",
+        block="The block type (High, Low or Mixed). High and Low blocks are high reward (20, 40, or 80μL) or low reward (5, 10, or 20μL) blocks. The mixed blocks offered all volumes.",
         hits="Whether the subject received reward for each trial.",
         vios="Whether the subject violated the trial by not maintaining center poke for the time required by 'nose_in_center'.",
         optout="Whether the subject opted out for each trial.",
@@ -232,7 +251,7 @@ def session_to_nwb(
     )
 
     # Path to the output NWB file
-    nwbfile_path = Path("/Volumes/T9/Constantinople/nwbfiles/C005_RWTautowait_20190909_145629.nwb")
+    nwbfile_path = Path("/Users/weian/data/demo/C005_RWTautowait_20190909_145629.nwb")
 
     # Whether to overwrite the NWB file if it already exists
     overwrite = True

src/constantinople_lab_to_nwb/mah_2024/metadata/mah_2024_behavior_metadata.yaml

@@ -1,29 +1,34 @@
 Behavior:
+  TimeIntervals:
+    name: processed_trials
+    description: Contains the processed trials.
+  # The metadata for the raw Bpod trials.
   TrialsTable:
     description: |
-      LED illumination from the center port indicated that the animal could initiate a trial by poking its nose in that 
+      Contains the raw Bpod trials.
+      LED illumination from the center port indicated that the animal could initiate a trial by poking its nose in that
       port - upon trial initiation the center LED turned off. While in the center port, rats needed to maintain center
-      fixation for a duration drawn uniformly from [0.8, 1.2] seconds. During the fixation period, a tone played from 
-      both speakers, the frequency of which indicated the volume of the offered water reward for that trial 
-      [1, 2, 4, 8, 16kHz, indicating 5, 10, 20, 40, 80μL rewards]. Following the fixation period, one of the two side 
-      LEDs was illuminated, indicating that the reward might be delivered at that port; the side was randomly chosen on 
-      each trial.This event (side LED ON) also initiated a variable and unpredictable delay period, which was randomly 
-      drawn from an exponential distribution with mean=2.5s. The reward port LED remained illuminated for the duration 
-      of the delay period, and rats were not required to maintain fixation during this period, although they tended to 
-      fixate in the reward port. When reward was available, the reward port LED turned off, and rats could collect the 
+      fixation for a duration drawn uniformly from [0.8, 1.2] seconds. During the fixation period, a tone played from
+      both speakers, the frequency of which indicated the volume of the offered water reward for that trial
+      [1, 2, 4, 8, 16kHz, indicating 5, 10, 20, 40, 80μL rewards]. Following the fixation period, one of the two side
+      LEDs was illuminated, indicating that the reward might be delivered at that port; the side was randomly chosen on
+      each trial.This event (side LED ON) also initiated a variable and unpredictable delay period, which was randomly
+      drawn from an exponential distribution with mean=2.5s. The reward port LED remained illuminated for the duration
+      of the delay period, and rats were not required to maintain fixation during this period, although they tended to
+      fixate in the reward port. When reward was available, the reward port LED turned off, and rats could collect the
       offered reward by nose poking in that port. The rat could also choose to terminate the trial (opt-out) at any time
-      by nose poking in the opposite, un-illuminated side port, after which a new trial would immediately begin. On a 
-      proportion of trials (15–25%), the delay period would only end if the rat opted out (catch trials). If rats did 
-      not opt-out within 100s on catch trials, the trial would terminate. The trials were self-paced: after receiving 
-      their reward or opting out, rats were free to initiate another trial immediately. However, if rats terminated 
-      center fixation prematurely, they were penalized with a white noise sound and a time out penalty (typically 2s, 
-      although adjusted to individual animals). Following premature fixation breaks, the rats received the same offered 
-      reward, in order to disincentivize premature terminations for small volume offers. We introduced semi-observable, 
+      by nose poking in the opposite, un-illuminated side port, after which a new trial would immediately begin. On a
+      proportion of trials (15–25%), the delay period would only end if the rat opted out (catch trials). If rats did
+      not opt-out within 100s on catch trials, the trial would terminate. The trials were self-paced: after receiving
+      their reward or opting out, rats were free to initiate another trial immediately. However, if rats terminated
+      center fixation prematurely, they were penalized with a white noise sound and a time out penalty (typically 2s,
+      although adjusted to individual animals). Following premature fixation breaks, the rats received the same offered
+      reward, in order to disincentivize premature terminations for small volume offers. We introduced semi-observable,
       hidden states in the task by including uncued blocks of trials with varying reward statistics: high and low blocks
-      , which offered the highest three or lowest three rewards, respectively, and were interspersed with mixed blocks, 
-      which offered all volumes. There was a hierarchical structure to the blocks, such that high and low blocks 
-      alternated after mixed blocks (e.g., mixed-high-mixed-low, or mixed-low-mixed-high). The first block of each 
-      session was a mixed block. Blocks transitioned after 40 successfully completed trials. Because rats prematurely 
+      , which offered the highest three or lowest three rewards, respectively, and were interspersed with mixed blocks,
+      which offered all volumes. There was a hierarchical structure to the blocks, such that high and low blocks
+      alternated after mixed blocks (e.g., mixed-high-mixed-low, or mixed-low-mixed-high). The first block of each
+      session was a mixed block. Blocks transitioned after 40 successfully completed trials. Because rats prematurely
       broke fixation on a subset of trials, in practice, block durations were variable.
   StateTypesTable:
     description: Contains the name of the states in the task.