Add logging and usage of pattern for glob

calcium_imaging_automation/core/reader.py

@@ -4,16 +4,19 @@
 
 class ReadAllPathsInFolder:
     def __init__(
-        self, raw_data_folder: Path, filetypes_of_interest: List[str]
+        self,
+        raw_data_folder: Path,
+        filetypes_of_interest: List[str],
+        folder_read_pattern: str,
     ):
         self.filetypes_of_interest = filetypes_of_interest
+        self.folder_read_pattern = folder_read_pattern
+
         self.datasets_paths = self.get_folders_first_layer(raw_data_folder)
-        self.dataset_names = [
-            dataset_path.name for dataset_path in self.datasets_paths
-        ]
+        self.dataset_names = [dataset_path.name for dataset_path in self.datasets_paths]
 
     def get_folders_first_layer(self, file_path: Path) -> List[Path]:
-        return list(file_path.glob("*"))
+        return list(file_path.glob(self.folder_read_pattern))
 
     def get_files_paths(self, folder: Path) -> List[Path]:
         return [

examples/example_usage.py

@@ -1,45 +1,67 @@
 import argparse
 from pathlib import Path
+import logging
 
 from calcium_imaging_automation.core.reader import ReadAllPathsInFolder
 from calcium_imaging_automation.core.writer import DatashuttleWrapper
 
 
-def main(raw_data_path: Path, output_path: Path, filetypes_of_interest: list):
+def main(
+    raw_data_path: Path,
+    output_path: Path,
+    filetypes_of_interest: list,
+    folder_read_pattern: str,
+):
     """
     Draft usage of the pipeline, now consisting of read and write operations.
     """
-    reader = ReadAllPathsInFolder(raw_data_path, filetypes_of_interest)
+    logging.basicConfig(
+        filename=output_path / "logs" / "pipeline.log",
+        level=logging.INFO,
+        format="%(asctime)s - %(message)s",
+    ) 
+
+    reader = ReadAllPathsInFolder(
+        raw_data_path, filetypes_of_interest, folder_read_pattern
+    )
+    logging.info(f"Found {len(reader.datasets_paths)} datasets.")
+    logging.info(f"Dataset names: {reader.dataset_names}")
 
     writer = DatashuttleWrapper(output_path)
+
     number_of_tiffs = reader.max_session_number(filetype="tif")
     writer.create_folders(reader.dataset_names, session_number=number_of_tiffs)
 
     # [Placeholder for data processing]
 
+    logging.info("Pipeline finished.")
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Example usage of the pipeline manager."
     )
 
-    parser.add_argument(
-        "raw_data_path", type=Path, help="Path to the raw data."
-    )
-    parser.add_argument(
-        "output_path", type=Path, help="Path to the output data."
-    )
+    parser.add_argument("raw_data_path", type=Path, help="Path to the raw data.")
+    parser.add_argument("output_path", type=Path, help="Path to the output data.")
     parser.add_argument(
         "--filetypes",
         type=list,
         nargs="+",
         help="Filetypes of interest.",
         default=["tif", "bin"],
     )
+    parser.add_argument(
+        "--folder_read_pattern",
+        type=str,
+        help="Glob pattern for reading files.",
+        default="*",
+    )
 
     args = parser.parse_args()
     raw_data_path = args.raw_data_path
     output_path = args.output_path
     file_types = args.filetypes
+    folder_read_pattern = args.folder_read_pattern
 
-    main(raw_data_path, output_path, file_types)
+    main(raw_data_path, output_path, file_types, folder_read_pattern)

examples/example_usage.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
 python ./examples/example_usage.py \
-    /Users/lauraporta/local_data/rotation/ \
-    /Users/lauraporta/local_data/test/
+    /Volumes/winstor/swc/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/ \
+    /Users/laura/local_data/calcimaut/ \
+    --folder_read_pattern '2*' \