Merge branch 'calib_prod_id' of github.com:cta-observatory/lstosa int…

…o calib_prod_id
cta-observatory · Sep 21, 2023 · e0d4d75 · e0d4d75
2 parents eae4d92 + 6d05f68
commit e0d4d75
Show file tree

Hide file tree

Showing 13 changed files with 286 additions and 41 deletions.
diff --git a/osa/configs/sequencer.cfg b/osa/configs/sequencer.cfg
@@ -20,8 +20,6 @@ CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
 PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
-MUON_DIR: %(BASE)s/DL1
-DATACHECK_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
 OSA_DIR: %(BASE)s/OSA
@@ -110,6 +108,7 @@ DL1ABPREFIX: dl1_LST-1
 DATACHECKPREFIX: datacheck_dl1_LST-1
 MUONPREFIX: muons_LST-1
 DL2PREFIX: dl2_LST-1
+INTERLEAVEDPREFIX: interleaved_LST-1
 
 # Suffixes
 R0SUFFIX: .fits.fz
@@ -121,6 +120,7 @@ DL1ABSUFFIX: .h5
 DATACHECKSUFFIX: .h5
 MUONSUFFIX: .fits
 DL2SUFFIX: .h5
+INTERLEAVEDSUFFIX: .h5
 
 # File type pattern for DB
 CALIBTYPE = Calibration
@@ -131,6 +131,7 @@ DL1ABTYPE = DL1
 DL2TYPE = DL2
 MUONTYPE = Muons
 DATACHECKTYPE = Datacheck
+INTERLEAVEDTYPE = Interleaved
 
 [LSTOSA]
 GRAPH: dot

diff --git a/osa/conftest.py b/osa/conftest.py
@@ -181,15 +181,17 @@ def test_observed_data(running_analysis_dir, dl1b_subdir, dl2_subdir):
     """Mock observed data files for testing."""
     dl1_file = running_analysis_dir / "dl1_LST-1.Run01808.0011.h5"
     muons_file = running_analysis_dir / "muons_LST-1.Run01808.0011.fits"
+    interleaved_file = running_analysis_dir / "interleaved_LST-1.Run01808.0011.h5"
     dl1ab_file = dl1b_subdir / "dl1_LST-1.Run01808.0011.h5"
     datacheck_file = dl1b_subdir / "datacheck_dl1_LST-1.Run01808.0011.h5"
     dl2_file = dl2_subdir / "dl2_LST-1.Run01808.0011.h5"
     dl1_file.touch()
     muons_file.touch()
+    interleaved_file.touch()
     dl1ab_file.touch()
     datacheck_file.touch()
     dl2_file.touch()
-    return dl1_file, dl1ab_file, dl2_file, muons_file, datacheck_file
+    return dl1_file, dl1ab_file, dl2_file, muons_file, datacheck_file, interleaved_file
 
 
 @pytest.fixture(scope="session")

diff --git a/osa/nightsummary/database.py b/osa/nightsummary/database.py
@@ -27,9 +27,10 @@ def db_available():
         caco_client.server_info()
         tcu_client.server_info()
     except ConnectionFailure:
-        log.info("TCU or CaCo database not available. No source info will be added.")
+        log.warning("TCU or CaCo database not available. No source info will be added.")
         return False
     else:
+        log.debug("TCU and CaCo database are available. Source info will be added.")
         return True
 
 

diff --git a/osa/nightsummary/extract.py b/osa/nightsummary/extract.py
@@ -170,23 +170,31 @@ def extract_runs(summary_table):
             dtype=["int32", str, "float64", "float64"],
         )
         for run in run_list:
-            run.source_name = database.query(
-                obs_id=run.run, property_name="DriveControl_SourceName"
-            )
-            run.source_ra = database.query(obs_id=run.run, property_name="DriveControl_RA_Target")
-            run.source_dec = database.query(obs_id=run.run, property_name="DriveControl_Dec_Target")
-            # Store this source information (run_id, source_name, source_ra, source_dec)
-            # into an astropy Table and save to disk. In this way, the information can be
-            # dumped anytime later more easily than accessing the TCU database itself.
-            if run.source_name is not None:
-                line = [
-                    run.run,
-                    run.source_name,
-                    run.source_ra,
-                    run.source_dec,
-                ]
-                log.debug(f"Adding line with source info to RunCatalog: {line}")
-                run_table.add_row(line)
+            # Make sure we are looking at actual data runs. Avoid test runs.
+            if run.run > 0 and run.type == "DATA":
+                log.debug(f"Looking info in TCU DB for run {run.run}")
+                run.source_name = database.query(
+                    obs_id=run.run, property_name="DriveControl_SourceName"
+                )
+                run.source_ra = database.query(
+                    obs_id=run.run, property_name="DriveControl_RA_Target"
+                    )
+                run.source_dec = database.query(
+                    obs_id=run.run, property_name="DriveControl_Dec_Target"
+                    )
+                # Store this source information (run_id, source_name, source_ra, source_dec)
+                # into an astropy Table and save to disk in RunCatalog files. In this way, the
+                # information can be dumped anytime later more easily than accessing the
+                # TCU database.
+                if run.source_name is not None:
+                    line = [
+                        run.run,
+                        run.source_name,
+                        run.source_ra,
+                        run.source_dec,
+                    ]
+                    log.debug(f"Adding line with source info to RunCatalog: {line}")
+                    run_table.add_row(line)
 
         # Save table to disk
         run_table.write(source_catalog_file, overwrite=True, delimiter=",")

diff --git a/osa/paths.py b/osa/paths.py
@@ -277,7 +277,7 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
     Parameters
     ----------
     concept : str
-        Expected: MUON, DL1AB, DATACHECK, DL2, PEDESTAL, CALIB, TIMECALIB
+        Expected: MUON, DL1AB, DATACHECK, INTERLEAVED, DL2, PEDESTAL, CALIB, TIMECALIB
     create_dir : bool
         Set it to True (default) if you want to create the directory.
         Otherwise, it just returns the path
@@ -305,7 +305,22 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
         )
     elif concept == "DL1AB":
         directory = (
-            Path(cfg.get(options.tel_id, concept + "_DIR"))
+            Path(cfg.get(options.tel_id, "DL1_DIR"))
+            / nightdir
+            / options.prod_id
+            / "interleaved"
+        )
+    elif concept == "DATACHECK":
+        directory = (
+            Path(cfg.get(options.tel_id, "DL1_DIR"))
+            / nightdir
+            / options.prod_id
+            / options.dl1_prod_id
+            / "datacheck"
+        )
+    elif concept == "DL1AB":
+        directory = (
+            Path(cfg.get(options.tel_id, "DL1_DIR"))
             / nightdir
             / options.prod_id
             / options.dl1_prod_id

diff --git a/osa/scripts/closer.py b/osa/scripts/closer.py
@@ -196,13 +196,15 @@ def post_process_files(seq_list: list):
     DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
     MUONS_RE = re.compile(r"muons.*.fits")
     DATACHECK_RE = re.compile(r"datacheck_dl1.*.(?:h5|hdf5|hdf)")
+    INTERLEAVED_RE = re.compile(r"interleaved.*.(?:h5|hdf5|hdf)")
 
     pattern_files = dict(
         [
             ("DL1AB", DL1AB_RE),
             ("DL2", DL2_RE),
             ("MUON", MUONS_RE),
             ("DATACHECK", DATACHECK_RE),
+            ("INTERLEAVED", INTERLEAVED_RE),
         ]
     )
 
@@ -295,7 +297,7 @@ def merge_dl1_datacheck(seq_list) -> List[str]:
     log.debug("Merging dl1 datacheck files and producing PDFs")
 
     muons_dir = destination_dir("MUON", create_dir=False)
-    dl1_dir = destination_dir("DL1AB", create_dir=False)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
 
     list_job_id = []
 
@@ -312,8 +314,8 @@ def merge_dl1_datacheck(seq_list) -> List[str]:
                 f"log/merge_dl1_datacheck_{sequence.run:05d}_%j.err",
                 "lstchain_check_dl1",
                 "--input-file",
-                f"{dl1_dir}/datacheck_dl1_LST-1.Run{sequence.run:05d}.*.h5",
-                f"--output-dir={dl1_dir}",
+                f"{datacheck_dir}/datacheck_dl1_LST-1.Run{sequence.run:05d}.*.h5",
+                f"--output-dir={datacheck_dir}",
                 f"--muons-dir={muons_dir}",
             ]
             if not options.simulate and not options.test:
@@ -452,7 +454,7 @@ def merge_muon_files(sequence_list):
 def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     """Build the daily longterm command."""
     nightdir = date_to_dir(options.date)
-    dl1_dir = destination_dir("DL1AB", create_dir=False)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
@@ -465,7 +467,7 @@ def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
         "log/longterm_daily_%j.log",
         f"--dependency=afterok:{','.join(parent_job_ids)}",
         "lstchain_longterm_dl1_check",
-        f"--input-dir={dl1_dir}",
+        f"--input-dir={datacheck_dir}",
         f"--output-file={longterm_output_file}",
         f"--muons-dir={muons_dir}",
         "--batch",

diff --git a/osa/scripts/reprocess_longterm.py b/osa/scripts/reprocess_longterm.py
@@ -30,8 +30,8 @@ def run_longterm(date: str, prod_id: str, new_prod_id: str, log_dir: Path):
     log_dir : Path
         Path to the directory where job logs will be stored.
     """
-    dl1_dir = ANALYSIS_PATH / "DL1" / date / prod_id / "tailcut84"
-    muons_dir = ANALYSIS_PATH / "DL1" / date / prod_id
+    dl1_dir = ANALYSIS_PATH / "DL1" / date / prod_id / "tailcut84" / "datacheck"
+    muons_dir = ANALYSIS_PATH / "DL1" / date / prod_id / "muons"
     new_longterm_dir = LONGTERM_PATH / new_prod_id / date
     longterm_output_file = new_longterm_dir / f"DL1_datacheck_{date}.h5"
 

diff --git a/osa/scripts/tests/test_osa_scripts.py b/osa/scripts/tests/test_osa_scripts.py
@@ -201,24 +201,27 @@ def test_closer(
 
     # Check that files have been moved to their final destinations
     assert os.path.exists(
-        "./test_osa/test_files0/DL1/20200117/v0.1.0/muons_LST-1.Run01808.0011.fits"
+       "./test_osa/test_files0/DL1/20200117/v0.1.0/muons/muons_LST-1.Run01808.0011.fits"
+    )
+    assert os.path.exists(
+       "./test_osa/test_files0/DL1/20200117/v0.1.0/interleaved/interleaved_LST-1.Run01808.0011.h5"
     )
     assert os.path.exists(
         "./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/dl1_LST-1.Run01808.0011.h5"
     )
     assert os.path.exists(
-        "./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/"
+        "./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/datacheck/"
         "datacheck_dl1_LST-1.Run01808.0011.h5"
     )
     assert os.path.exists(
-        "./test_osa/test_files0/DL2/20200117/v0.1.0/model2/" "dl2_LST-1.Run01808.0011.h5"
+        "./test_osa/test_files0/DL2/20200117/v0.1.0/model2/dl2_LST-1.Run01808.0011.h5"
     )
     # Assert that the link to dl1 and muons files have been created
     assert os.path.islink(
-        "./test_osa/test_files0/running_analysis/20200117/" "v0.1.0/muons_LST-1.Run01808.0011.fits"
+        "./test_osa/test_files0/running_analysis/20200117/v0.1.0/muons_LST-1.Run01808.0011.fits"
     )
     assert os.path.islink(
-        "./test_osa/test_files0/running_analysis/20200117/" "v0.1.0/dl1_LST-1.Run01808.0011.h5"
+        "./test_osa/test_files0/running_analysis/20200117/v0.1.0/dl1_LST-1.Run01808.0011.h5"
     )
 
     assert night_finished_flag.exists()
@@ -319,9 +322,9 @@ def test_daily_longterm_cmd():
         "log/longterm_daily_%j.log",
         "--dependency=afterok:12345,54321",
         "lstchain_longterm_dl1_check",
-        "--input-dir=test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84",
+        "--input-dir=test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/datacheck",
         "--output-file=test_osa/test_files0/OSA/DL1DataCheck_LongTerm/v0.1.0/20200117/DL1_datacheck_20200117.h5",
-        "--muons-dir=test_osa/test_files0/DL1/20200117/v0.1.0",
+        "--muons-dir=test_osa/test_files0/DL1/20200117/v0.1.0/muons",
         "--batch",
     ]