Fixed ws call to only return gpubox or mwax fits files

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = mwax_mover
-version = 0.17.21
+version = 0.17.22
 author = Greg Sleap, Dev Null
 author_email = [email protected], [email protected]
 description = A module to manage input and archiving for the MWAX correlator

src/mwax_mover/mwax_calvin_processor.py

@@ -302,18 +302,11 @@ def check_obs_is_ready_to_process(self, obs_id, obsid_assembly_dir) -> bool:
             #
             # perform web service call to get list of data files from obsid
             #
-            json_metadata = utils.get_data_files_for_obsid_from_webservice(
+            web_service_filenames = utils.get_data_files_for_obsid_from_webservice(
                 self.logger, obs_id, self.metadata_webservice_url
             )
 
-            if json_metadata:
-                #
-                # we need a list of files from the web service
-                # this should just be the filenames
-                #
-                web_service_filenames = [filename for filename in json_metadata]
-                web_service_filenames.sort()
-
+            if web_service_filenames:
                 # we need a list of files from the work dir
                 # this first list has the full path
                 # put a basic UNIX pattern so we don't pick up the metafits

src/mwax_mover/utils.py

@@ -72,6 +72,7 @@ def is_voltage_buffer(mode_string: str) -> bool:
 class MWADataFileType(Enum):
     """Enum for the possible MWA data file types"""
 
+    HW_LFILES = 8
     MWA_FLAG_FILE = 10
     MWA_PPD_FILE = 14
     MWAX_VOLTAGES = 17
@@ -880,16 +881,29 @@ def should_project_be_archived(project_id: str) -> bool:
         return True
 
 
-def get_data_files_for_obsid_from_webservice(logger, obsid: int, metadata_webservice_url: str):
-    """Calls an MWA webservice, passing in an obsid and returning a json metadata structure
-    of all the data_files - all_files: True means to get all files whether they are
-    archived at Pawsey or not"""
+def get_data_files_for_obsid_from_webservice(
+    logger,
+    obsid: int,
+    metadata_webservice_url: str,
+) -> list[str]:
+    """Calls an MWA webservice, passing in an obsid and returning a list of filenames
+    of all the data_files (MWAX_VISIBILITIES or HW_LFILES) of the given filetype or None if there was an error.
+    metadata_webservice_url is the base url - e.g. http://ws.mwatelescope.org
+    - all_files: True means to get all files whether they are archived at Pawsey or not"""
     result = requests.get(
         f"{metadata_webservice_url}/metadata/data_files",
         data={"obs_id": obsid, "terse": False, "all_files": True},
     )
     if result.text:
-        return json.loads(result.text)
+        files = json.loads(result.text)
+        file_list = [
+            file
+            for file in files
+            if files[file]["filetype"] == MWADataFileType.MWAX_VISIBILITIES.value
+            or files[file]["filetype"] == MWADataFileType.HW_LFILES.value
+        ]
+        file_list.sort()
+        return file_list
     else:
         logger.error(f"Error getting data files for obsid {obsid}, status code {result.status_code}")
         return None

tests/utils_test.py

@@ -727,3 +727,41 @@ def test_should_project_be_archived():
     assert utils.should_project_be_archived("c001") is True
     assert utils.should_project_be_archived("C123") is False
     assert utils.should_project_be_archived("c123") is False
+
+
+def test_get_data_files_for_obsid_from_webservice():
+    logger = logging.getLogger("test")
+
+    # Uknown obsid- returns None
+    assert utils.get_data_files_for_obsid_from_webservice(logger, 1234567890, "http://ws.mwatelescope.org") is None
+
+    # Good obsid with 24 gpubox files and 1 flags and 1 metafits. Only return the 24 gpubox files
+    file_list = utils.get_data_files_for_obsid_from_webservice(logger, 1157306584, "http://ws.mwatelescope.org")
+    assert len(file_list) == 24
+
+    assert file_list == [
+        "1157306584_20160907180249_gpubox01_00.fits",
+        "1157306584_20160907180249_gpubox02_00.fits",
+        "1157306584_20160907180249_gpubox03_00.fits",
+        "1157306584_20160907180249_gpubox04_00.fits",
+        "1157306584_20160907180249_gpubox05_00.fits",
+        "1157306584_20160907180249_gpubox06_00.fits",
+        "1157306584_20160907180249_gpubox07_00.fits",
+        "1157306584_20160907180249_gpubox08_00.fits",
+        "1157306584_20160907180249_gpubox09_00.fits",
+        "1157306584_20160907180249_gpubox10_00.fits",
+        "1157306584_20160907180249_gpubox11_00.fits",
+        "1157306584_20160907180249_gpubox12_00.fits",
+        "1157306584_20160907180249_gpubox13_00.fits",
+        "1157306584_20160907180249_gpubox14_00.fits",
+        "1157306584_20160907180249_gpubox15_00.fits",
+        "1157306584_20160907180249_gpubox16_00.fits",
+        "1157306584_20160907180249_gpubox17_00.fits",
+        "1157306584_20160907180249_gpubox18_00.fits",
+        "1157306584_20160907180249_gpubox19_00.fits",
+        "1157306584_20160907180249_gpubox20_00.fits",
+        "1157306584_20160907180249_gpubox21_00.fits",
+        "1157306584_20160907180249_gpubox22_00.fits",
+        "1157306584_20160907180249_gpubox23_00.fits",
+        "1157306584_20160907180249_gpubox24_00.fits",
+    ]