From 89c046a2991322f289c9e4d4ea6de13be955a427 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Wed, 24 Jul 2024 20:35:51 +0530 Subject: [PATCH 1/2] observation mode added --- sunpy_soar/attrs.py | 11 +++++++++++ sunpy_soar/client.py | 15 +++++++++++---- sunpy_soar/tests/test_sunpy_soar.py | 20 ++++++++++++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/sunpy_soar/attrs.py b/sunpy_soar/attrs.py index 32c463e..9cf7713 100644 --- a/sunpy_soar/attrs.py +++ b/sunpy_soar/attrs.py @@ -25,6 +25,12 @@ class SOOP(SimpleAttr): """ +class ObservationMode(SimpleAttr): + """ + The mode of observation to search for. + """ + + walker = AttrWalker() @@ -141,3 +147,8 @@ def _(wlk, attr, params): # NOQA: ARG001 wavemin = attr.min.value wavemax = attr.max.value params.append(f"Wavemin='{wavemin}'+AND+Wavemax='{wavemax}'") + + +@walker.add_applier(ObservationMode) +def _(wlk, attr, params): # NOQA: ARG001 + params.append(f"Observation_mode='{attr.value}'") diff --git a/sunpy_soar/client.py b/sunpy_soar/client.py index 6ed600a..2117c96 100644 --- a/sunpy_soar/client.py +++ b/sunpy_soar/client.py @@ -11,7 +11,7 @@ from sunpy.net.base_client import BaseClient, QueryResponseTable from sunpy.time import parse_time -from sunpy_soar.attrs import SOOP, Product, walker +from sunpy_soar.attrs import SOOP, ObservationMode, Product, walker __all__ = ["SOARClient"] @@ -70,7 +70,13 @@ def add_join_to_query(query: list[str], data_table: str, instrument_table: str): parameter = f"Wavelength='{wavemin_match.group(1)}'" elif wavemin_match and wavemax_match: parameter = f"Wavemin='{wavemin_match.group(1)}'+AND+h2.Wavemax='{wavemax_match.group(1)}'" - prefix = "h1." if not parameter.startswith("Detector") and not parameter.startswith("Wave") else "h2." + prefix = ( + "h1." + if not parameter.startswith("Detector") + and not parameter.startswith("Wave") + and not parameter.startswith("Observation") + else "h2." + ) if parameter.startswith("begin_time"): time_list = parameter.split("+AND+") final_query += f"h1.{time_list[0]}+AND+h1.{time_list[1]}+AND+" @@ -89,7 +95,7 @@ def add_join_to_query(query: list[str], data_table: str, instrument_table: str): ) if instrument_table: from_part += f" JOIN {instrument_table} AS h2 USING (data_item_oid)" - select_part += ", h2.detector, h2.wavelength, h2.dimension_index" + select_part += ", h2.detector, h2.wavelength, h2.dimension_index, h2.observation_mode" return where_part, from_part, select_part @staticmethod @@ -198,6 +204,7 @@ def _do_search(query): ) if "detector" in info: result_table["Detector"] = info["detector"] + result_table["Observation mode"] = info["observation_mode"] if "wavelength" in info: result_table["Wavelength"] = info["wavelength"] result_table.sort("Start time") @@ -247,7 +254,7 @@ def _can_handle_query(cls, *query): True if this client can handle the given query. """ required = {a.Time} - optional = {a.Instrument, a.Detector, a.Wavelength, a.Level, a.Provider, Product, SOOP} + optional = {a.Instrument, a.Detector, a.Wavelength, a.Level, a.Provider, Product, SOOP, ObservationMode} if not cls.check_attr_types_in_query(query, required, optional): return False # check to make sure the instrument attr passed is one provided by the SOAR. diff --git a/sunpy_soar/tests/test_sunpy_soar.py b/sunpy_soar/tests/test_sunpy_soar.py index 4a3a301..9d0906e 100644 --- a/sunpy_soar/tests/test_sunpy_soar.py +++ b/sunpy_soar/tests/test_sunpy_soar.py @@ -232,6 +232,18 @@ def test_wavelength_range(): assert all(table["Wavelength"] == 174) +def test_search_observation_mode(): + instrument = a.Instrument("METIS") + time = a.Time("2022-03-01", "2022-03-02") + level = a.Level(2) + product = a.soar.Product("metis-vl-pol-angle") + observation = a.soar.ObservationMode("METIS_GENERIC") + res = Fido.search(instrument & time & level & product & observation) + assert len(res[0]) == 32 + for table in res: + assert all(table["Observation mode"] == "METIS_GENERIC") + + def test_join_science_query(): result = SOARClient._construct_payload( # NOQA: SLF001 [ @@ -244,8 +256,8 @@ def test_join_science_query(): assert result["QUERY"] == ( "SELECT+h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, " - "h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, " - "h2.dimension_index+FROM+v_sc_data_item AS h1 JOIN v_eui_sc_fits AS h2 USING (data_item_oid)" + "h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, h2.dimension_index, " + "h2.observation_mode+FROM+v_sc_data_item AS h1 JOIN v_eui_sc_fits AS h2 USING (data_item_oid)" "+WHERE+h1.instrument='EUI'+AND+h1.begin_time>='2021-02-01+00:00:00'+AND+h1.begin_time<='2021-02-02+00:00:00'" "+AND+h2.dimension_index='1'+AND+h1.level='L1'+AND+h1.descriptor='eui-fsi174-image'" ) @@ -263,8 +275,8 @@ def test_join_low_latency_query(): assert result["QUERY"] == ( "SELECT+h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, " - "h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, " - "h2.dimension_index+FROM+v_ll_data_item AS h1 JOIN v_eui_ll_fits AS h2 USING (data_item_oid)" + "h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, h2.dimension_index, " + "h2.observation_mode+FROM+v_ll_data_item AS h1 JOIN v_eui_ll_fits AS h2 USING (data_item_oid)" "+WHERE+h1.instrument='EUI'+AND+h1.begin_time>='2021-02-01+00:00:00'+AND+h1.begin_time<='2021-02-02+00:00:00'" "+AND+h2.dimension_index='1'+AND+h1.level='LL01'+AND+h1.descriptor='eui-fsi174-image'" ) From 9ce5a55d47264c4eddc3d192b7b0f504bd409634 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Mon, 5 Aug 2024 20:24:16 +0530 Subject: [PATCH 2/2] observation_attrs file added and attrs updated --- sunpy_soar/client.py | 8 + sunpy_soar/data/attrs.json | 39 ++- sunpy_soar/data/observation_attrs.json | 324 +++++++++++++++++++++++++ tools/update_data.py | 36 ++- 4 files changed, 396 insertions(+), 11 deletions(-) create mode 100644 sunpy_soar/data/observation_attrs.json diff --git a/sunpy_soar/client.py b/sunpy_soar/client.py index b67ed20..70673b6 100644 --- a/sunpy_soar/client.py +++ b/sunpy_soar/client.py @@ -297,15 +297,23 @@ def load_dataset_values(): all_instr = json.load(instr_attrs_file) all_instr = list(all_instr.items()) + # SOOP attrs soop_path = pathlib.Path(__file__).parent / "data" / "soop_attrs.json" with soop_path.open() as soop_path_file: all_soops = json.load(soop_path_file) all_soops = list(all_soops.items()) + # Observation modes + obs_modes_path = pathlib.Path(__file__).parent / "data" / "observation_attrs.json" + with obs_modes_path.open() as obs_modes_file: + observation_modes = json.load(obs_modes_file) + observation_modes = list(observation_modes.items()) + return { Product: all_datasets, a.Instrument: all_instr, SOOP: all_soops, a.Provider: [("SOAR", "Solar Orbiter Archive.")], + ObservationMode: observation_modes, } diff --git a/sunpy_soar/data/attrs.json b/sunpy_soar/data/attrs.json index d612b19..2a19795 100644 --- a/sunpy_soar/data/attrs.json +++ b/sunpy_soar/data/attrs.json @@ -2,6 +2,8 @@ "RPW-SBM1": "Solar Orbiter Radio/Plasma Wave, LL01 parameters", "RPW-SBM2": "Solar Orbiter Radio/Plasma Wave, LL01 parameters", "RPW-TNR": "Solar Orbiter Radio/Plasma Wave, LL01 parameters", + "SOC-ORBIT": "Solar Orbiter SOC Ancillary Orbit Data", + "epd-ept-1min": "Solar Orbiter, Level 3 Data, Energetic Particle Detector, Electron Proton Telescope, 1 minute resolution data", "epd-ept-asun-burst-ele-close": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Electron Proton Telescope, Anti-Sun direction, Burst, Electrons, Close mode", "epd-ept-asun-burst-ion": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Electron Proton Telescope, Anti-Sun direction, Burst, Ions", "epd-ept-asun-hcad": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Electron Proton Telescope, Anti-Sun direction, High Cadence", @@ -43,13 +45,13 @@ "epd-sis-a-hehist": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, A Telescope, Helium histogram", "epd-sis-a-rates": "Solar Orbiter, Level 1 Low Latency Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, A Telescope, Particle Rates", "epd-sis-a-rates-fast": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, A Telescope, Particle rates, fast cadence", - "epd-sis-a-rates-medium": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, A Telescope, Particle rates, medium cadence", + "epd-sis-a-rates-medium": "Solar Orbiter, Level 1 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, A Telescope, Particle rates, medium cadence", "epd-sis-a-rates-slow": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, A Telescope, Particle rates, slow cadence", "epd-sis-b-hehist": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Helium histogram", "epd-sis-b-rates": "Solar Orbiter, Level 1 Low Latency Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle Rates", - "epd-sis-b-rates-fast": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle rates, fast cadence", - "epd-sis-b-rates-medium": "Solar Orbiter, Level 1 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle rates, medium cadence", - "epd-sis-b-rates-slow": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle rates, slow cadence", + "epd-sis-b-rates-fast": "Solar Orbiter, Level 1 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle rates, fast cadence", + "epd-sis-b-rates-medium": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle rates, medium cadence", + "epd-sis-b-rates-slow": "Solar Orbiter, Level 1 Data, Energetic Particle Detector, Suprathermal Ion Spectrograph, B Telescope, Particle rates, slow cadence", "epd-step-aux": "Solar Orbiter, Level 1 Data, Energetic Particle Detector, SupraThermal Electrons and Protons, Auxiliary product", "epd-step-burst": "Solar Orbiter, Level 2 Data, Energetic Particle Detector, SupraThermal Electrons and Protons, Burst", "epd-step-burst1-close": "Solar Orbiter, Level 1 Data, Energetic Particle Detector, SupraThermal Electrons and Protons, Burst product 1 close mode", @@ -112,6 +114,13 @@ "metis-vl-pol-angle": "", "metis-vl-stokes": "", "metis-vl-tb": "", + "phi-fdt-bazi": "", + "phi-fdt-binc": "", + "phi-fdt-blos": "", + "phi-fdt-bmag": "", + "phi-fdt-icnt": "", + "phi-fdt-stokes": "", + "phi-fdt-vlos": "", "phi-hrt-bazi": "", "phi-hrt-binc": "", "phi-hrt-blos": "", @@ -119,7 +128,7 @@ "phi-hrt-icnt": "", "phi-hrt-stokes": "", "phi-hrt-vlos": "", - "rates": "Solar Orbiter Level 2 Solar Wind Analyser Heavy Ion Sensor Rates", + "rates": "Solar Orbiter Level 1 Solar Wind Analyser Heavy Ion Sensor Rates", "rpw-bia-density": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential", "rpw-bia-density-10-seconds": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential, downsampled", "rpw-bia-density-10-seconds-cdag": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential, downsampled", @@ -132,16 +141,17 @@ "rpw-bia-scpot-10-seconds": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential, downsampled", "rpw-bia-scpot-10-seconds-cdag": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential, downsampled", "rpw-bia-scpot-cdag": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential", + "rpw-bia-vht": "Solar Orbiter Radio/Plasma Wave, LFR L3 de Hoffmann-Teller solar wind velocity", "rpw-hfr-surv": "Solar Orbiter Radio/Plasma Wave, HFR L2 parameters", "rpw-lfr-surv-asm": "Solar Orbiter Radio/Plasma Wave, LFR L2 parameters", "rpw-lfr-surv-bp1": "Solar Orbiter Radio/Plasma Wave, LFR L2 parameters", "rpw-lfr-surv-bp2": "Solar Orbiter Radio/Plasma Wave, LFR L2 parameters", "rpw-lfr-surv-cwf": "Solar Orbiter Radio/Plasma Wave, LFR L1 parameters", - "rpw-lfr-surv-cwf-b": "Solar Orbiter Radio/Plasma Wave, LFR L2 magnetic parameters", + "rpw-lfr-surv-cwf-b": "Solar Orbiter, Level 2, Radio and Plasma Waves, Low Frequency Receiver, Continous Waveform of magnetic data in survey mode", "rpw-lfr-surv-cwf-e": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters", "rpw-lfr-surv-cwf-e-cdag": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters", "rpw-lfr-surv-swf": "Solar Orbiter Radio/Plasma Wave, LFR L1 parameters", - "rpw-lfr-surv-swf-b": "Solar Orbiter Radio/Plasma Wave, LFR L2 magnetic parameters", + "rpw-lfr-surv-swf-b": "Solar Orbiter, Level 2, Radio and Plasma Waves, Low Frequency Receiver, Snapshot Waveform of magnetic data in survey mode", "rpw-lfr-surv-swf-e": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters", "rpw-lfr-surv-swf-e-cdag": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters", "rpw-sbm1": "Solar Orbiter Radio/Plasma Wave, LL02 parameters", @@ -150,26 +160,37 @@ "rpw-tds-surv-hist2d": "Solar Orbiter Radio/Plasma Wave, TDS L2 parameters", "rpw-tds-surv-mamp": "Solar Orbiter Radio/Plasma Wave, TDS L2 parameters", "rpw-tds-surv-rswf": "Solar Orbiter Radio/Plasma Wave, TDS L1 parameters", - "rpw-tds-surv-rswf-b": "Solar Orbiter Radio/Plasma Wave, TDS L2 magnetic parameters", + "rpw-tds-surv-rswf-b": "Solar Orbiter, Level 2, Radio and Plasma Waves, Time Domain Sampler, Regular Snapshot Waveform of magnetic data in survey mode", "rpw-tds-surv-rswf-e": "Solar Orbiter Radio/Plasma Wave, TDS L2 waveform snapshots", "rpw-tds-surv-stat": "Solar Orbiter Radio/Plasma Wave, TDS L2R parameters", "rpw-tds-surv-tswf": "Solar Orbiter Radio/Plasma Wave, TDS L1 parameters", - "rpw-tds-surv-tswf-b": "Solar Orbiter Radio/Plasma Wave, TDS L2 magnetic parameters", + "rpw-tds-surv-tswf-b": "Solar Orbiter, Level 2, Radio and Plasma Waves, Time Domain Sampler, Triggered Snapshot Waveform of magnetic data in survey mode", "rpw-tds-surv-tswf-e": "Solar Orbiter Radio/Plasma Wave, TDS L2 waveform snapshots", "rpw-tnr": "Solar Orbiter Radio/Plasma Wave, LL02 parameters", "rpw-tnr-fp": "Solar Orbiter Radio/Plasma Wave, data from plasma peak tracking L3", "rpw-tnr-surv": "Solar Orbiter Radio/Plasma Wave, TNR L2 parameters", "sensorrates": "solo_L1_swa-his-sensor_rates", + "solohi-11t": "", + "solohi-12t": "", "solohi-1ft": "", "solohi-1ut": "", "solohi-1vt": "", + "solohi-21s": "", + "solohi-21t": "", + "solohi-22t": "", + "solohi-23t": "", + "solohi-24t": "", "solohi-2ft": "", "solohi-2us": "", "solohi-2ut": "", + "solohi-31t": "", + "solohi-32t": "", "solohi-3fg": "", "solohi-3ft": "", "solohi-3ut": "", "solohi-4-001800001780": "", + "solohi-41t": "", + "solohi-42t": "", "solohi-4fg": "", "solohi-4ft": "", "solohi-4ut": "", diff --git a/sunpy_soar/data/observation_attrs.json b/sunpy_soar/data/observation_attrs.json new file mode 100644 index 0000000..6f363cb --- /dev/null +++ b/sunpy_soar/data/observation_attrs.json @@ -0,0 +1,324 @@ +{ + "CAL_CMPR-B-01_SC_SL02_10.0S_FD": "", + "CAL_CMPR-B-01_SC_SL04_10.0S_FD": "", + "CAL_CMPR-B-01_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-01_SC_SL06_10.0S_FD": "", + "CAL_CMPR-B-02_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-04_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-06_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-08_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-10_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-SHC_SC_SL02_10.0S_FD": "", + "CAL_CMPR-B-SHC_SC_SL04_10.0S_FD": "", + "CAL_CMPR-B-SHC_SC_SL04_30.0S_FD": "", + "CAL_CMPR-B-SHC_SC_SL06_10.0S_FD": "", + "CAL_CMPR-F-01_SC_SL04_120.0S_FD": "", + "CAL_CMPR-F-SHC_SC_SL04_120.0S_FD": "", + "CAL_CMPR-FULL_FS_SL04_60.0S_FD": "", + "CAL_COALIGN-FULL_SC_SL04_19.7S_FD": "", + "CAL_COALIGN-FULL_SC_SL04_4.7S_FD": "", + "CAL_COALIGN-FULL_SS_SL04_19.7S_FD": "", + "CAL_COALIGN-REDUCED_SC_SL04_19.7S_FD": "", + "CAL_CP-30-MOVIE-180_TS_SL30_9.8S_FD": "", + "CAL_CP-30-MOVIE-192_TS_SL30_5.0S_FD": "", + "CAL_CP-30-MOVIE-384_TS_SL30_5.0S_FD": "", + "CAL_CP-90-MOVIE_SS_SL30_5.0S_FD": "", + "CAL_CP-MOSAIC_SS_SL04_4.7S_FD": "", + "CAL_CP_30_MOVIE_TS_SL30_5.0S_FD": "", + "CAL_CP_COALIGN_SC_SL04_20.0S_FD": "", + "CAL_CP_COALIGN_SC_SL04_59.7S_FD": "", + "CAL_CP_COALIGN_SC_SL04_9.7S_FD": "", + "CAL_CP_COALIGN_SC_SL04_9.7S_FF": "", + "CAL_DARK-ND_FS_SL04_0.5S_FD": "", + "CAL_DARK-ND_FS_SL04_10.0S_FD": "", + "CAL_DARK-ND_FS_SL04_60.0S_FD": "", + "CAL_DARK-SUB_FS_SL04_60.0S_FD": "", + "CAL_DARK-SUB_FS_SL06_30.0S_FD": "", + "CAL_DARK_FS_SL04_0.5S_FD": "", + "CAL_DARK_FS_SL04_1.0S_FD": "", + "CAL_DARK_FS_SL04_10.0S_FD": "", + "CAL_DARK_FS_SL04_100.0S_FD": "", + "CAL_DARK_FS_SL04_120.0S_FD": "", + "CAL_DARK_FS_SL04_14.7S_FD": "", + "CAL_DARK_FS_SL04_180.0S_FD": "", + "CAL_DARK_FS_SL04_2.5S_FD": "", + "CAL_DARK_FS_SL04_20.0S_FD": "", + "CAL_DARK_FS_SL04_30.0S_FD": "", + "CAL_DARK_FS_SL04_360.0S_FD": "", + "CAL_DARK_FS_SL04_5.0S_FD": "", + "CAL_DARK_FS_SL04_58.0S_FD": "", + "CAL_DARK_FS_SL04_60.0S_FD": "", + "CAL_DARK_FS_SL04_600.0S_FD": "", + "CAL_DARK_FS_SL04_90.0S_FD": "", + "CAL_Doppler-Test_SC_SL04_10.0S_FF": "", + "CAL_Doppler-Test_SC_SL04_20.0S_FF": "", + "CAL_FIRST-LIGHT-ND_FS_SL04_10.0S_FD": "", + "CAL_FIRST-LIGHT-ND_FS_SL04_60.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL02_180.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL02_60.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL04_60.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL06_30.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL06_60.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL30_30.0S_FD": "", + "CAL_FIRST-LIGHT_FS_SL30_60.0S_FD": "", + "CAL_FLAT-COLD_FS_SL30_60.0S_FD": "", + "CAL_FLAT-WARM_FS_SL30_60.0S_FD": "", + "CAL_FOCUS-COLD_TS_SL30_10.0S_FD": "", + "CAL_FOCUS-COLD_TS_SL30_30.0S_FD": "", + "CAL_FOCUS-TEST_SC_SL02_20.0S_FF": "", + "CAL_FOCUS-TEST_SC_SL04_20.0S_FF": "", + "CAL_FOCUS-TEST_SC_SL06_20.0S_FF": "", + "CAL_FOCUS-TEST_SS_SL02_20.0S_FF": "", + "CAL_FOCUS-TEST_SS_SL04_20.0S_FF": "", + "CAL_FOCUS-TEST_SS_SL06_20.0S_FF": "", + "CAL_FOCUS-WARM_TS_SL30_10.0S_FD": "", + "CAL_FOCUS-WARM_TS_SL30_30.0S_FD": "", + "CAL_FULL-SPEC-COMP_FS_SL02_60.0S_FD": "", + "CAL_FULLSP-PREP_TS_SL04_0.5S_FD": "", + "CAL_INT-REF_SC_SL02_20.0S_FD": "", + "CAL_INT-TEST_SC_SL02_20.0S_FD": "", + "CAL_LBIN-REF_SC_SL02_20.0S_FD": "", + "CAL_LBIN-REF_SC_SL04_20.0S_FD": "", + "CAL_LBIN-TEST2_SC_SL02_20.0S_FD": "", + "CAL_LBIN-TEST2_SC_SL04_20.0S_FD": "", + "CAL_LBIN-TEST4_SC_SL02_20.0S_FD": "", + "CAL_LIMB-FAST_SC_SL04_20.0S_FD": "", + "CAL_LIMB-SLOW_SC_SL02_60.0S_FD": "", + "CAL_LL_FIRST-LIGHT_FS_SL04_60.0S_FD": "", + "CAL_MOSAIC_SC_SL06_9.7S_FD": "", + "CAL_MOSAIC_SS_SL30_10.0S_FD": "", + "CAL_MOSAIC_SS_SL30_5.0S_FD": "", + "CAL_MOSAIC_SS_SL30_8.9S_FD": "", + "CAL_MOSAIC_SS_SL30_9.9S_FD": "", + "CAL_OFFLIMB_FS_SL04_120.0S_FD": "", + "CAL_OFFLIMB_FS_SL04_180.0S_FD": "", + "CAL_OFFLIMB_FS_SL04_360.0S_FD": "", + "CAL_POINT-COLD-14STEP_SS_SL30_30.0S_FD": "", + "CAL_POINT-COLD-2STEP_SS_SL30_30.0S_FD": "", + "CAL_POINT-COLD-4STEP_SS_SL30_30.0S_FD": "", + "CAL_POINT-COLD-6STEP_SS_SL30_30.0S_FD": "", + "CAL_POINT-COLD-QUICK_SS_SL30_10.0S_FD": "", + "CAL_POINT-COLD-QUICK_SS_SL30_5.0S_FD": "", + "CAL_POINT-COLD_SC_SL02_120.0S_FD": "", + "CAL_POINT-COLD_SC_SL02_5.0S_FD": "", + "CAL_POINT-COLD_SC_SL04_60.0S_FD": "", + "CAL_POINT-COLD_SC_SL06_60.0S_FD": "", + "CAL_POINT-COMP-ND_SS_SL04_59.7S_FD": "", + "CAL_POINT-COMP-ND_SS_SL04_9.7S_FD": "", + "CAL_POINT-COMP_SC_SL02_20.0S_FD": "", + "CAL_POINT-COMP_SC_SL04_20.0S_FD": "", + "CAL_POINT-COMP_SC_SL06_20.0S_FD": "", + "CAL_POINT-COMP_SS_SL02_19.7S_FD": "", + "CAL_POINT-COMP_SS_SL02_19.7S_FF": "", + "CAL_POINT-COMP_SS_SL04_20.0S_FD": "", + "CAL_POINT-COMP_SS_SL04_59.7S_FD": "", + "CAL_POINT-COMP_SS_SL04_9.7S_FD": "", + "CAL_POINT-COMP_SS_SL04_9.7S_FF": "", + "CAL_POINT-NOSCAN_SC_SL04_9.7S_FD": "", + "CAL_POINT-QUICK_SC_SL02_1.0S_FD": "", + "CAL_POINT-QUICK_SC_SL02_5.0S_FD": "", + "CAL_POINT-WARM-14STEP_SS_SL30_30.0S_FD": "", + "CAL_POINT-WARM_SC_SL02_10.0S_FD": "", + "CAL_POINT-WARM_SC_SL02_120.0S_FD": "", + "CAL_POINT-WARM_SC_SL02_5.0S_FD": "", + "CAL_POINT-WARM_SC_SL04_60.0S_FD": "", + "CAL_POINT-WARM_SC_SL06_60.0S_FD": "", + "CAL_POINT-WARM_SS_SL30_30.0S_FD": "", + "CAL_POINT_SC_SL02_120.0S_FD": "", + "CAL_POINT_SC_SL02_5.0S_FD": "", + "CAL_POINT_SC_SL04_60.0S_FD": "", + "CAL_POLAR-FULL-SPEC_FS_SL04_120.0S_FD": "", + "CAL_POLAR_SC_SL04_60.0S_FD": "", + "CAL_ROLL-TEST_SC_SL02_10.0S_FD": "", + "CAL_ROLL-TEST_SC_SL06_20.0S_FD": "", + "CAL_ROLL-TEST_SS_SL02_20.0S_FD": "", + "CAL_ROLL-TEST_SS_SL04_20.0S_FD": "", + "CAL_ROLL-TEST_SS_SL04_9.7S_FD": "", + "CAL_SPARSE-2_SC_SL02_20.0S_FF": "", + "CAL_SPARSE-2_SC_SL04_20.0S_FF": "", + "CAL_SPARSE-3_SC_SL04_20.0S_FF": "", + "CAL_SPARSE-4-C01_SC_SL02_20.0S_FF": "", + "CAL_SPARSE-4-C04_SC_SL02_20.0S_FF": "", + "CAL_SPARSE-4-C10_SC_SL02_20.0S_FF": "", + "CAL_SPARSE-8_SC_SL02_20.0S_FF": "", + "CAL_SPARSE-REF_SC_SL02_20.0S_FF": "", + "CAL_SPARSE-REF_SC_SL04_20.0S_FF": "", + "CAL_SPECTRAL-RESPONSE_FS_SL02_60.0S_FD": "", + "CAL_SPECTRAL-RESPONSE_FS_SL04_100.0S_FD": "", + "CAL_SPECTRAL-RESPONSE_FS_SL04_120.0S_FD": "", + "CAL_SPECTRAL-RESPONSE_FS_SL04_60.0S_FD": "", + "CAL_SPECTRAL-RESPONSE_FS_SL04_90.0S_FD": "", + "CAL_SPECTRAL-RESPONSE_FS_SL06_60.0S_FD": "", + "CAL_STAR-OBS_SC_SL04_60.0S_FD": "", + "CAL_STAR-OBS_SC_SL30_30.0S_FD": "", + "CAL_STAR-OBS_SC_SL30_60.0S_FD": "", + "CAL_YBIN-REF_SC_SL02_29.2S_FD": "", + "CAL_YBIN-TEST_SC_SL02_29.2S_FD": "", + "EMC_interference": "", + "EUI_ANNEALING": "", + "EUV_AR": "", + "EUV_CALIBRATION": "", + "EUV_CH": "", + "EUV_FLATFIELD": "", + "EUV_GENERIC": "", + "EUV_QS": "", + "FSI_CALIBRATION": "", + "FSI_ERUPTIVE_B": "", + "FSI_FAINTHIGHCORONA": "", + "FSI_FINDEVENT_S_B": "", + "FSI_FLATFIELD": "", + "FSI_GENERIC_S_B": "", + "FSI_GLOBAL_S_B": "", + "FSI_SYNOPTIC_B": "", + "FSI_SYNOPTIC_B;GENERIC_CALIB": "", + "GENERIC_CALIB": "", + "HI_CALIB_GENERIC": "", + "HI_CALIB_STRAYLIGHT": "", + "HI_GENERIC": "", + "HI_SHOCK": "", + "HI_SHOCK_NEAR": "", + "HI_SHOCK_PERIH": "", + "HI_SYN_FAR": "", + "HI_SYN_MID": "", + "HI_SYN_NEAR": "", + "HI_TURB_JITTER": "", + "HI_TURB_NEAR": "", + "IU-10_1-2": "", + "IU-10_4": "", + "IU-10_4_EUV_offset": "", + "IU-10_4_FSI_occulter": "", + "IU-10_4_FSI_reconstruction": "", + "IU-10_5-10": "", + "IU-10_5_FSI_linearity": "", + "IU-10_6": "", + "IU-10_6_FSI_compression": "", + "IU-12_10": "", + "IU-12_11": "", + "IU-12_12": "", + "IU-12_2": "", + "IU-12_5": "", + "IU-12_6": "", + "IU-12_9": "", + "IU-2b": "", + "IU-4_1-3a": "", + "IU-4_3b": "", + "IU-4_4-5": "", + "IU-4_contingency_3": "", + "IU-5": "", + "IU-7": "", + "IU-8_1": "", + "IU-8_2-7": "", + "IU-9_5-14": "", + "IU-9_5-14b": "", + "LYA_AR": "", + "LYA_CALIBRATION": "", + "LYA_CH": "", + "LYA_FLATFIELD": "", + "LYA_GENERIC": "", + "LYA_QS": "", + "METIS_FLUCTS_TBF": "", + "METIS_GENERIC": "", + "METIS_GENERIC_OBS": "", + "METIS_GENERIC_UV": "", + "METIS_GENERIC_VL": "", + "METIS_GLOBAL": "", + "METIS_MAGTOP": "", + "METIS_PROBE_COMET": "", + "METIS_SYNOPTIC": "", + "METIS_UV_RC": "", + "METIS_VL_RC": "", + "METIS_WIND": "", + "NECP_alignment": "", + "NO_LTP_ID": "", + "SCI_AR-HEATING_SC_SL02_19.7S_FF": "", + "SCI_AR-HEATING_SC_SL04_9.7S_FF": "", + "SCI_BP-SURVEY_SC_SL04_19.7S_FF": "", + "SCI_COMP-CH-BOUNDARY_SC_SL04_60.0S_FF": "", + "SCI_COMP-VS-HEIGHT-TEST1_SC_SL04_60.0S_FF": "", + "SCI_COMP-VS-HEIGHT-TEST2_SC_SL04_60.0S_FF": "", + "SCI_COMPO-COOLHOT_SC_SL04_60.0S_FF": "", + "SCI_COMPO-COOL_SC_SL04_120.0S_FF": "", + "SCI_COMPO-LIMB_SC_SL04_120.0S_FF": "", + "SCI_COMPO-LIMB_SC_SL04_60.0S_FF": "", + "SCI_COMPO-SLOW-WIND_SC_SL04_60.0S_FF": "", + "SCI_COMPO-SLOW_SC_SL06_60.0S_FF": "", + "SCI_COMPO-TEST1_SC_SL04_120.0S_FF": "", + "SCI_COMPO-TEST1_SC_SL04_60.0S_FF": "", + "SCI_COMPO-TEST1_SC_SL04_90.0S_FF": "", + "SCI_COMPO-TEST2_SC_SL04_120.0S_FF": "", + "SCI_COMPO-TEST2_SC_SL04_60.0S_FF": "", + "SCI_COMPO-TEST2_SC_SL04_90.0S_FF": "", + "SCI_COMPO-WIDE_SC_SL04_60.0S_FF": "", + "SCI_COMPO_TEST3_SC_SL06_180.0S_FF": "", + "SCI_CONN-MOSAIC_SC_SL04_120.0S_FF": "", + "SCI_CONN-MOSAIC_SC_SL04_60.0S_FF": "", + "SCI_CONN-MOSAIC_SC_SL04_90.0S_FF": "", + "SCI_CONTEXT-QS-12ARC_SC_SL04_20.0S_FF": "", + "SCI_CONTEXT-QS-2ARC_SS_SL02_10.0S_FF": "", + "SCI_CONTEXT-SUNSPOT_SC_SL04_19.7S_FF": "", + "SCI_CONTEXT_AR_SC_SL04_10.0S_FF": "", + "SCI_CONTEXT_CH_SC_SL04_20.0S_FF": "", + "SCI_CONTEXT_FLR_SC_SL04_10.0S_FF": "", + "SCI_CONTEXT_QS_SC_SL04_15.0S_FF": "", + "SCI_CONTEXT_QS_SC_SL04_20.0S_FF": "", + "SCI_CONTEXT_QS_SS_SL04_10.0S_FF": "", + "SCI_DYN-DEEP_SC_SL04_19.7S_FF": "", + "SCI_DYN-DEEP_SC_SL04_30.0S_FF": "", + "SCI_DYN-FAST-FLARE_SC_SL04_4.7S_FF": "", + "SCI_DYN-FAST_SC_SL04_5.0S_FF": "", + "SCI_DYN-FAST_SS_SL04_4.7S_FF": "", + "SCI_DYN-FAST_SS_SL04_9.7S_FF": "", + "SCI_DYN-MED_SC_SL04_10.0S_FF": "", + "SCI_DYN-MED_SC_SL04_20.0S_FF": "", + "SCI_DYN-MED_SC_SL04_5.0S_FF": "", + "SCI_DYN-MED_SS_SL04_10.0S_FF": "", + "SCI_DYN-MED_SS_SL04_14.7S_FF": "", + "SCI_DYN-MED_SS_SL04_20.0S_FF": "", + "SCI_DYN-QS-MEDIUM-1ARC_SS_SL02_5.0S_FF": "", + "SCI_DYN-SLOW_SC_SL02_5.0S_FF": "", + "SCI_DYN-SLOW_SC_SL04_10.0S_FF": "", + "SCI_DYN-SLOW_SC_SL04_5.0S_FF": "", + "SCI_FLARE_NOSCAN_SC_SL04_4.8S_FF": "", + "SCI_HIGH-CAD_SS_SL04_4.7S_FF": "", + "SCI_HIGH-CAD_SS_SL04_5.0S_FF": "", + "SCI_HIGH-CAD_SS_SL04_9.7S_FF": "", + "SCI_LIMB-EAST_SC_SL04_30.0S_FF": "", + "SCI_LIMB-EAST_SC_SL04_59.7S_FF": "", + "SCI_LIMB-EAST_SS_SL04_59.7S_FF": "", + "SCI_LIMB-FAST_SC_SL04_19.7S_FF": "", + "SCI_LIMB-FAST_SC_SL04_9.7S_FF": "", + "SCI_LIMB-FAST_SS_SL04_19.7S_FF": "", + "SCI_LIMB-WEST-COMPOSITION_SC_SL04_60.0S_FF": "", + "SCI_LIMB-WEST-DYNAMICS_SC_SL04_10.0S_FF": "", + "SCI_LIMB-WEST_SC_SL04_120.0S_FF": "", + "SCI_LIMB-WEST_SC_SL04_59.7S_FF": "", + "SCI_LIMB-WEST_SC_SL04_60.0S_FF": "", + "SCI_LL_CONTEXT-SUNSPOT_SC_SL04_19.7S_FF": "", + "SCI_LL_SPROUT-TEST_SC_SL04_60.0S_FF": "", + "SCI_LL_SYNOPTIC_SC_SL04_5.0S_FF": "", + "SCI_Limb-East_SC_SL04_60.0S_FF": "", + "SCI_MOVIE_SS_SL30_5.0S_FD": "", + "SCI_MOVIE_TS_SL30_4.7S_FD": "", + "SCI_POLAR-NOSCAN_SC_SL04_14.7S_FF": "", + "SCI_POLAR-PLUMES_SC_SL04_359.5S_FF": "", + "SCI_POLAR_SC_SL04_60.0S_FF": "", + "SCI_PROM-EAST_SC_SL04_60.0S_FF": "", + "SCI_SPROUT-TEST_SC_SL04_60.0S_FF": "", + "SCI_SPROUT-TEST_SC_SL06_60.0S_FF": "", + "SCI_SUNSPOT-NOSCAN_SC_SL04_9.7S_FF": "", + "SCI_SYNOPTIC_SC_SL02_20.0S_FF": "", + "SCI_SYNOPTIC_SC_SL04_19.7S_FF": "", + "SCI_SYNOPTIC_SC_SL04_20.0S_FF": "", + "SCI_SYNOPTIC_SC_SL04_60.0S_FF": "", + "SCI_WAVES-QS_TS_SL04_10.0S_FF": "", + "SCI_WAVES-QS_TS_SL04_9.9S_FF": "", + "SCI_WAVES_SS_SL30_9.8S_FA": "", + "SCI_WAVES_TS_SL02_10.0S_FD": "", + "SCI_WAVES_TS_SL04_10.0S_FD": "", + "SCI_WAVES_TS_SL04_10.0S_FF": "", + "SCI_WAVES_TS_SL04_9.8S_FF": "", + "SCI_WAVES_TS_SL30_4.8S_FA": "", + "SCI_WAVES_TS_SL30_9.8S_FA": "", + "Star_calibration_leo_alpha": "", + "unknown_obs_mode": "" +} diff --git a/tools/update_data.py b/tools/update_data.py index 45b9fb6..fed918a 100644 --- a/tools/update_data.py +++ b/tools/update_data.py @@ -3,7 +3,7 @@ from astroquery.utils.tap.core import TapPlus -soar = TapPlus("http://soar.esac.esa.int/soar-sl-tap/tap") +soar = TapPlus(url="http://soar.esac.esa.int/soar-sl-tap/tap") def get_cdf_descriptors(): @@ -21,7 +21,7 @@ def get_cdf_descriptors(): def get_fits_descriptors(): # Get data descriptors for FITS files print("Updating FITS descriptors...") - soar = TapPlus("http://soar.esac.esa.int/soar-sl-tap/tap") + soar = TapPlus(url="http://soar.esac.esa.int/soar-sl-tap/tap") job = soar.launch_job("select * from soar.fits_dataset") res = job.get_results() descriptors = {} @@ -79,6 +79,31 @@ def get_all_soops(): return soop_names +def get_observation_modes(): + # Fetch observation modes for all instruments combined + print("Updating observation modes...") + obs_modes = {} + instruments = [ + "eui", "phi", "shi", "spi", "met" + ] + + # Set to hold unique observation modes + unique_modes = set() + + for instrument in instruments: + query = f"SELECT DISTINCT observation_mode FROM v_{instrument}_sc_fits" + job = soar.launch_job(query) + res = job.get_results() + for row in res: + mode = row['observation_mode'].strip() # Strip leading and trailing whitespace + if mode: # Ensure mode is not an empty string + unique_modes.add(mode) + + # Convert the set to a sorted list and create a dictionary + obs_modes = {mode: "" for mode in sorted(unique_modes)} + + return obs_modes + if __name__ == "__main__": attr_file = ( pathlib.Path(__file__).parent.parent / "sunpy_soar" / "data" / "attrs.json" @@ -103,3 +128,10 @@ def get_all_soops(): soop_descriptors = get_all_soops() with soop_file.open("w") as soops_file: json.dump(dict(sorted(soop_descriptors.items())), soops_file, indent=2) + + obs_modes_file = ( + pathlib.Path(__file__).parent.parent / "sunpy_soar" / "data" / "observation_attrs.json" + ) + observation_modes = get_observation_modes() + with obs_modes_file.open("w") as obs_file: + json.dump(observation_modes, obs_file, indent=2)