Mag l1b CDF cleanup (#1173)

* Best version so far * Fixing mag CDF attributes * cleaning up * fixing test data * Update imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml Co-authored-by: Tenzin Choedon <[email protected]> * Fixing pre-commit --------- Co-authored-by: Tenzin Choedon <[email protected]>
IMAP-Science-Operations-Center · Nov 21, 2024 · 8d1ad62 · 8d1ad62
1 parent b88e2c7
commit 8d1ad62
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 43 deletions.
diff --git a/imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml
@@ -1,20 +1,5 @@
 # <=== Label Attributes ===>
 # LABL_PTR_i expects VAR_TYPE of metadata with char data type.
-# We need to define this if we have DEPEND_1 or more.
-# TODO: I am not sure what the FIELDNAM should be.
-# I tried best to match this: https://spdf.gsfc.nasa.gov/istp_guide/variables.html#Metadata_eg1
-direction_label:
-  CATDESC: magnetic field vector data
-  FIELDNAM: Magnetic Field Vector
-  FORMAT: A3
-  VAR_TYPE: metadata
-
-compression_label:
-  CATDESC: Compression and width
-  FIELDNAM: Compression and width
-  FORMAT: A2
-  VAR_TYPE: metadata
-
 default_attrs: &default
   # Assumed values for all variable attrs unless overwritten
   DEPEND_0: epoch
@@ -29,40 +14,50 @@ default_attrs: &default
 default_coords: &default_coords
   # Assumed values for all coordinate attrs unless overwritten
   FORMAT: F2.4  # Float with 4 digits
-  VAR_TYPE: data
+  VAR_TYPE: support_data
   DISPLAY_TYPE: time_series
   FILLVAL: -9223372036854775808
   VALIDMIN: -9223372036854775808
   VALIDMAX: 9223372036854775807
   UNITS: counts
 
+mag_support_attrs: &support_default
+    <<: *default
+    VAR_TYPE: support_data
+
+mag_metadata_attrs: &metadata_default
+    <<: *default
+    VAR_TYPE: metadata
+
+direction_label:
+  CATDESC: magnetic field vector data
+  FIELDNAM: Magnetic Field Vector
+  FORMAT: A3
+  VAR_TYPE: metadata
+
+compression_label:
+  CATDESC: Compression and width
+  FIELDNAM: Compression and width
+  FORMAT: A2
+  VAR_TYPE: metadata
+
 raw_vector_attrs:
-  <<: *default_coords
+  <<: *default
   CATDESC: Raw unprocessed magnetic field vector data in bytes
-  DEPEND_0: epoch
   DEPEND_1: direction
   LABL_PTR_1: direction_label
   FIELDNAM: Magnetic Field Vector
   FORMAT: I3
 
 vector_attrs:
-    <<: *default_coords
+    <<: *default
     CATDESC: Magnetic field vector with x y z and sensor range varying by time
-    DEPEND_0: epoch
     DEPEND_1: direction
     FIELDNAM: Magnetic Field Vector
     LABL_PTR_1: direction_label
     FILLVAL: 9223372036854775807
     FORMAT: I3
 
-mag_support_attrs: &support_default
-    <<: *default
-    VAR_TYPE: support_data
-
-mag_metadata_attrs: &metadata_default
-    <<: *default
-    VAR_TYPE: metadata
-
 mag_flag_attrs:
     <<: *default
     VALIDMIN: 0
@@ -100,6 +95,7 @@ compression_flags_attrs:
   <<: *support_default
   CATDESC: Compression information per time stamp, includes a flag and the compression width
   FIELDNAM: Compression information per time stamp
+  DEPEND_0: epoch
   DEPEND_1: compression
   FORMAT: I2
   VALIDMAX: 21

diff --git a/imap_processing/mag/l0/decom_mag.py b/imap_processing/mag/l0/decom_mag.py
@@ -129,7 +129,9 @@ def generate_dataset(
         direction.astype(str),
         name="direction_label",
         dims=["direction_label"],
-        attrs=attribute_manager.get_variable_attributes("direction_label"),
+        attrs=attribute_manager.get_variable_attributes(
+            "direction_label", check_schema=False
+        ),
     )
 
     # TODO: Epoch here refers to the start of the sample. Confirm that this is
@@ -154,11 +156,10 @@ def generate_dataset(
         coords={
             "epoch": epoch_time,
             "direction": direction,
-            "direction_label": direction_label,
         },
         attrs=attribute_manager.get_global_attributes(logical_id),
     )
-
+    output["direction_label"] = direction_label
     output["raw_vectors"] = raw_vectors
 
     for key, value in support_data.items():

diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py
@@ -158,7 +158,6 @@ def process_packets(
                 "timedelta64[ns]"
             )
         ).astype("datetime64[D]")
-
         primary_packet_properties = MagL1aPacketProperties(
             mag_l0.SHCOARSE,
             primary_start_time,
@@ -284,9 +283,7 @@ def generate_dataset(
 
     # TODO: Just leave time in datetime64 type with vector as dtype object to avoid this
     # Get the timestamp from the end of the vector
-    time_data = single_file_l1a.vectors[:, 4].astype(
-        np.dtype("datetime64[ns]"), copy=False
-    )
+    time_data = single_file_l1a.vectors[:, 4]
 
     compression = xr.DataArray(
         np.arange(2),
@@ -326,30 +323,33 @@ def generate_dataset(
     )
 
     direction_label = xr.DataArray(
-        direction.astype(str),
+        direction.values.astype(str),
         name="direction_label",
         dims=["direction_label"],
-        attrs=attribute_manager.get_variable_attributes("direction_label"),
+        attrs=attribute_manager.get_variable_attributes(
+            "direction_label", check_schema=False
+        ),
     )
 
     compression_label = xr.DataArray(
-        compression.astype(str),
+        compression.values.astype(str),
         name="compression_label",
         dims=["compression_label"],
-        attrs=attribute_manager.get_variable_attributes("compression_label"),
+        attrs=attribute_manager.get_variable_attributes(
+            "compression_label", check_schema=False
+        ),
     )
 
     output = xr.Dataset(
         coords={
             "epoch": epoch_time,
             "direction": direction,
             "compression": compression,
-            "direction_label": direction_label,
-            "compression_label": compression_label,
         },
         attrs=attribute_manager.get_global_attributes(logical_file_id),
     )
-
+    output["direction_label"] = direction_label
+    output["compression_label"] = compression_label
     output["vectors"] = vectors
     output["compression_flags"] = compression_flags
 

diff --git a/imap_processing/mag/l1b/mag_l1b.py b/imap_processing/mag/l1b/mag_l1b.py
@@ -64,6 +64,8 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
     """
     # TODO: There is a time alignment step that will add a lot of complexity.
     # This needs to be done once we have some SPICE time data.
+    mag_attributes = ImapCdfAttributes()
+    mag_attributes.add_instrument_variable_attrs("mag", "l1")
 
     dims = [["direction"], ["compression"]]
     new_dims = [["direction"], ["compression"]]
@@ -91,7 +93,20 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
     output_dataset = input_dataset.copy()
     output_dataset["vectors"].data = l1b_fields[0].data
 
-    # TODO add/update attributes
+    output_dataset["epoch"].attrs = mag_attributes.get_variable_attributes("epoch")
+    output_dataset["direction"].attrs = mag_attributes.get_variable_attributes(
+        "direction_attrs"
+    )
+    output_dataset["compression"].attrs = mag_attributes.get_variable_attributes(
+        "compression_attrs"
+    )
+    output_dataset["direction_label"].attrs = mag_attributes.get_variable_attributes(
+        "direction_label", check_schema=False
+    )
+    output_dataset["compression_label"].attrs = mag_attributes.get_variable_attributes(
+        "compression_label", check_schema=False
+    )
+
     return output_dataset
 
 

diff --git a/imap_processing/tests/mag/test_mag_l1b.py b/imap_processing/tests/mag/test_mag_l1b.py
@@ -13,6 +13,19 @@ def mag_l1a_dataset():
     epoch = xr.DataArray(np.arange(20), name="epoch", dims=["epoch"])
     direction = xr.DataArray(np.arange(4), name="direction", dims=["direction"])
     compression = xr.DataArray(np.arange(2), name="compression", dims=["compression"])
+
+    direction_label = xr.DataArray(
+        direction.values.astype(str),
+        name="direction_label",
+        dims=["direction_label"],
+    )
+
+    compression_label = xr.DataArray(
+        compression.values.astype(str),
+        name="compression_label",
+        dims=["compression_label"],
+    )
+
     vectors = xr.DataArray(
         np.zeros((20, 4)),
         dims=["epoch", "direction"],
@@ -29,6 +42,8 @@ def mag_l1a_dataset():
     )
     output_dataset["vectors"] = vectors
     output_dataset["compression_flags"] = compression_flags
+    output_dataset["direction_label"] = direction_label
+    output_dataset["compression_label"] = compression_label
 
     return output_dataset