address requested PR changes part 1

seaice_ecdr/ancillary.py

@@ -24,6 +24,7 @@
     ProductVersion,
 )
 from seaice_ecdr.grid_id import get_grid_id
+from seaice_ecdr.nc_util import remove_FillValue_from_coordinate_vars
 from seaice_ecdr.platforms import PLATFORM_CONFIG, Platform
 from seaice_ecdr.platforms.config import N07_PLATFORM
 
@@ -47,21 +48,21 @@ def get_ancillary_filepath(
     return filepath
 
 
-def fix_ds_attrs(ds):
-    """Correct potential errors in the ancillary datasets"""
-    # Remove fill values from x, if it exists
-    try:
-        del ds.variables["x"].encoding["_FillValue"]
-    except KeyError:
-        pass
-
-    # Remove fill values from y, if it exists
-    try:
-        del ds.variables["y"].encoding["_FillValue"]
-    except KeyError:
-        pass
-
-    return ds
+# def fix_ds_attrs(ds):
+#    """Correct potential errors in the ancillary datasets"""
+#    # Remove fill values from x, if it exists
+#    try:
+#        del ds.variables["x"].encoding["_FillValue"]
+#    except KeyError:
+#        pass
+#
+#    # Remove fill values from y, if it exists
+#    try:
+#        del ds.variables["y"].encoding["_FillValue"]
+#    except KeyError:
+#        pass
+#
+#    return ds
 
 
 @cache
@@ -86,7 +87,8 @@ def get_ancillary_ds(
     )
     ds = xr.load_dataset(filepath)
 
-    ds = fix_ds_attrs(ds)
+    # ds = fix_ds_attrs(ds)
+    ds = remove_FillValue_from_coordinate_vars(ds)
 
     return ds
 

seaice_ecdr/cli/daily.py

@@ -54,8 +54,7 @@ def make_25km_ecdr(
     # separately:
     amsr2_start_date = start_date
     if end_date >= AM2_START_DATE:
-        if start_date < AM2_START_DATE:
-            amsr2_start_date = AM2_START_DATE
+        amsr2_start_date = max(start_date, AM2_START_DATE)
 
     if amsr2_start_date >= AM2_START_DATE:
         run_cmd(

seaice_ecdr/daily_aggregate.py

@@ -20,7 +20,11 @@
 from pm_tb_data._types import Hemisphere
 
 from seaice_ecdr._types import ECDR_SUPPORTED_RESOLUTIONS
-from seaice_ecdr.ancillary import ANCILLARY_SOURCES, get_ancillary_ds
+from seaice_ecdr.ancillary import (
+    ANCILLARY_SOURCES,
+    get_ancillary_ds,
+    remove_FillValue_from_coordinate_vars,
+)
 from seaice_ecdr.checksum import write_checksum_file
 from seaice_ecdr.constants import DEFAULT_BASE_OUTPUT_DIR
 from seaice_ecdr.nc_attrs import get_global_attrs
@@ -159,6 +163,7 @@ def _update_ncrcat_daily_ds(
         platform_ids=[platform_id_from_filename(fp.name) for fp in daily_filepaths],
         resolution=resolution,
         hemisphere=hemisphere,
+        ancillary_source=ancillary_source,
     )
     ds.attrs = daily_aggregate_ds_global_attrs  # type: ignore[assignment]
 
@@ -215,8 +220,9 @@ def make_daily_aggregate_netcdf_for_year(
             fix_daily_aggregate_ncattrs(daily_ds)
 
             # Write out the aggregate daily file.
-            daily_ds.variables["x"].encoding["_FillValue"] = None
-            daily_ds.variables["y"].encoding["_FillValue"] = None
+            # daily_ds.variables["x"].encoding["_FillValue"] = None
+            # daily_ds.variables["y"].encoding["_FillValue"] = None
+            daily_ds = remove_FillValue_from_coordinate_vars(daily_ds)
             daily_ds.to_netcdf(
                 output_path,
             )

seaice_ecdr/days_treated_differently.py

@@ -82,8 +82,6 @@ def periods_of_cdr_missing_data(
     """Returns a list of date ranges for which CDR's output should be
     all-missing.
     """
-    # missing_ranges = _load_missing_ranges()
-    # ranges = missing_ranges['treat_outputs_as_missing'][hemisphere_long]
     dates_treated_differently = _load_dates_treated_differently()
     try:
         ranges = dates_treated_differently["treat_outputs_as_missing"][platform_id][

seaice_ecdr/fill_polehole.py

@@ -11,7 +11,6 @@
 from scipy.ndimage import binary_dilation
 
 
-# TODO: differentiate this from the function in `compute_bt_ic`
 def fill_pole_hole(
     *, conc: npt.NDArray, near_pole_hole_mask: npt.NDArray[np.bool_]
 ) -> npt.NDArray:
@@ -20,6 +19,8 @@ def fill_pole_hole(
     Assumes that some data is available in and adjacent to the masked area.
 
     Missing areas are given by `np.nan`.
+
+    If no nearby values are available the pole hole is unchanged.
     """
     extended_nearpole_mask = binary_dilation(near_pole_hole_mask)
 

seaice_ecdr/initial_daily_ecdr.py

@@ -49,6 +49,7 @@
     day_has_all_bad_tbs,
 )
 from seaice_ecdr.grid_id import get_grid_id
+from seaice_ecdr.nc_util import remove_FillValue_from_coordinate_vars
 from seaice_ecdr.platforms import PLATFORM_CONFIG, SUPPORTED_PLATFORM_ID
 from seaice_ecdr.regrid_25to12 import reproject_ideds_25to12
 from seaice_ecdr.spillover import LAND_SPILL_ALGS, land_spillover
@@ -218,7 +219,7 @@ def calc_cdr_conc(
     return cdr_conc
 
 
-def setup_ecdr_ds(
+def _setup_ecdr_ds(
     *,
     date: dt.date,
     tb_data: EcdrTbData,
@@ -468,7 +469,6 @@ def compute_initial_daily_ecdr_dataset(
     tb_data: EcdrTbData,
     land_spillover_alg: LAND_SPILL_ALGS,
     ancillary_source: ANCILLARY_SOURCES,
-    skip_dynamic_BT_tiepoints: bool = False,
 ) -> xr.Dataset:
     """Create intermediate daily ECDR xarray dataset.
 
@@ -479,7 +479,7 @@ def compute_initial_daily_ecdr_dataset(
     fields. Its difficult to understand what's in the resulting dataset without
     manually inspecting the result of running this code.
     """
-    ecdr_ide_ds = setup_ecdr_ds(
+    ecdr_ide_ds = _setup_ecdr_ds(
         date=date,
         tb_data=tb_data,
         hemisphere=hemisphere,
@@ -488,7 +488,7 @@ def compute_initial_daily_ecdr_dataset(
 
     # Spatially interpolate the brightness temperatures
     for tbname in EXPECTED_ECDR_TB_NAMES:
-        # TODO: Replace this TB assignment with
+        # TODO: WIP:  Replace this TB assignment with
         #       util.py's create_tb_dataarray()
         tb_day_name = f"{tbname}_day"
 
@@ -859,40 +859,23 @@ def compute_initial_daily_ecdr_dataset(
             water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],  # note: water
         )
 
-        # TODO: Keeping commented-out weather_mask kwarg calls to highlight
-        #       the transition from weather_mask to water_mask in these function
-        #       calls
-        # TODO: Temporarily forcing skipping of water tiepoint calculation to examine
-        #       using skip_dynamic_BT_tiepoints kwarg
-        if skip_dynamic_BT_tiepoints:
-            logger.warning("SKIPPING calculation of water tiepoint to match CDRv4")
-            bt_coefs["bt_wtp_v37"] = bt_coefs_init["bt_wtp_v37"]
-        else:
-            bt_coefs["bt_wtp_v37"] = bt.calculate_water_tiepoint(
-                wtp_init=bt_coefs_init["bt_wtp_v37"],
-                water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],
-                tb=bt_v37,
-            )
+        bt_coefs["bt_wtp_v37"] = bt.calculate_water_tiepoint(
+            wtp_init=bt_coefs_init["bt_wtp_v37"],
+            water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],
+            tb=bt_v37,
+        )
 
-        if skip_dynamic_BT_tiepoints:
-            logger.warning("SKIPPING calculation of water tiepoint to match CDRv4")
-            bt_coefs["bt_wtp_h37"] = bt_coefs_init["bt_wtp_h37"]
-        else:
-            bt_coefs["bt_wtp_h37"] = bt.calculate_water_tiepoint(
-                wtp_init=bt_coefs_init["bt_wtp_h37"],
-                water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],
-                tb=bt_h37,
-            )
+        bt_coefs["bt_wtp_h37"] = bt.calculate_water_tiepoint(
+            wtp_init=bt_coefs_init["bt_wtp_h37"],
+            water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],
+            tb=bt_h37,
+        )
 
-        if skip_dynamic_BT_tiepoints:
-            logger.warning("SKIPPING calculation of water tiepoint to match CDRv4")
-            bt_coefs["bt_wtp_v19"] = bt_coefs_init["bt_wtp_v19"]
-        else:
-            bt_coefs["bt_wtp_v19"] = bt.calculate_water_tiepoint(
-                wtp_init=bt_coefs_init["bt_wtp_v19"],
-                water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],
-                tb=bt_v19,
-            )
+        bt_coefs["bt_wtp_v19"] = bt.calculate_water_tiepoint(
+            wtp_init=bt_coefs_init["bt_wtp_v19"],
+            water_mask=ecdr_ide_ds["bt_weather_mask"].data[0, :, :],
+            tb=bt_v19,
+        )
 
         bt_coefs["ad_line_offset"] = bt.get_adj_ad_line_offset(
             wtp_x=bt_coefs["bt_wtp_v37"],
@@ -1275,8 +1258,9 @@ def write_ide_netcdf(
                 "zlib": True,
             }
 
-    ide_ds.variables["x"].encoding["_FillValue"] = None
-    ide_ds.variables["y"].encoding["_FillValue"] = None
+    # ide_ds.variables["x"].encoding["_FillValue"] = None
+    # ide_ds.variables["y"].encoding["_FillValue"] = None
+    ide_ds = remove_FillValue_from_coordinate_vars(ide_ds)
     ide_ds.to_netcdf(
         output_filepath,
         encoding=nc_encoding,

seaice_ecdr/intermediate_monthly.py

@@ -36,7 +36,11 @@
 from pm_tb_data._types import NORTH, Hemisphere
 
 from seaice_ecdr._types import ECDR_SUPPORTED_RESOLUTIONS
-from seaice_ecdr.ancillary import ANCILLARY_SOURCES, flag_value_for_meaning
+from seaice_ecdr.ancillary import (
+    ANCILLARY_SOURCES,
+    flag_value_for_meaning,
+    remove_FillValue_from_coordinate_vars,
+)
 from seaice_ecdr.constants import DEFAULT_BASE_OUTPUT_DIR
 from seaice_ecdr.days_treated_differently import months_of_cdr_missing_data
 from seaice_ecdr.intermediate_daily import get_ecdr_filepath
@@ -666,6 +670,7 @@ def make_intermediate_monthly_ds(
         platform_ids=[platform_id],
         resolution=resolution,
         hemisphere=hemisphere,
+        ancillary_source=ancillary_source,
     )
     monthly_ds.attrs.update(monthly_ds_global_attrs)
 
@@ -745,8 +750,9 @@ def make_intermediate_monthly_nc(
 
     # Set `x` and `y` `_FillValue` to `None`. Although unset initially, `xarray`
     # seems to default to `np.nan` for variables without a FillValue.
-    monthly_ds.x.encoding["_FillValue"] = None
-    monthly_ds.y.encoding["_FillValue"] = None
+    # monthly_ds.x.encoding["_FillValue"] = None
+    # monthly_ds.y.encoding["_FillValue"] = None
+    monthly_ds = remove_FillValue_from_coordinate_vars(monthly_ds)
     monthly_ds.to_netcdf(
         output_path,
         unlimited_dims=[

seaice_ecdr/monthly_aggregate.py

@@ -12,7 +12,11 @@
 from pm_tb_data._types import Hemisphere
 
 from seaice_ecdr._types import ECDR_SUPPORTED_RESOLUTIONS
-from seaice_ecdr.ancillary import ANCILLARY_SOURCES, get_ancillary_ds
+from seaice_ecdr.ancillary import (
+    ANCILLARY_SOURCES,
+    get_ancillary_ds,
+    remove_FillValue_from_coordinate_vars,
+)
 from seaice_ecdr.checksum import write_checksum_file
 from seaice_ecdr.constants import DEFAULT_ANCILLARY_SOURCE, DEFAULT_BASE_OUTPUT_DIR
 from seaice_ecdr.nc_attrs import get_global_attrs
@@ -123,6 +127,7 @@ def _update_ncrcat_monthly_ds(
         platform_ids=[platform_id_from_filename(fp.name) for fp in monthly_filepaths],
         resolution=resolution,
         hemisphere=hemisphere,
+        ancillary_source=ancillary_source,
     )
     agg_ds.attrs = monthly_aggregate_ds_global_attrs  # type: ignore[assignment]
 
@@ -237,8 +242,10 @@ def cli(
             # After the ncrcat process, a few attributes need to be cleaned up
             fix_monthly_aggregate_ncattrs(ds)
 
-            ds.variables["x"].encoding["_FillValue"] = None
-            ds.variables["y"].encoding["_FillValue"] = None
+            # ds.variables["x"].encoding["_FillValue"] = None
+            # ds.variables["y"].encoding["_FillValue"] = None
+            ds = remove_FillValue_from_coordinate_vars(ds)
+
             ds.to_netcdf(
                 output_filepath,
             )