diff --git a/README.md b/README.md index 12363ab8..e2813723 100644 --- a/README.md +++ b/README.md @@ -46,82 +46,7 @@ TODO ## Development/contributing -### For developers at NSIDC - -For developers at NSIDC, the [seaice_ecdr_vm -repository](https://bitbucket.org/nsidc/seaice_ecdr_vm/src/main/) provides the -NSIDC VM configuration for this project. - -An initial copy of the pm_icecon "cdr" generation can be executed from the VM directory: - -`~/seaice_ecdr/` - -using the cli.sh command: - -``` -./scripts/cli.sh bootstrap amsr2 --date 2022-08-01 --hemisphere north --output-dir /tmp/ --resolution 12 -``` - -### Adding dependencies - -To add new dependencies to this project, update the `environment.yml` file with -the new dependency. Then update your conda environment: - -``` -$ mamba env update -``` - -Once the conda environment has been updated, lock the environment using `conda-lock`: - -``` -$ conda-lock -``` - -Commit the changes for the `environment.yml` and the `conda-lock.yml` files. - - -### Running tests/CI - -#### Linting / formatting -This project uses [pre-commit](https://pre-commit.com/) to run pre-commit hooks -that check and format this project's code for stylistic consistency (using -`ruff` and `black`) . - -The pre-commit configuration for this project can be found in -`.pre-commit-config.yaml`. Configuration for specific tools (e.g., `mypy`) is -given in the included `pyproject.toml`. - -For more information about using `pre-commit`, please sese the [Scientific -Python Library Development Guide's section on -pre-commit](https://learn.scientific-python.org/development/guides/gha-basic/#pre-commit). - -To install pre-commit to run checks for each commit you make: - -``` -$ pre-commit install -``` - -To manually run the pre-commit hooks without a commit: - -``` -$ pre-commit run --all-files -``` - -#### Running unit tests - -Use `pytest` to run unit tests: - -``` -$ python -m pytest -``` - -#### Type-checking - -Use `mypy` to run static typechecking - -``` -$ mypy -``` +See [doc/development.md](doc/development.md) for more information. ## License diff --git a/README_dev.txt b/README_dev.txt deleted file mode 100644 index 731ec2d0..00000000 --- a/README_dev.txt +++ /dev/null @@ -1,22 +0,0 @@ - -README_dev.txt - -While developing, the workflow is: - -pm_icecon: - Download pm_icecon repo to ~/pm_icecon - run 'python setup.py develop' - Current branch: origin/update_for_nise_cdr_cetb - -seaice_ecdr: - Download seaice_ecdr repo to ~/seaice_ecdr - run 'python setup.py develop' - Current branch: origin/initial_pmicecon_ecdr_gen - -in ~/seaice_ecdr/: - - Can run unit and integration tests -- or manual subsets of -- with: - ./run_ecdr_pytest.sh - - Can generate a sample netCDF file with: - python seaice_ecdr/tests/integration/gen_ide_sample.py diff --git a/doc/development.md b/doc/development.md new file mode 100644 index 00000000..f39711a1 --- /dev/null +++ b/doc/development.md @@ -0,0 +1,83 @@ +While developing at NSIDC, the workflow is: + +### For developers at NSIDC + +For developers at NSIDC, the [seaice_ecdr_vm +repository](https://bitbucket.org/nsidc/seaice_ecdr_vm/src/main/) provides the +NSIDC VM configuration for this project. + +`pm_icecon`, `seaice_ecdr`, and `pm_tb_data` will be checked out to +`/home/vagrant/{project_name}`. + +### Adding dependencies + +To add new dependencies to this project, update the `environment.yml` file with +the new dependency. Then update your conda environment: + +``` +$ mamba env update +``` + +Once the conda environment has been updated, lock the environment using `conda-lock`: + +``` +$ conda-lock +``` + +Commit the changes for the `environment.yml` and the `conda-lock.yml` files. + + +### Running tests/CI + +#### Linting / formatting +This project uses [pre-commit](https://pre-commit.com/) to run pre-commit hooks +that check and format this project's code for stylistic consistency (using +`ruff` and `black`) . + +The pre-commit configuration for this project can be found in +`.pre-commit-config.yaml`. Configuration for specific tools (e.g., `mypy`) is +given in the included `pyproject.toml`. + +For more information about using `pre-commit`, please sese the [Scientific +Python Library Development Guide's section on +pre-commit](https://learn.scientific-python.org/development/guides/gha-basic/#pre-commit). + +To install pre-commit to run checks for each commit you make: + +``` +$ pre-commit install +``` + +To manually run the pre-commit hooks without a commit: + +``` +$ pre-commit run --all-files +``` + +#### Running unit tests + +Use `pytest` to run unit tests: + +``` +$ python -m pytest +``` + +Alternatively, use `scripts/run_ecdr_pytest.sh`. + +#### Type-checking + +Use `mypy` to run static typechecking + +``` +$ mypy +``` + +## Running tests / generating sample data + +Can run unit and integration tests -- or manual subsets of -- with: + + ./scripts/run_ecdr_pytest.sh + +Create an Initial Daily ECDR file with e.g.,: + + ./scripts/cli.sh idecdr --date 2021-04-05 --hemisphere north --resolution 12 --output-dir /tmp/ diff --git a/gen_sample.sh b/gen_sample.sh deleted file mode 100755 index 08297623..00000000 --- a/gen_sample.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python seaice_ecdr/tests/integration/gen_ide_sample.py diff --git a/scripts/cli.sh b/scripts/cli.sh index 993cf963..86d793ab 100755 --- a/scripts/cli.sh +++ b/scripts/cli.sh @@ -2,9 +2,4 @@ ARGS=$@ -THIS_DIR="$( cd "$(dirname "$0")"; pwd -P )" -PM_ICECON_DIR=/home/vagrant/pm_icecon - -# PYTHONPATH=$THIS_DIR/.. python -m pm_icecon.cli.entrypoint $ARGS -# PYTHONPATH=$THIS_DIR/..:${PM_ICECON_DIR} python -m seaice_ecdr.cli.entrypoint $ARGS -PYTHONPATH=$THIS_DIR/..:${PM_ICECON_DIR} python -m pm_icecon.cli.entrypoint $ARGS +python -m seaice_ecdr.cli.entrypoint $ARGS diff --git a/run_ecdr_pytest.sh b/scripts/run_ecdr_pytest.sh similarity index 100% rename from run_ecdr_pytest.sh rename to scripts/run_ecdr_pytest.sh diff --git a/seaice_ecdr/cdr_alg.py b/seaice_ecdr/cdr_alg.py deleted file mode 100644 index 2af49cb8..00000000 --- a/seaice_ecdr/cdr_alg.py +++ /dev/null @@ -1,406 +0,0 @@ -"""Create a 'simplified' CDR for comparison purposes. - -This code directly copied from `pm_icecon.cdr`. WIP. - -Temporary code for simulating the sea ice CDR for comparison and demonstration purposes. - -The CDR algorithm is: - -* spatial interpolation on input Tbs. The NT and BT API for AMSR2 currently have - this implemented. -* Choose bootstrap unless nasateam is larger where bootstrap has ice. - -Eventually this code will be removed/migrated to the sea ice cdr project. This -project should be primarily responsible for generating concentration fields from -input Tbs. -""" -import datetime as dt -import sys -import traceback -from pathlib import Path -from typing import get_args - -import click -import numpy as np -import numpy.typing as npt -import pm_icecon.bt.compute_bt_ic as bt -import pm_icecon.bt.params.ausi_amsr2 as bt_amsr2_params -import pm_icecon.nt.compute_nt_ic as nt -import pm_icecon.nt.params.amsr2 as nt_amsr2_params -import xarray as xr -from loguru import logger -from pm_icecon._types import Hemisphere -from pm_icecon.cli.util import datetime_to_date -from pm_icecon.config.models.bt import BootstrapParams -from pm_icecon.constants import DEFAULT_FLAG_VALUES -from pm_icecon.fill_polehole import fill_pole_hole -from pm_icecon.interpolation import spatial_interp_tbs -from pm_icecon.land_spillover import apply_nt2_land_spillover -from pm_icecon.nt._types import NasateamGradientRatioThresholds -from pm_icecon.nt.tiepoints import NasateamTiePoints -from pm_icecon.util import date_range, standard_output_filename -from pm_tb_data.fetch.au_si import AU_SI_RESOLUTIONS, get_au_si_tbs - -from seaice_ecdr.constants import CDR_DATA_DIR -from seaice_ecdr.land_spillover import load_or_create_land90_conc, read_adj123_file -from seaice_ecdr.masks import psn_125_near_pole_hole_mask - - -def cdr( - date: dt.date, - tb_h19: npt.NDArray, - tb_v37: npt.NDArray, - tb_h37: npt.NDArray, - tb_v19: npt.NDArray, - tb_v22: npt.NDArray, - bt_params: BootstrapParams, - nt_tiepoints: NasateamTiePoints, - nt_gradient_thresholds: NasateamGradientRatioThresholds, - nt_invalid_ice_mask: npt.NDArray[np.bool_], - nt_minic: npt.NDArray, - nt_shoremap: npt.NDArray, - missing_flag_value, - use_only_nt2_spillover=True, -) -> npt.NDArray: - """Run the CDR algorithm.""" - # First, get bootstrap conc. - bt_tb_mask = bt.tb_data_mask( - tbs=( - tb_v37, - tb_h37, - tb_v19, - tb_v22, - ), - min_tb=bt_params.mintb, - max_tb=bt_params.maxtb, - ) - - season_params = bt._get_wx_params( - date=date, weather_filter_seasons=bt_params.weather_filter_seasons - ) - bt_weather_mask = bt.get_weather_mask( - v37=tb_v37, - h37=tb_h37, - v22=tb_v22, - v19=tb_v19, - land_mask=bt_params.land_mask, - tb_mask=bt_tb_mask, - ln1=bt_params.vh37_params.lnline, - date=date, - wintrc=season_params.wintrc, - wslope=season_params.wslope, - wxlimt=season_params.wxlimt, - ) - bt_conc = bt.bootstrap( - tb_v37=tb_v37, - tb_h37=tb_h37, - tb_v19=tb_v19, - params=bt_params, - tb_mask=bt_tb_mask, - weather_mask=bt_weather_mask, - ) - - # Next, get nasateam conc. Note that concentrations from nasateam may be - # >100%. - nt_pr_1919 = nt.compute_ratio(tb_v19, tb_h19) - nt_gr_3719 = nt.compute_ratio(tb_v37, tb_v19) - nt_conc = nt.calc_nasateam_conc( - pr_1919=nt_pr_1919, - gr_3719=nt_gr_3719, - tiepoints=nt_tiepoints, - ) - - # Now calculate CDR SIC - is_bt_seaice = (bt_conc > 0) & (bt_conc <= 100) - use_nt_values = (nt_conc > bt_conc) & is_bt_seaice - cdr_conc = bt_conc.copy() - cdr_conc[use_nt_values] = nt_conc[use_nt_values] - - # Apply masks - # Get Nasateam weather filter - nt_gr_2219 = nt.compute_ratio(tb_v22, tb_v19) - nt_weather_mask = nt.get_weather_filter_mask( - gr_2219=nt_gr_2219, - gr_3719=nt_gr_3719, - gr_2219_threshold=nt_gradient_thresholds["2219"], - gr_3719_threshold=nt_gradient_thresholds["3719"], - ) - # Apply weather filters and invalid ice masks - # TODO: can we just use a single invalid ice mask? - set_to_zero_sic = ( - nt_weather_mask - | bt_weather_mask - | nt_invalid_ice_mask - | bt_params.invalid_ice_mask - ) - cdr_conc[set_to_zero_sic] = 0 - - # Apply land spillover corrections - # TODO: eventually, we want each of these routines to return a e.g., delta - # that can be applied to the input concentration instead of returning a new - # conc. Then we would have a seprate algorithm for choosing how to apply - # multiple spillover deltas to a given conc field. - # TODO: The land spillover routines should be moved out of this code and - # into their own methods. Then, they can be called as the recipe requires - if use_only_nt2_spillover: - logger.info("Applying NT2 land spillover technique...") - # TODO: Use gridid to indicate the necessary information for - # the spillover algorithm. Array shape is too fragile. - if tb_h19.shape == (896, 608): - # NH - l90c = load_or_create_land90_conc( - gridid="psn12.5", - xdim=608, - ydim=896, - overwrite=False, - ) - adj123 = read_adj123_file( - gridid="psn12.5", - xdim=608, - ydim=896, - ) - cdr_conc = apply_nt2_land_spillover( - conc=cdr_conc, - adj123=adj123, - l90c=l90c, - ) - elif tb_h19.shape == (664, 632): - # SH - l90c = load_or_create_land90_conc( - gridid="pss12.5", - xdim=632, - ydim=664, - overwrite=False, - ) - adj123 = read_adj123_file( - gridid="pss12.5", - xdim=632, - ydim=664, - ) - cdr_conc = apply_nt2_land_spillover( - conc=cdr_conc, - adj123=adj123, - l90c=l90c, - ) - - else: - raise RuntimeError( - f"Could not determine hemisphere from tb shape: {tb_h19.shape}" - " while attempting to apply NT2 land spillover algorithm" - ) - else: - # nasateam first: - logger.info("Applying NASA TEAM land spillover technique...") - cdr_conc = nt.apply_nt_spillover( - conc=cdr_conc, - shoremap=nt_shoremap, - minic=nt_minic, - ) - - # then bootstrap: - logger.info("Applying Bootstrap land spillover technique...") - cdr_conc = bt.coastal_fix( - conc=cdr_conc, - missing_flag_value=missing_flag_value, - land_mask=bt_params.land_mask, - minic=bt_params.minic, - ) - - # Fill the NH pole hole - if cdr_conc.shape == (896, 608): - near_pole_hole_mask = psn_125_near_pole_hole_mask() - cdr_conc = fill_pole_hole( - conc=cdr_conc, near_pole_hole_mask=near_pole_hole_mask - ) - - # Apply land flag value and clamp max conc to 100. - # TODO: extract this func from nt and allow override of flag values - cdr_conc = nt._clamp_conc_and_set_flags( - shoremap=nt_shoremap, - conc=cdr_conc, - ) - - # Return CDR. - # TODO: return an xr dataset with variables containing the outputs of - # intermediate steps above. - return cdr_conc - - -def amsr2_cdr( - *, - date: dt.date, - hemisphere: Hemisphere, - resolution: AU_SI_RESOLUTIONS, -) -> xr.Dataset: - """Create a CDR-like concentration field from AMSR2 data.""" - # Get AMSR2 TBs - xr_tbs = get_au_si_tbs( - date=date, - hemisphere=hemisphere, - resolution=resolution, - ) - - bt_params = bt_amsr2_params.get_amsr2_params( - date=date, - hemisphere=hemisphere, - resolution=resolution, - ) - - nt_params = nt_amsr2_params.get_amsr2_params( - hemisphere=hemisphere, - resolution=resolution, - ) - - # finally, compute the CDR. - conc = cdr( - date=date, - tb_h19=spatial_interp_tbs(xr_tbs["h18"].data), - tb_v37=spatial_interp_tbs(xr_tbs["v36"].data), - tb_h37=spatial_interp_tbs(xr_tbs["h36"].data), - tb_v19=spatial_interp_tbs(xr_tbs["v18"].data), - tb_v22=spatial_interp_tbs(xr_tbs["v23"].data), - bt_params=bt_params, - nt_tiepoints=nt_params.tiepoints, - nt_gradient_thresholds=nt_params.gradient_thresholds, - # TODO: this is the same as the bootstrap mask! - nt_invalid_ice_mask=bt_params.invalid_ice_mask, - nt_minic=nt_params.minic, - nt_shoremap=nt_params.shoremap, - missing_flag_value=DEFAULT_FLAG_VALUES.missing, - # TODO: do we need the land flag value? Currently unused. - # land_flag_value=DEFAULT_FLAG_VALUES.land, - ) - - cdr_conc_ds = xr.Dataset({"conc": (("y", "x"), conc)}) - - return cdr_conc_ds - - -def make_cdr_netcdf( - *, - date: dt.date, - hemisphere: Hemisphere, - resolution: AU_SI_RESOLUTIONS, - output_dir: Path, -) -> None: - logger.info(f"Creating CDR for {date=}, {hemisphere=}, {resolution=}") - conc_ds = amsr2_cdr( - date=date, - hemisphere=hemisphere, - resolution=resolution, - ) - - output_fn = standard_output_filename( - hemisphere=hemisphere, - date=date, - sat="u2", - algorithm="cdr", - resolution=f"{resolution}km", - ) - output_path = output_dir / output_fn - conc_ds.to_netcdf( - output_path, - encoding={"conc": {"zlib": True}}, - ) - logger.info(f"Wrote AMSR2 CDR concentration field: {output_path}") - - -def create_cdr_for_date_range( - *, - hemisphere: Hemisphere, - start_date: dt.date, - end_date: dt.date, - resolution: AU_SI_RESOLUTIONS, - output_dir: Path, -) -> None: - for date in date_range(start_date=start_date, end_date=end_date): - try: - make_cdr_netcdf( - date=date, - hemisphere=hemisphere, - resolution=resolution, - output_dir=output_dir, - ) - except Exception: - logger.error( - f"Failed to create NetCDF for {hemisphere=}, {date=}, {resolution=}." - ) - err_filename = standard_output_filename( - hemisphere=hemisphere, - date=date, - sat="u2", - algorithm="cdr", - resolution=f"{resolution}km", - ) - err_filename += ".error" - logger.info(f"Writing error info to {err_filename}") - with open(output_dir / err_filename, "w") as f: - traceback.print_exc(file=f) - traceback.print_exc(file=sys.stdout) - - -@click.command(name="cdr") -@click.option( - "-d", - "--date", - required=True, - type=click.DateTime(formats=("%Y-%m-%d",)), - callback=datetime_to_date, -) -@click.option( - "-h", - "--hemisphere", - required=True, - type=click.Choice(get_args(Hemisphere)), -) -@click.option( - "-o", - "--output-dir", - required=True, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True, - path_type=Path, - ), -) -@click.option( - "-r", - "--resolution", - required=True, - type=click.Choice(get_args(AU_SI_RESOLUTIONS)), -) -def cli( - *, - date: dt.date, - hemisphere: Hemisphere, - output_dir: Path, - resolution: AU_SI_RESOLUTIONS, -) -> None: - """Run the CDR algorithm with AMSR2 data.""" - create_cdr_for_date_range( - start_date=date, - end_date=date, - hemisphere=hemisphere, - resolution=resolution, - output_dir=output_dir, - ) - - -if __name__ == "__main__": - # vvvv MODIFY THESE PARAMETERS AS NEEDED vvvv - start_date = dt.date(2012, 7, 2) - end_date = dt.date(2021, 2, 11) - resolution: AU_SI_RESOLUTIONS = "12" - output_dir = CDR_DATA_DIR - # ^^^^ MODIFY THESE PARAMETERS AS NEEDED ^^^^ - for hemisphere in get_args(Hemisphere): - create_cdr_for_date_range( - start_date=start_date, - end_date=end_date, - hemisphere=hemisphere, - resolution=resolution, - output_dir=output_dir, - ) diff --git a/seaice_ecdr/cli/__pycache__/__init__.cpython-310.pyc b/seaice_ecdr/cli/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 8f69f98f..00000000 Binary files a/seaice_ecdr/cli/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/seaice_ecdr/cli/__pycache__/entrypoint.cpython-310.pyc b/seaice_ecdr/cli/__pycache__/entrypoint.cpython-310.pyc deleted file mode 100644 index b8318575..00000000 Binary files a/seaice_ecdr/cli/__pycache__/entrypoint.cpython-310.pyc and /dev/null differ diff --git a/seaice_ecdr/cli/entrypoint.py b/seaice_ecdr/cli/entrypoint.py index f6326803..8c98946a 100644 --- a/seaice_ecdr/cli/entrypoint.py +++ b/seaice_ecdr/cli/entrypoint.py @@ -1,27 +1,17 @@ """entrypoint.py Contains click commands fo seaice_ecdr.""" import click -from pm_icecon.bt.cli import cli as bt_cli -from pm_icecon.nt.cli import cli as nt_cli -from seaice_ecdr.cdr_alg import cli as cdr_cli from seaice_ecdr.initial_daily_ecdr import cli as ecdr_cli @click.group() def cli(): - """Run the nasateam or bootstrap algorithm.""" + """Run the Sea Ice EDCDR.""" ... -cli.add_command(bt_cli) -cli.add_command(nt_cli) -cli.add_command(cdr_cli) cli.add_command(ecdr_cli) if __name__ == "__main__": - from pm_icecon.cli.entrypoint import cli - - # from seaice_ecdr.cli.entrypoint import cli_ecdr - cli() diff --git a/seaice_ecdr/compare/visualize.py b/seaice_ecdr/compare/visualize.py index 5e25bb90..214a233e 100644 --- a/seaice_ecdr/compare/visualize.py +++ b/seaice_ecdr/compare/visualize.py @@ -23,8 +23,8 @@ from pm_icecon.tests.regression import test_nt from pm_tb_data.fetch import au_si -from seaice_ecdr.cdr_alg import amsr2_cdr from seaice_ecdr.compare.ref_data import get_au_si_bt_conc, get_cdr, get_sea_ice_index +from seaice_ecdr.initial_daily_ecdr import compute_initial_daily_ecdr_dataset OUTPUT_DIR = Path("/tmp/diffs/") OUTPUT_DIR.mkdir(parents=True, exist_ok=True) @@ -420,7 +420,9 @@ def compare_cdr( cdr_ds = get_cdr(date=date, hemisphere=hemisphere, resolution=resolution) our_cdr_ds = _flip( - amsr2_cdr(date=date, hemisphere=hemisphere, resolution=resolution) + compute_initial_daily_ecdr_dataset( + date=date, hemisphere=hemisphere, resolution=resolution + ) ) do_comparisons( diff --git a/seaice_ecdr/initial_daily_ecdr.py b/seaice_ecdr/initial_daily_ecdr.py index 820e307a..e3e4d40f 100644 --- a/seaice_ecdr/initial_daily_ecdr.py +++ b/seaice_ecdr/initial_daily_ecdr.py @@ -1,13 +1,15 @@ """Create the initial daily ECDR file. -Initially based on pm_icecon's 'pm_cdr.py' routines +Notes: + +* `idecdr` is shorthand for == "Initial Daily ECDR" """ import datetime as dt import sys import traceback from pathlib import Path -from typing import TypedDict, cast, get_args +from typing import TypedDict, get_args import click import numpy as np @@ -820,16 +822,6 @@ def compute_initial_daily_ecdr_dataset( return ecdr_ide_ds -def amsr2_cdr( - *, - date: dt.date, - hemisphere: Hemisphere, - resolution: AU_SI_RESOLUTIONS, -): - """Obsolete reference to code that creates CDR using AMSR2.""" - raise RuntimeError("amsr2_cdr() is nowcompute_initial_daily_ecdr_dataset()") - - def make_cdr_netcdf( *, date: dt.date, @@ -839,7 +831,7 @@ def make_cdr_netcdf( ) -> None: """Create the cdr netCDF file.""" logger.info(f"Creating CDR for {date=}, {hemisphere=}, {resolution=}") - conc_ds = amsr2_cdr( + conc_ds = compute_initial_daily_ecdr_dataset( date=date, hemisphere=hemisphere, resolution=resolution, @@ -935,7 +927,12 @@ def cli( output_dir: Path, resolution: AU_SI_RESOLUTIONS, ) -> None: - """Run the initial daily ECDR algorithm with AMSR2 data.""" + """Run the initial daily ECDR algorithm with AMSR2 data. + + TODO: eventually we want to be able to specify: date, grid (grid includes + projection, resolution, and bounds), and TBtype (TB type includes source and + methodology for getting those TBs onto the grid) + """ create_idecdr_for_date_range( hemisphere=hemisphere, start_date=date, @@ -945,38 +942,3 @@ def cli( resolution=resolution, output_dir=output_dir, ) - - -def parse_cmdline_iedcdr_params(): - """Extract info from command line call of initial_daily_ecdr.py.""" - import sys - - print(f"cmdline args: {sys.argv}") - raise RuntimeError("in parse_cmdline_iedcdr_params") - - -if __name__ == "__main__": - # vvvv MODIFY THESE PARAMETERS AS NEEDED vvvv - start_date, end_date, gridid, tb_source, output_dir = parse_cmdline_iedcdr_params() - - if tb_source is None: - raise ValueError("tb_source should not be None") - - if gridid == "e2n12.5": - hemisphere = "north" - resolution = "12" - elif gridid == "e2s12.5": - hemisphere = "south" - resolution = "12" - else: - raise RuntimeError(f"Could not parse gridid: {gridid}") - - hemisphere = cast(Hemisphere, hemisphere) - resolution = cast(AU_SI_RESOLUTIONS, resolution) - create_idecdr_for_date_range( - hemisphere=hemisphere, - start_date=start_date, - end_date=end_date, - output_dir=output_dir, - resolution=resolution, - ) diff --git a/seaice_ecdr/tests/integration/gen_ide_sample.py b/seaice_ecdr/tests/integration/gen_ide_sample.py deleted file mode 100644 index 46810210..00000000 --- a/seaice_ecdr/tests/integration/gen_ide_sample.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Generate a sample ide file for use in development. - -./gen_ide_sample.py - -Modeled after code in the integration test case -""" - -import datetime as dt -import sys - -from loguru import logger - -from seaice_ecdr.initial_daily_ecdr import ( - compute_initial_daily_ecdr_dataset as compute_idecdr_ds, -) - -# Set the default minimum log notification to Warning -logger.remove(0) # Removes previous logger info -logger.add(sys.stderr, level="INFO") - - -def gen_sample_idecdr_dataset( - date, - hemisphere, - resolution, - sample_filename=None, -): - """Generate sample initial daily cdr file from seaice_ecdr repo.""" - if sample_filename is None: - sample_filename = ( - f"sample_idecdr_{hemisphere}_{resolution}_" - + f'{date.strftime("%Y%m%d")}.nc' - ) - """Set up sample data set using pm_icecon.""" - log_string = f""" - - Creating sample seaice_ecdr initial daily data set with: - date: {date} - hemisphere: {hemisphere} - resolution: {resolution} - - Output file name: - {sample_filename} - - """ - logger.info(log_string) - - ide_ds = compute_idecdr_ds( - date=date, - hemisphere=hemisphere, - resolution=resolution, - ) - ide_ds.to_netcdf(sample_filename) - logger.info( - f""" - - Wrote: {sample_filename} - - """ - ) - - -if __name__ == "__main__": - date = dt.datetime(2021, 4, 5).date() - hemisphere = "north" - resolution = "12" - - gen_sample_idecdr_dataset(date, hemisphere, resolution) diff --git a/seaice_ecdr/tests/integration/test_initial_daily_ecdr_generation.py b/seaice_ecdr/tests/integration/test_initial_daily_ecdr_generation.py index 49a5cc22..2ddf7320 100644 --- a/seaice_ecdr/tests/integration/test_initial_daily_ecdr_generation.py +++ b/seaice_ecdr/tests/integration/test_initial_daily_ecdr_generation.py @@ -1,23 +1,13 @@ -"""Verify no change in initial amsr2_cdr data sets. - -test_initial_daily_ecdr_generation.py - -Verify that the initial daily ecdr files are the same as -pm_icecon's initial amsr2_cdr results. - -""" +"""Tests for initial daily ECDR generation.""" import datetime as dt import sys from typing import Final -import numpy as np import pytest import xarray as xr from loguru import logger -from numpy.testing import assert_equal -from seaice_ecdr.cdr_alg import amsr2_cdr as pmi_amsr2_cdr from seaice_ecdr.initial_daily_ecdr import ( compute_initial_daily_ecdr_dataset as compute_idecdr_ds, ) @@ -61,97 +51,22 @@ def sample_idecdr_dataset_sh(): return ide_conc_ds -@pytest.fixture(scope="session") -def sample_pmicecon_dataset(): - """Set up sample data set using pm_icecon.""" - logger.info("testing: Creating sample pmicecon dataset") - - test_date = dt.datetime(2021, 4, 5).date() - test_hemisphere: Final = "north" - test_resolution: Final = "12" - - pmicecon_conc_ds = pmi_amsr2_cdr( - date=test_date, - hemisphere=test_hemisphere, - resolution=test_resolution, - ) - return pmicecon_conc_ds - - -@pytest.fixture(scope="session") -def sample_idecdr_dataset(): - """Set up sample data set using idecdr.""" - logger.info("testing: Creating sample idecdr dataset") - - test_date = dt.datetime(2021, 4, 5).date() - test_hemisphere: Final = "north" - test_resolution: Final = "12" - - ide_conc_ds = compute_idecdr_ds( - date=test_date, - hemisphere=test_hemisphere, - resolution=test_resolution, - ) - return ide_conc_ds - - -def test_testing_initial_daily_ecdr_generation(): - """Test that this integration test is being performed.""" - assert True - - -def test_pmicecon_fixture(sample_pmicecon_dataset): - """Test that pm_icecon yields a 'conc' field.""" - # assert type(sample_pmicecon_dataset) == type(xr.Dataset()) - assert isinstance(sample_pmicecon_dataset, type(xr.Dataset())) - - -def test_pmicecon_conc_generation(sample_pmicecon_dataset): - """Test that pm_icecon yields a 'conc' field.""" - # pmicecon_conc_ds = sample_pmicecon_dataset - - pmicecon_conc_varname = "conc" - assert ( - type(sample_pmicecon_dataset.variables[pmicecon_conc_varname]) - == xr.core.variable.Variable - ) - - -def test_seaice_idecdr_and_pmicecon_conc_identical( - sample_pmicecon_dataset, sample_idecdr_dataset -): - """Test that pm_icecon yields a 'conc' field.""" - pmicecon_conc_ds = sample_pmicecon_dataset - pmi_conc_field = np.array(pmicecon_conc_ds.variables["conc"]) - - ide_conc_ds = sample_idecdr_dataset - ide_conc_field = np.squeeze(np.array(ide_conc_ds.variables["conc"])) - - # We know that the original conc field has zeros where TBs were not - # available, so only check where idecdr is not nan - indexes_to_check = ~np.isnan(ide_conc_field) - assert_equal( - pmi_conc_field[indexes_to_check], - ide_conc_field[indexes_to_check], - ) - - def test_seaice_idecdr_can_output_to_netcdf( sample_idecdr_dataset_nh, sample_idecdr_dataset_sh, + tmp_path, ): """Test that xarray dataset can be saved to a netCDF file.""" - import os # NH - sample_output_filename_nh = "./sample_ecdr_nh.nc" - sample_idecdr_dataset_nh.to_netcdf(sample_output_filename_nh) - assert os.path.isfile(sample_output_filename_nh) + sample_output_filepath_nh = tmp_path / "sample_ecdr_nh.nc" + sample_idecdr_dataset_nh.to_netcdf(sample_output_filepath_nh) + assert sample_output_filepath_nh.is_file() # SH - sample_output_filename_sh = "./sample_ecdr_sh.nc" - sample_idecdr_dataset_sh.to_netcdf(sample_output_filename_sh) - assert os.path.isfile(sample_output_filename_sh) + sample_output_filepath_sh = tmp_path / "sample_ecdr_sh.nc" + sample_idecdr_dataset_sh.to_netcdf(sample_output_filepath_sh) + assert sample_output_filepath_sh.is_file() def test_seaice_idecdr_has_crs( @@ -168,7 +83,5 @@ def test_seaice_idecdr_is_Dataset( sample_idecdr_dataset_sh, ): """Test that pm_icecon yields a 'conc' field.""" - # assert type(sample_idecdr_dataset_nh) == type(xr.Dataset()) - # assert type(sample_idecdr_dataset_sh) == type(xr.Dataset()) assert isinstance(sample_idecdr_dataset_nh, type(xr.Dataset())) assert isinstance(sample_idecdr_dataset_sh, type(xr.Dataset()))