diff --git a/CHANGELOG.md b/CHANGELOG.md index f1741d0..80287f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,15 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) +## [0.2.5] + +### Update +* Updated workflows to hyp3lib v2.0.2, which uses the new Copernicus Dataspace Ecosystem API got download orbit files. +* Calls to `downloadSentinelOrbitFile` to specify the `esa_credentials` argument. + +### Added +* `check_esa_credentials` function to `__main__.py` to check for the existence of Dataspace credentials before processing begins. + ## [0.2.5] ### Fixed diff --git a/README.md b/README.md index 54b19e9..d92afa5 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,12 @@ We note all the input datasets are publicly available using a NASA Earthdata acc machine urs.earthdata.nasa.gov login password - ``` - The `username`/`password` are the appropriate Earthdata Login credentials that are used to access NASA data. This file is necessary for downloading the Sentinel-1 orbit files from the ASF DAAC. Additionally, the [`requests`](https://docs.python-requests.org/en/latest/) library automatically uses credentials stored in the `~/.netrc` for authentification when none are supplied. + machine dataspace.copernicus.eu + login + password + ``` + The first `username`/`password` pair are the appropriate Earthdata Login credentials that are used to access NASA data. The second pair are your credentials for the [Copernicus Data Space Ecosystem](https://dataspace.copernicus.eu). This file is necessary for downloading the Sentinel-1 files, and auxiliary data. Additionally, the [`requests`](https://docs.python-requests.org/en/latest/) library automatically uses credentials stored in the `~/.netrc` for authentification when none are supplied. ## Generate a GUNW @@ -112,6 +115,8 @@ isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20230125T140019_20230125T14004 ``` isce2_topsapp --username \ --password \ + --esa-username \ + --esa-password \ --reference-scenes S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 \ --secondary-scenes S1B_IW_SLC__1SDV_20210711T014922_20210711T014949_027740_034F80_859D \ S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 \ @@ -141,6 +146,8 @@ docker run -ti -v $PWD:/home/ops/topsapp_data topsapp_img \ S1B_IW_SLC__1SDV_20210711T015011_20210711T015038_027740_034F80_376C \ --username --password + --esa-username \ + --esa-password \ ``` where the `username`/`password` are the Earthdata credentials for accessing NASA data. We note the command line magic of the above is taken care of the `isce2_topsapp/etc/entrypoint.sh` (written by Joe Kennedy) which automatically runs certain bash commands on startup of the container, i.e. the run commands also calls the `isce2_topsapp` command line function as can be seen [here](isce2_topsapp/etc/entrypoint.sh). diff --git a/environment.yml b/environment.yml index 72d8e67..43161fe 100644 --- a/environment.yml +++ b/environment.yml @@ -15,7 +15,7 @@ dependencies: - fsspec - gdal - geopandas - - hyp3lib>=1.7 + - hyp3lib>=2,<3 - ipykernel - isce2==2.6.1 - jinja2 diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index 080cb19..7902c60 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -6,6 +6,7 @@ from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace from importlib.metadata import entry_points from pathlib import Path +from platform import system from typing import Optional from isce2_topsapp import (BurstParams, @@ -29,6 +30,9 @@ from isce2_topsapp.solid_earth_tides import update_gunw_with_solid_earth_tide +ESA_HOST = 'dataspace.copernicus.eu' + + def localize_data( reference_scenes: list, secondary_scenes: list, @@ -62,13 +66,13 @@ def localize_data( # For ionospheric correction computation if water_mask_flag: out_water_mask = download_water_mask( - out_slc["extent"], water_name='SWBD') + out_slc["extent"], water_name="SWBD") out_aux_cal = download_aux_cal() - out = {'reference_scenes': reference_scenes, - 'secondary_scenes': secondary_scenes, - 'frame_id': frame_id, + out = {"reference_scenes": reference_scenes, + "secondary_scenes": secondary_scenes, + "frame_id": frame_id, **out_slc, **out_dem, **out_water_mask, @@ -113,6 +117,35 @@ def ensure_earthdata_credentials( ) +def check_esa_credentials(username: Optional[str], password: Optional[str]) -> None: + netrc_name = '_netrc' if system().lower() == 'windows' else '.netrc' + netrc_file = Path.home() / netrc_name + + if (username is not None) != (password is not None): + raise ValueError('Both username and password arguments must be provided') + + if username is not None: + os.environ["ESA_USERNAME"] = username + os.environ["ESA_PASSWORD"] = password + return + + if "ESA_USERNAME" in os.environ and "ESA_PASSWORD" in os.environ: + return + + if netrc_file.exists(): + netrc_credentials = netrc.netrc(netrc_file) + if ESA_HOST in netrc_credentials.hosts: + os.environ["ESA_USERNAME"] = netrc_credentials.hosts[ESA_HOST][0] + os.environ["ESA_PASSWORD"] = netrc_credentials.hosts[ESA_HOST][2] + return + + raise ValueError( + "Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the " + "--esa-username and --esa-password options, " + "the ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file." + ) + + def true_false_string_argument(s: str) -> bool: s = s.lower() if s not in ("true", "false"): @@ -177,6 +210,7 @@ def gunw_slc(): # Validation ensure_earthdata_credentials(args.username, args.password) + check_esa_credentials(args.esa_username, args.esa_password) cli_params = vars(args).copy() [cli_params.pop(key) for key in ['username', 'password', 'bucket', 'bucket_prefix', 'dry_run']] topsapp_params_obj = topsappParams(**cli_params) @@ -276,6 +310,8 @@ def gunw_burst(): parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("--username") parser.add_argument("--password") + parser.add_argument("--esa-username") + parser.add_argument("--esa-password") parser.add_argument("--bucket") parser.add_argument("--bucket-prefix", default="") parser.add_argument("--dry-run", action="store_true") @@ -291,6 +327,7 @@ def gunw_burst(): args = parser.parse_args() ensure_earthdata_credentials(args.username, args.password) + check_esa_credentials(args.esa_username, args.esa_password) ref_obj, sec_obj = get_asf_slc_objects( [args.reference_scene, args.secondary_scene]) diff --git a/isce2_topsapp/localize_orbits.py b/isce2_topsapp/localize_orbits.py index a1929be..5a1e7e6 100644 --- a/isce2_topsapp/localize_orbits.py +++ b/isce2_topsapp/localize_orbits.py @@ -1,10 +1,13 @@ +import os from pathlib import Path import requests from hyp3lib import get_orb -def _spoof_orbit_download(scene, _, providers=('ESA', 'ASF'), orbit_types=('AUX_POEORB', 'AUX_RESORB')): +def _spoof_orbit_download( + scene, _, providers=('ESA', 'ASF'), orbit_types=('AUX_POEORB', 'AUX_RESORB'), esa_credentials=('name', 'password') +): for orbit_type in orbit_types: for provider in providers: try: @@ -17,10 +20,11 @@ def _spoof_orbit_download(scene, _, providers=('ESA', 'ASF'), orbit_types=('AUX_ return None, None -def download_orbits(reference_scenes: list, - secondary_scenes: list, - orbit_directory: str = None, - dry_run: bool = False) -> dict: +def download_orbits( + reference_scenes: list, secondary_scenes: list, orbit_directory: str = None, dry_run: bool = False +) -> dict: + esa_credentials = (os.environ['ESA_USERNAME'], os.environ['ESA_PASSWORD']) + orbit_directory = orbit_directory or 'orbits' orbit_dir = Path(orbit_directory) orbit_dir.mkdir(exist_ok=True) @@ -29,12 +33,12 @@ def download_orbits(reference_scenes: list, reference_orbits = [] for scene in reference_scenes: - orbit_file, _ = orbit_fetcher(scene, str(orbit_dir)) + orbit_file, _ = orbit_fetcher(scene, str(orbit_dir), esa_credentials=esa_credentials) reference_orbits.append(orbit_file) secondary_orbits = [] for scene in secondary_scenes: - orbit_file, _ = orbit_fetcher(scene, str(orbit_dir)) + orbit_file, _ = orbit_fetcher(scene, str(orbit_dir), esa_credentials=esa_credentials) secondary_orbits.append(orbit_file) reference_orbits = list(set(reference_orbits)) diff --git a/sample_run.sh b/sample_run.sh index ae37a11..c791418 100644 --- a/sample_run.sh +++ b/sample_run.sh @@ -1,7 +1,9 @@ isce2_topsapp --username \ --password \ + --esa-username \ + --esa-password \ --reference-scenes S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 \ --secondary-scenes S1B_IW_SLC__1SDV_20210711T014922_20210711T014949_027740_034F80_859D \ S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 \ S1B_IW_SLC__1SDV_20210711T015011_20210711T015038_027740_034F80_376C \ - > topsapp_img.out 2> topsapp_img.err \ No newline at end of file + > topsapp_img.out 2> topsapp_img.err diff --git a/setup.py b/setup.py index 014fe87..7c41711 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'dateparser', 'dem_stitcher>=2.1', 'geopandas', - 'hyp3lib>=1.7', + 'hyp3lib>=2,<3', 'jinja2', 'lxml', 'matplotlib', diff --git a/tests/localize-data.ipynb b/tests/localize-data.ipynb index ea39b87..dae8da4 100644 --- a/tests/localize-data.ipynb +++ b/tests/localize-data.ipynb @@ -116,7 +116,8 @@ "id": "mounted-title", "metadata": {}, "source": [ - "# Orbits" + "# Orbits\n", + "Make sure `ESA_USERNAME` and `ESA_PASSWORD` environment variables are set before running this." ] }, { @@ -249,9 +250,9 @@ ], "metadata": { "kernelspec": { - "display_name": "topsapp_env", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "topsapp_env" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -263,7 +264,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/tests/prepare-for-delivery.ipynb b/tests/prepare-for-delivery.ipynb index dd6839b..6de7ca6 100644 --- a/tests/prepare-for-delivery.ipynb +++ b/tests/prepare-for-delivery.ipynb @@ -485,9 +485,9 @@ ], "metadata": { "kernelspec": { - "display_name": "topsapp_env", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "topsapp_env" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -499,7 +499,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/tests/solid_earth_tides.ipynb b/tests/solid_earth_tides.ipynb index ddcb095..7fff3fe 100644 --- a/tests/solid_earth_tides.ipynb +++ b/tests/solid_earth_tides.ipynb @@ -24,6 +24,7 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "import os\n", "import xarray as xr\n", "import requests\n", "from hyp3lib import get_orb\n", @@ -89,6 +90,14 @@ "slc_id" ] }, + { + "cell_type": "markdown", + "id": "44fe249b-5691-4d96-9e11-1418ea9c2f4b", + "metadata": {}, + "source": [ + "**Make sure the `ESA_USERNAME` and `ESA_PASSWORD` environment variables are set before running this.**" + ] + }, { "cell_type": "code", "execution_count": 7, @@ -114,7 +123,8 @@ } ], "source": [ - "orb_file, _ = get_orb.downloadSentinelOrbitFile(slc_id)\n", + "esa_credentials = (os.environ['ESA_USERNAME'], os.environ['ESA_PASSWORD'])\n", + "orb_file, _ = get_orb.downloadSentinelOrbitFile(slc_id, esa_credentials=esa_credentials)\n", "orb_file" ] }, @@ -7727,9 +7737,9 @@ "hash": "83c8da8660bbe18150fdc09aeaa55e9731d119a6641346a233b76fde249768bb" }, "kernelspec": { - "display_name": "topsapp_env", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "topsapp_env" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -7741,7 +7751,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/tests/test_main.py b/tests/test_main.py index 973191d..d548f5c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,14 @@ +import os + import pytest -from isce2_topsapp.__main__ import ensure_earthdata_credentials, esd_threshold_argument, true_false_string_argument +from isce2_topsapp.__main__ import ( + ESA_HOST, + check_esa_credentials, + ensure_earthdata_credentials, + esd_threshold_argument, + true_false_string_argument, +) def test_main_check_earthdata_credentials_prefer_netrc(tmp_path, monkeypatch): @@ -73,6 +81,65 @@ def test_main_check_earthdata_credentials_env_variables(tmp_path, monkeypatch): assert netrc.read_text() == 'machine foobar.nasa.gov login fizz password baz' +def test_check_esa_credentials_params(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.setenv('ESA_USERNAME', 'env_username') + m.setenv('ESA_PASSWORD', 'env_password') + m.setenv('HOME', str(tmp_path)) + (tmp_path / '.netrc').write_text(f'machine {ESA_HOST} login netrc_username password netrc_password') + + check_esa_credentials('foo', 'bar') + + assert os.environ['ESA_USERNAME'] == 'foo' + assert os.environ['ESA_PASSWORD'] == 'bar' + + +def test_check_esa_credentials_env(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.setenv('ESA_USERNAME', 'foo') + m.setenv('ESA_PASSWORD', 'bar') + m.setenv('HOME', str(tmp_path)) + (tmp_path / '.netrc').write_text(f'machine {ESA_HOST} login netrc_username password netrc_password') + + check_esa_credentials(None, None) + + assert os.environ['ESA_USERNAME'] == 'foo' + assert os.environ['ESA_PASSWORD'] == 'bar' + + +def test_check_esa_credentials_netrc(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.delenv('ESA_USERNAME', raising=False) + m.delenv('ESA_PASSWORD', raising=False) + m.setenv('HOME', str(tmp_path)) + (tmp_path / '.netrc').write_text(f'machine {ESA_HOST} login foo password bar') + + check_esa_credentials(None, None) + + assert os.environ['ESA_USERNAME'] == 'foo' + assert os.environ['ESA_PASSWORD'] == 'bar' + + +def test_check_esa_credentials_missing(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.delenv('ESA_USERNAME', raising=False) + m.setenv('ESA_PASSWORD', 'env_password') + m.setenv('HOME', str(tmp_path)) + (tmp_path / '.netrc').write_text('') + msg = 'Please provide.*' + with pytest.raises(ValueError, match=msg): + check_esa_credentials(None, None) + + with monkeypatch.context() as m: + m.setenv('ESA_USERNAME', 'env_username') + m.delenv('ESA_PASSWORD', raising=False) + m.setenv('HOME', str(tmp_path)) + (tmp_path / '.netrc').write_text('') + msg = 'Please provide.*' + with pytest.raises(ValueError, match=msg): + check_esa_credentials(None, None) + + def test_true_false_string_argument(): assert true_false_string_argument('true') is True assert true_false_string_argument('TRUE') is True @@ -90,11 +157,11 @@ def test_true_false_string_argument(): def test_esd_threshold_argument(): - assert esd_threshold_argument('-1') == -1. - assert esd_threshold_argument('0') == 0. + assert esd_threshold_argument('-1') == -1.0 + assert esd_threshold_argument('0') == 0.0 assert esd_threshold_argument('.25') == 0.25 assert esd_threshold_argument('0.5') == 0.5 - assert esd_threshold_argument('1') == 1. + assert esd_threshold_argument('1') == 1.0 with pytest.raises(ValueError): esd_threshold_argument('-1.1') diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index 7c3c434..33b7c23 100644 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize('notebook_name', notebooks) -def test_notebooks(notebook_name): +def test_notebooks(notebook_name, monkeypatch): test_dir = Path(__file__).parents[0].absolute() @@ -18,6 +18,8 @@ def test_notebooks(notebook_name): out_notebook = out_dir / f'{notebook_name}' + monkeypatch.setenv('ESA_USERNAME', 'foo') + monkeypatch.setenv('ESA_PASSWORD', 'bar') pm.execute_notebook(test_dir / f'{notebook_name}', output_path=out_notebook ) diff --git a/tests/test_set.py b/tests/test_set.py index 2d6ad3d..15566b1 100644 --- a/tests/test_set.py +++ b/tests/test_set.py @@ -159,7 +159,7 @@ def test_magnitude_of_set_with_variable_timing(acq_type: str, orbit_files_for_se with xr.open_dataset(gunw_path_for_set_2, group=group) as ds: slc_id = ds['L1InputGranules'].values[0] - orb_file, _ = get_orb.downloadSentinelOrbitFile(slc_id) + orb_file, _ = get_orb.downloadSentinelOrbitFile(slc_id, esa_credentials=('username', 'password')) ``` """ for gunw_path_for_set, orbit_dict in zip(gunw_paths_for_set, orbit_files_for_set):