From e3895fa67466758b2ca35778afd560b3663f05d8 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 12:20:08 -0800 Subject: [PATCH 01/19] add tile-mate --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index f41c51b..ec7302c 100644 --- a/environment.yml +++ b/environment.yml @@ -40,6 +40,7 @@ dependencies: - scipy<1.10 - setuptools - setuptools_scm + - tile_mate - shapely - tqdm - dem_stitcher>=2.4.0 From 767c7e42218b851488ad93e51871f87aa3f19bef Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 12:20:32 -0800 Subject: [PATCH 02/19] use esa landcover for iono water mask and browse --- isce2_topsapp/__main__.py | 2 +- isce2_topsapp/delivery_prep.py | 4 +- isce2_topsapp/localize_mask.py | 19 ++++-- isce2_topsapp/water_mask.py | 110 +++---------------------------- tests/prepare-for-delivery.ipynb | 107 +----------------------------- 5 files changed, 29 insertions(+), 213 deletions(-) diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index aa43569..94551ac 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -66,7 +66,7 @@ def localize_data( # For ionospheric correction computation if water_mask_flag: out_water_mask = download_water_mask( - out_slc["extent"], water_name="SWBD") + out_slc["extent"], water_mask_name="esa_world_cover_2021_10m") out_aux_cal = download_aux_cal() diff --git a/isce2_topsapp/delivery_prep.py b/isce2_topsapp/delivery_prep.py index 4b925ee..0c55269 100644 --- a/isce2_topsapp/delivery_prep.py +++ b/isce2_topsapp/delivery_prep.py @@ -9,7 +9,7 @@ from matplotlib import cm from isce2_topsapp.packaging import DATASET_VERSION -from isce2_topsapp.water_mask import get_water_mask_raster +from isce2_topsapp.water_mask import get_water_mask_raster_for_browse_image TEMPLATE_DIR = (Path(__file__).parent/'templates').absolute() SCHEMA_PATH = TEMPLATE_DIR/'daac_ingest_schema.json' @@ -94,7 +94,7 @@ def get_wrapped_ifg(nc_path: Path) -> np.ndarray: unw, _ = open_science_grid(nc_path, 'unwrappedPhase') mask_cc = get_connected_component_mask(cc) - mask_water = get_water_mask_raster(profile) + mask_water = get_water_mask_raster_for_browse_image(profile) mask = mask_cc | mask_water wrapped = np.zeros(mask.shape) diff --git a/isce2_topsapp/localize_mask.py b/isce2_topsapp/localize_mask.py index 0fbfa4e..3b6faf2 100644 --- a/isce2_topsapp/localize_mask.py +++ b/isce2_topsapp/localize_mask.py @@ -1,12 +1,14 @@ from pathlib import Path import numpy as np +import rasterio from isce.components.isceobj.Alos2Proc.runDownloadDem import download_wbd from shapely.geometry import box +from tile_mate import get_raster_from_tiles def download_water_mask( - extent: list, water_name: str = "SWBD", buffer: float = 0.1 + extent: list, water_mask_name: str = "esa_world_cover_2021_10m", buffer: float = 0.1 ) -> dict: output_dir = Path(".").absolute() @@ -19,7 +21,15 @@ def download_water_mask( np.ceil(extent_buffered[3]), ] - if water_name == "SWBD": + if water_mask_name == 'esa_world_cover_2021_10m': + X, p = get_raster_from_tiles(extent_buffered, tile_shortname='esa_world_cover_2021') + mask = (X == 80).astype(np.uint8) + mask[mask.astype(bool)] = 255 + mask_filename = 'water_mask_derived_from_esa_world_cover_2021_10m' + with rasterio.open(mask_filename, 'w', **p) as ds: + ds.write(X) + + elif water_mask_name == "SWBD": # Download SRTM-SWDB water mask # Water mask dataset extent # Latitude S55 - N60 @@ -38,8 +48,7 @@ def download_water_mask( 'Skip downloading water mask!!') mask_filename = '' - elif water_name == "GSHHS": - # from water_mask import get_water_mask_raster - raise NotImplementedError("TODO, GSHHS not yet available") + else: + raise NotImplementedError("Water mask not available") return {"water_mask": mask_filename} diff --git a/isce2_topsapp/water_mask.py b/isce2_topsapp/water_mask.py index 4bb4e33..2cec0bb 100644 --- a/isce2_topsapp/water_mask.py +++ b/isce2_topsapp/water_mask.py @@ -1,94 +1,10 @@ -from typing import Union - -import geopandas as gpd import numpy as np -import pandas as pd -from rasterio import features +from dem_stitcher.rio_tools import reproject_arr_to_match_profile from rasterio.transform import array_bounds +from tile_mate import get_raster_from_tiles -def rasterize_shapes_to_array(shapes: list, - attributes: list, - profile: dict, - all_touched: bool = False, - dtype: str = 'float32', - fill_value: Union[float, int] = 0) -> np.ndarray: - """ - Takes a list of geometries and attributes to create an array. For example, - `shapes = df.geometry` and `attributes = df.label`, where df is a geopandas - GeoDataFrame. We note the array is initialized as array of zeros. - - Parameters - ---------- - shapes : list - List of Shapely geometries. - attributes : list - List of attributes corresponding to shapes. - profile : dict - Rasterio profile in which shapes will be projected into, importantly - the transform and dimensions specified. - all_touched : bool - Whether factionally covered pixels are written with specific value or - ignored. See `rasterio.features.rasterize`. - dtype : str - The initial array is np.zeros and dtype can be specified as a numpy - dtype or appropriate string. - Returns - ------- - np.ndarray: - The output array determined with profile. - """ - out_arr = np.full((profile['height'], profile['width']), - fill_value=fill_value, - dtype=dtype) - - # this is where we create a generator of geom, value pairs to use in - # rasterizing - shapes = [(geom, value) for geom, value in zip(shapes, attributes)] - burned = features.rasterize(shapes=shapes, - out=out_arr, - transform=profile['transform'], - all_touched=all_touched) - - return burned - - -def get_water_mask_dataframe(bounds: list) -> gpd.GeoDataFrame: - mask_location = ('/vsicurl/https://asf-dem-west.s3.us-west-2.amazonaws.co' - 'm/WATER_MASK/GSHHG/GSHHS_shp/f/GSHHS_f_L1.shp') - - if (bounds[0] < -200) or (bounds[1] > 200): - raise ValueError('The bounds need to be within -200 and 200') - - # If crosses over the dateline - if (bounds[0] < -180) or (bounds[2] > 180): - bounds_e = list(bounds) - bounds_w = list(bounds) - - # Grab both sides of dateline and then translate back - if (bounds[0] < -180): - bounds_e[0] += 360 - bounds_e[2] += 360 - df_e = gpd.read_file(mask_location, bbox=bounds_e) - df_w = gpd.read_file(mask_location, bbox=bounds_w) - df_e.geometry = df_e.geometry.translate(xoff=-360) - - else: # bounds[2] > 180 - bounds_w[0] -= 360 - bounds_w[2] -= 360 - df_e = gpd.read_file(mask_location, bbox=bounds_e) - df_w = gpd.read_file(mask_location, bbox=bounds_w) - df_w.geometry = df_w.geometry.translate(xoff=360) - - df = pd.concat([df_e, df_w], axis=0) - - else: - df = gpd.read_file(mask_location, bbox=bounds) - - return df - - -def get_water_mask_raster(profile: dict) -> np.ndarray: +def get_water_mask_raster_for_browse_image(profile: dict) -> np.ndarray: """ Water mask where True indicates water. Uses GSHHS_f_L1 with 3km hosted by ASF. @@ -103,18 +19,10 @@ def get_water_mask_raster(profile: dict) -> np.ndarray: np.ndarray Water mask (boolean) with True indicated water """ - bounds = array_bounds(profile['height'], - profile['width'], - profile['transform']) - crs = profile['crs'] - df = get_water_mask_dataframe(bounds).to_crs(crs) + extent = array_bounds(profile["height"], profile["width"], profile["transform"]) - geometries = df.geometry - land_values = [0] * df.shape[0] - X = rasterize_shapes_to_array(geometries, - land_values, - profile, - fill_value=1, - dtype='uint8', - all_touched=False) - return X.astype(bool) + X_esa, p_esa = get_raster_from_tiles(extent, tile_shortname="esa_world_cover_2021") + X_esa_r, _ = reproject_arr_to_match_profile(X_esa, p_esa, profile, resampling='nearest') + mask = (X_esa_r == 80).astype(bool) + mask = mask[0, ...] + return mask diff --git a/tests/prepare-for-delivery.ipynb b/tests/prepare-for-delivery.ipynb index 6de7ca6..88d0b17 100644 --- a/tests/prepare-for-delivery.ipynb +++ b/tests/prepare-for-delivery.ipynb @@ -18,7 +18,7 @@ "import json\n", "import isce2_topsapp\n", "from isce2_topsapp.delivery_prep import prepare_for_delivery\n", - "from isce2_topsapp.water_mask import get_water_mask_dataframe, get_water_mask_raster\n", + "from isce2_topsapp.water_mask import get_water_mask_raster_for_browse_image\n", "from isce2_topsapp.delivery_prep import get_dataset_schema\n", "import rasterio\n", "import matplotlib.pyplot as plt\n", @@ -363,7 +363,7 @@ "source": [ "%%time\n", "\n", - "X = get_water_mask_raster(p)" + "X = get_water_mask_raster_for_browse_image(p)" ] }, { @@ -380,107 +380,6 @@ "source": [ "plt.imshow(X)" ] - }, - { - "cell_type": "markdown", - "id": "dff5cc91", - "metadata": {}, - "source": [ - "Get a profile over the dateline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f82ca5c", - "metadata": { - "ExecuteTime": { - "end_time": "2022-01-14T02:35:11.957495Z", - "start_time": "2022-01-14T02:35:11.953663Z" - } - }, - "outputs": [], - "source": [ - "t = p['transform']\n", - "t2 = t.translation(xoff=-61, yoff=31) * t\n", - "t2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "406f8941", - "metadata": { - "ExecuteTime": { - "end_time": "2022-01-14T02:38:07.686730Z", - "start_time": "2022-01-14T02:35:11.959253Z" - } - }, - "outputs": [], - "source": [ - "%%time\n", - "\n", - "p2 = p.copy()\n", - "p2['transform'] = t2\n", - "X2 = get_water_mask_raster(p2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b22ecb17", - "metadata": { - "ExecuteTime": { - "end_time": "2022-01-14T02:38:08.098353Z", - "start_time": "2022-01-14T02:38:07.688128Z" - } - }, - "outputs": [], - "source": [ - "plt.imshow(X2)" - ] - }, - { - "cell_type": "markdown", - "id": "0ee5142d", - "metadata": {}, - "source": [ - "## Read Vectorfile\n", - "\n", - "Again over the dateline in the +180 degree region." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "89914a9b", - "metadata": { - "ExecuteTime": { - "end_time": "2022-01-14T02:41:23.420704Z", - "start_time": "2022-01-14T02:38:08.099834Z" - } - }, - "outputs": [], - "source": [ - "bounds = [175, 64, 185, 66]\n", - "\n", - "df = get_water_mask_dataframe(bounds)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60784b3d", - "metadata": { - "ExecuteTime": { - "end_time": "2022-01-14T02:41:23.671304Z", - "start_time": "2022-01-14T02:41:23.422084Z" - } - }, - "outputs": [], - "source": [ - "df.plot()" - ] } ], "metadata": { @@ -499,7 +398,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.9.16" } }, "nbformat": 4, From d6c1a855b3dd50ac5027622b476570a77b3464b3 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 12:22:24 -0800 Subject: [PATCH 03/19] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a43fe6..1a1d12c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.3] +* Uses ESA World Cover 2021 Permanent Water Class for ionosphere water mask and brose imagery water mask +* Finalizes global attributes:.... + ## [0.3.2] ### Changed From af14deb051aca4d5bb1771b8b2bd349cbc07f838 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 13:46:09 -0800 Subject: [PATCH 04/19] update env for pip install --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index ec7302c..83114ab 100644 --- a/environment.yml +++ b/environment.yml @@ -40,8 +40,9 @@ dependencies: - scipy<1.10 - setuptools - setuptools_scm - - tile_mate - shapely - tqdm - dem_stitcher>=2.4.0 - aiohttp # only needed for manifest and swath download + - pip: + - tile-mate From 2b1cdd325f8aa33a1b15017820f803eab472eed0 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 15:20:52 -0800 Subject: [PATCH 05/19] baseline reading --- isce2_topsapp/packaging.py | 20 ++++++++++++++++++ tests/conftest.py | 6 ++++++ tests/test_packaging.py | 42 ++++++++++++++++++++++++++++++++------ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index a4045d7..8bdb02c 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -10,6 +10,7 @@ import numpy as np import rasterio from dateparser import parse +from lxml import etree import isce2_topsapp from isce2_topsapp.packaging_utils.additional_layers import add_2d_layer @@ -36,6 +37,25 @@ """ +def read_baselines(tops_proc_xml: str) -> dict: + with open(tops_proc_xml) as f: + xml_str = f.read() + # :_ are not properly formatted tags + xml_str = xml_str.replace(':_', '') + root = etree.fromstring(xml_str) + + element_path = ".//baseline/" + elements = root.findall(element_path) + + tags = [e.tag for e in elements] + vals = [float(e.text) for e in elements] + parallel_baselines = [vals[k] for (k, val) in enumerate(vals) if 'Bpar' in tags[k]] + perpendicular_baselines = [vals[k] for (k, val) in enumerate(vals) if 'Bperp' in tags[k]] + + return {'parallel_baselines': parallel_baselines, + 'perpendicular_baselines': perpendicular_baselines} + + def update_gunw_internal_version_attribute(nc_path: Path, new_version='1c'): with h5py.File(nc_path, mode='a') as file: # Deleting attribute removes the type information so there is no diff --git a/tests/conftest.py b/tests/conftest.py index 4b40466..801d3f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,3 +41,9 @@ def get_overlapping_orbits_for_set_test() -> list[Path]: p2 = test_dir / 'set_test_data' / 'S1A_OPER_AUX_POEORB_OPOD_20230706T080750_V20230615T225942_20230617T005942.EOF' data = [p1, p2] return data + + +@pytest.fixture(scope='session') +def tops_proc_xml_path() -> list[Path]: + """Is alignd with orbit_files_for_set""" + return test_dir / 'test_data' / 'topsProc.xml' diff --git a/tests/test_packaging.py b/tests/test_packaging.py index c2fa114..4774642 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,16 +1,46 @@ import json from pathlib import Path -from isce2_topsapp.packaging import DATASET_VERSION, get_gunw_id +from isce2_topsapp.packaging import DATASET_VERSION, get_gunw_id, read_baselines test_dir = Path(__file__).parent def test_gunw_id_generation_crossing_dateline(): - sample_json_path = test_dir / 'midnight_crossing_metadata.json' + sample_json_path = test_dir / "midnight_crossing_metadata.json" metadata = json.load(open(sample_json_path)) - gunw_id = get_gunw_id(metadata['reference_properties'], - metadata['secondary_properties'], - metadata['extent']) + gunw_id = get_gunw_id( + metadata["reference_properties"], + metadata["secondary_properties"], + metadata["extent"], + ) version_str = DATASET_VERSION.replace(".", "_") - assert gunw_id == f'S1-GUNW-D-R-048-tops-20230106_20221214-235959-00090E_00040N-PP-c254-v{version_str}' + assert ( + gunw_id + == f"S1-GUNW-D-R-048-tops-20230106_20221214-235959-00090E_00040N-PP-c254-v{version_str}" + ) + + +def test_extract_baselines(tops_proc_xml_path): + baseline_dict = read_baselines(tops_proc_xml_path) + + expected_output = { + "parallel_baselines": [ + 69.3390287211759, + 69.2260273326017, + 80.19268495122755, + 80.10734013261386, + 88.90972511062476, + 88.87192713792777, + ], + "perpendicular_baselines": [ + -136.44791837773963, + -136.90239009351285, + -130.29490212547398, + -130.8679894611411, + -124.43579341000144, + -125.01252133117457, + ], + } + + assert baseline_dict == expected_output From 9261e3a27bbfcc26e3f29325c5b191c430047e21 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 15:21:30 -0800 Subject: [PATCH 06/19] add test data --- .gitignore | 1 + tests/test_data/topsProc.xml | 162 +++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 tests/test_data/topsProc.xml diff --git a/.gitignore b/.gitignore index 202a92f..40f482e 100644 --- a/.gitignore +++ b/.gitignore @@ -119,6 +119,7 @@ tests/**/*.json !tests/test_data/ref_metadata.xml !tests/test_data/sec_manifest.xml !tests/test_data/sec_metadata.xml +!tests/test_data/topsProc.xml # Test for overlapping orbits !tests/set_test_data/S1A_OPER_AUX_POEORB_OPOD_20230705T080713_V20230614T225942_20230616T005942.EOF diff --git a/tests/test_data/topsProc.xml b/tests/test_data/topsProc.xml new file mode 100644 index 0000000..5dd5b95 --- /dev/null +++ b/tests/test_data/topsProc.xml @@ -0,0 +1,162 @@ + + + Release: 2.6.1, svn-, 20220811. Current svn- + + + [1, 2, 3] + + + + TOPS + None + Sentinel-1 + S1A + Airbus Defence and Space-Toulouse, France + Sentinel-1 IPF + 003.40 + 2022-02-12 22:15:44.741918 + 6 + topsswathslc + topsswathslc_name + + 21800 + 1499 + 5 + 25660 + 1510 + 6 + 24742 + 1515 + 6 + + + + TOPS + None + Sentinel-1 + S1A + Airbus Defence and Space-Toulouse, France + Sentinel-1 IPF + 003.40 + 2022-01-31 22:15:44.755474 + 6 + topsswathslc + topsswathslc_name + + 21800 + 1499 + 5 + 25660 + 1510 + 7 + 24742 + 1515 + 6 + + + 5 + 0 + 5 + 5 + 0 + 5 + 5 + 69.3390287211759 + -136.44791837773963 + 69.2260273326017 + -136.90239009351285 + 6 + 0 + 6 + 7 + 1 + 7 + 6 + 80.19268495122755 + -130.29490212547398 + 80.10734013261386 + -130.8679894611411 + 6 + 0 + 6 + 6 + 0 + 6 + 6 + 88.90972511062476 + -124.43579341000144 + 88.87192713792777 + -125.01252133117457 + + + /mnt/leffe-data2/cmarshak/dense_offsets_at_30m/full_res.dem.wgs84 + 5 + 1499 + 21800 + [47.46490837551095, 47.80056562813403, -70.95362214843759, -69.71372428980712] + 1499 + 21800 + [47.62988124409519, 47.96609286088484, -71.00217386970365, -69.76049483053818] + 1499 + 21800 + [47.795182046453796, 48.13143418674786, -71.05147406176378, -69.80376109115082] + 1499 + 21800 + [47.960232892527905, 48.29652702467832, -71.11185917636975, -69.84696567356822] + 1499 + 21800 + [48.124135103173, 48.46196226820751, -71.16612691819715, -69.89396922422401] + 6 + 1510 + 25660 + [47.49592910870538, 47.82862347158341, -69.83358735171403, -68.53426089547584] + 1510 + 25660 + [47.66060119802951, 47.99361900440074, -69.8780548687409, -68.57292100362392] + 1510 + 25660 + [47.82595599875509, 48.158885164490016, -69.91910888619861, -68.6143287522271] + 1510 + 25660 + [47.991733649348454, 48.32388389560181, -69.96614831609605, -68.65744895068374] + 1510 + 25660 + [48.15637707412191, 48.48866528865243, -70.01061776410393, -68.699294492612] + 1510 + 25660 + [48.321820191510184, 48.65426438288203, -70.05291385077454, -68.74378856869492] + 6 + 1515 + 24742 + [47.5251519870918, 47.82998133471941, -68.63539205825286, -67.49525521541409] + 1515 + 24742 + [47.69052536123378, 47.995278087498434, -68.67826476918917, -67.5310973243103] + 1515 + 24742 + [47.85572817723, 48.16061505328018, -68.72000641153207, -67.56898414835307] + 1515 + 24742 + [48.02097426046712, 48.32577880426323, -68.76105207562242, -67.60513947448277] + 1515 + 24742 + [48.186165346694835, 48.490688768587795, -68.80701719170645, -67.64371479796777] + 1515 + 24742 + [48.35057886469764, 48.65556983744244, -68.84893080015773, -67.6824507222432] + [47.46490837551095, 48.65556983744244, -71.16612691819715, -67.49525521541409] + + + /mnt/leffe-data2/cmarshak/dense_offsets_at_30m/low_res.dem.wgs84 + merged/filt_dense_offsets.bil + merged/filt_dense_offsets.bil.geo + 2128 + 271 + 7 + 3 + 47.460653524 + 48.668710046 + -71.180932678 + -67.38342964 + + From 886c5e61bf4ed97dc321434e37bd8ebdab1e14aa Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 17:51:19 -0800 Subject: [PATCH 07/19] mean layers --- isce2_topsapp/packaging.py | 375 +++++++++++++++++++++---------------- tests/test_packaging.py | 25 ++- 2 files changed, 233 insertions(+), 167 deletions(-) diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index 8bdb02c..5139614 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -15,12 +15,15 @@ import isce2_topsapp from isce2_topsapp.packaging_utils.additional_layers import add_2d_layer from isce2_topsapp.packaging_utils.ionosphere import ( - format_iono_burst_ramps, format_ionosphere_for_gunw) + format_iono_burst_ramps, + format_ionosphere_for_gunw, +) from isce2_topsapp.templates import read_netcdf_packaging_template +from isce2_topsapp.water_mask import get_water_mask_raster_for_browse_image -DATASET_VERSION = '3.0.0' -STANDARD_PROD_PREFIX = 'S1-GUNW' -CUSTOM_PROD_PREFIX = 'S1-GUNW_CUSTOM' +DATASET_VERSION = "3.0.0" +STANDARD_PROD_PREFIX = "S1-GUNW" +CUSTOM_PROD_PREFIX = "S1-GUNW_CUSTOM" """Warning: the packaging scripts were written as command line scripts and @@ -36,12 +39,19 @@ was changed during runtime of `F`. """ +# The filename in the ISCE2 merged folder +LAYER2PATH = { + "incidence_angle": {"file_name": "los.rdr.geo", "band": 1}, + "azimuth_angle": {"file_name": "los.rdr.geo", "band": 2}, + "filtered_coherence": {"file_name": "phsig.cor.geo", "band": 1}, +} + def read_baselines(tops_proc_xml: str) -> dict: with open(tops_proc_xml) as f: xml_str = f.read() # :_ are not properly formatted tags - xml_str = xml_str.replace(':_', '') + xml_str = xml_str.replace(":_", "") root = etree.fromstring(xml_str) element_path = ".//baseline/" @@ -49,47 +59,55 @@ def read_baselines(tops_proc_xml: str) -> dict: tags = [e.tag for e in elements] vals = [float(e.text) for e in elements] - parallel_baselines = [vals[k] for (k, val) in enumerate(vals) if 'Bpar' in tags[k]] - perpendicular_baselines = [vals[k] for (k, val) in enumerate(vals) if 'Bperp' in tags[k]] + parallel_baselines = [vals[k] for (k, val) in enumerate(vals) if "Bpar" in tags[k]] + perpendicular_baselines = [ + vals[k] for (k, val) in enumerate(vals) if "Bperp" in tags[k] + ] + + return { + "parallel_baselines": parallel_baselines, + "perpendicular_baselines": perpendicular_baselines, + } - return {'parallel_baselines': parallel_baselines, - 'perpendicular_baselines': perpendicular_baselines} +def get_mean_baseline_data(tops_proc_xml: str) -> dict: + baseline_data = read_baselines(tops_proc_xml) + mean_baseline_data = {key: np.mean(val) for (key, val) in baseline_data.items()} + return mean_baseline_data -def update_gunw_internal_version_attribute(nc_path: Path, new_version='1c'): - with h5py.File(nc_path, mode='a') as file: + +def update_gunw_internal_version_attribute(nc_path: Path, new_version="1c"): + with h5py.File(nc_path, mode="a") as file: # Deleting attribute removes the type information so there is no # truncation and a simple string can be written. - if 'version' in file.attrs: - del file.attrs['version'] - file.attrs['version'] = new_version + if "version" in file.attrs: + del file.attrs["version"] + file.attrs["version"] = new_version def get_gunw_hash_id(reference_ids: list, secondary_ids: list) -> str: - all_ids = json.dumps([' '.join(sorted(reference_ids)), - ' '.join(sorted(secondary_ids)) - ]).encode('utf8') + all_ids = json.dumps( + [" ".join(sorted(reference_ids)), " ".join(sorted(secondary_ids))] + ).encode("utf8") hash_id = hashlib.md5(all_ids).hexdigest() return hash_id def get_geo_str(extent: list) -> str: - lon, lat = extent[:2] - lon_dir = 'W' if lon <= 0 else 'E' - lat_dir = 'N' if lat >= 0 else 'S' + lon_dir = "W" if lon <= 0 else "E" + lat_dir = "N" if lat >= 0 else "S" lon_north, lat_north = abs(round(lon)), abs(round(lat)) - lon_str = f'{lon_north:05d}{lon_dir}' - lat_str = f'{lat_north:05d}{lat_dir}' - return f'{lon_str}_{lat_str}' + lon_str = f"{lon_north:05d}{lon_dir}" + lat_str = f"{lat_north:05d}{lat_dir}" + return f"{lon_str}_{lat_str}" def get_center_time(properties: list) -> datetime.datetime: - - start_times = sorted([parse(props['startTime']) for props in properties]) - stop_times = sorted([parse(props['stopTime']) for props in properties]) + start_times = sorted([parse(props["startTime"]) for props in properties]) + stop_times = sorted([parse(props["stopTime"]) for props in properties]) start_time = start_times[0] stop_time = stop_times[-1] @@ -98,64 +116,66 @@ def get_center_time(properties: list) -> datetime.datetime: return center_datetime -def get_gunw_id(reference_properties: list, - secondary_properties: list, - extent: list, - standard_product: bool = True, - ) -> str: - +def get_gunw_id( + reference_properties: list, + secondary_properties: list, + extent: list, + standard_product: bool = True, +) -> str: # asc_or_desc: will be "A" or "D" - flight_direction = reference_properties[0]['flightDirection'] + flight_direction = reference_properties[0]["flightDirection"] asc_or_desc = flight_direction[0] # path or track number; needs to be padded so it is 3 digits # e.g. 064 or 128 - track_num = int(reference_properties[0]['pathNumber']) - track = f'{track_num:03}' + track_num = int(reference_properties[0]["pathNumber"]) + track = f"{track_num:03}" # Center Datetimes ref_center_datetime = get_center_time(reference_properties) sec_center_datetime = get_center_time(secondary_properties) # Center Time string - ref_center_time_str = ref_center_datetime.strftime('%H%M%S') + ref_center_time_str = ref_center_datetime.strftime("%H%M%S") - reference_date_str = ref_center_datetime.strftime('%Y%m%d') - secondary_date_str = sec_center_datetime.strftime('%Y%m%d') + reference_date_str = ref_center_datetime.strftime("%Y%m%d") + secondary_date_str = sec_center_datetime.strftime("%Y%m%d") # date pair - date_pair = f'{reference_date_str}_{secondary_date_str}' + date_pair = f"{reference_date_str}_{secondary_date_str}" # Geo string geo_str = get_geo_str(extent) # hash_id - reference_ids = [p['sceneName'] for p in reference_properties] - secondary_ids = [p['sceneName'] for p in secondary_properties] + reference_ids = [p["sceneName"] for p in reference_properties] + secondary_ids = [p["sceneName"] for p in secondary_properties] ifg_hash = get_gunw_hash_id(reference_ids, secondary_ids) ifg_hash_trunc = ifg_hash[:4] # version - version = DATASET_VERSION.replace('.', '_') - version = f'v{version}' + version = DATASET_VERSION.replace(".", "_") + version = f"v{version}" gunw_prefix = STANDARD_PROD_PREFIX if standard_product else CUSTOM_PROD_PREFIX - ids = [gunw_prefix, - asc_or_desc, - # right looking - 'R', - track, - # legacy constant - 'tops', - date_pair, - ref_center_time_str, - geo_str, - # legacy constant - 'PP', - ifg_hash_trunc, - version] - - gunw_id = '-'.join(ids) + ids = [ + gunw_prefix, + asc_or_desc, + # right looking + "R", + track, + # legacy constant + "tops", + date_pair, + ref_center_time_str, + geo_str, + # legacy constant + "PP", + ifg_hash_trunc, + version, + ] + + gunw_id = "-".join(ids) return gunw_id @@ -177,20 +197,18 @@ def make_geocube(isce_data_directory: Union[str, Path]) -> Path: Relative path of the metadata.h5 file. """ cwd = Path.cwd() - merged_dir = Path(isce_data_directory)/'merged' + merged_dir = Path(isce_data_directory) / "merged" os.chdir(merged_dir) - cmd = 'makeGeocube --r ../reference --s ../secondary -o metadata.h5' + cmd = "makeGeocube --r ../reference --s ../secondary -o metadata.h5" subprocess.check_call(cmd, shell=True) os.chdir(cwd) - metadata_path = merged_dir/'metadata.h5' + metadata_path = merged_dir / "metadata.h5" return metadata_path -def _write_json_config(*, - gunw_id: str, - directory: Path) -> Path: +def _write_json_config(*, gunw_id: str, directory: Path) -> Path: """Reads the json template and writes a new entry: `file: '.nc'`. Then, the new json file is saved in the directory specified. The filename @@ -211,48 +229,48 @@ def _write_json_config(*, """ nc_template = read_netcdf_packaging_template() - nc_template['filename'] = f'{gunw_id}.nc' + nc_template["filename"] = f"{gunw_id}.nc" # This will be appended to the global source attribute - nc_template['software_statement'] = f'using the DockerizedTopsApp HyP3 plugin version {isce2_topsapp.__version__}' + nc_template[ + "software_statement" + ] = f"using the DockerizedTopsApp HyP3 plugin version {isce2_topsapp.__version__}" - out_path = directory/'tops_groups.json' - with open(out_path, 'w') as f: + out_path = directory / "tops_groups.json" + with open(out_path, "w") as f: json.dump(nc_template, f, indent=2, sort_keys=True) return out_path -def perform_netcdf_packaging(*, - gunw_id: str, - isce_data_dir: Union[str, Path]) -> Path: - +def perform_netcdf_packaging(*, gunw_id: str, isce_data_dir: Union[str, Path]) -> Path: # Check that the metadata.h5 exists isce_data_dir = Path(isce_data_dir) - merged_dir = isce_data_dir/'merged' - metadata_path = merged_dir/'metadata.h5' + merged_dir = isce_data_dir / "merged" + metadata_path = merged_dir / "metadata.h5" assert metadata_path.exists() # Write config file - _write_json_config(gunw_id=gunw_id, - directory=merged_dir) + _write_json_config(gunw_id=gunw_id, directory=merged_dir) cwd = Path.cwd() os.chdir(merged_dir) - cmd = 'nc_packaging' + cmd = "nc_packaging" subprocess.check_call(cmd, shell=True) os.chdir(cwd) - out_nc_file = merged_dir / f'{gunw_id}.nc' + out_nc_file = merged_dir / f"{gunw_id}.nc" # Check if the netcdf file was created assert out_nc_file.exists() return out_nc_file -def package_additional_layers_into_gunw(gunw_path: Path, - isce_data_directory: Path, - additional_2d_layers: list, - additional_attributes: dict): +def package_additional_layers_into_gunw( + gunw_path: Path, + isce_data_directory: Path, + additional_2d_layers: list, + additional_attributes: dict, +): # Current workflow of additional layers # 1. Do any additional processing/formatting outside of GUNW # 2. Add layer into GUNW @@ -261,95 +279,112 @@ def package_additional_layers_into_gunw(gunw_path: Path, # in case additional attributes is None additional_attributes = additional_attributes or {} if not set(additional_attributes.keys()).issubset(additional_2d_layers): - raise ValueError('Additional attributes dict must be within additional_2d_layers') + raise ValueError( + "Additional attributes dict must be within additional_2d_layers" + ) - if 'ionosphere' in additional_2d_layers: + if "ionosphere" in additional_2d_layers: # current working directory is ISCE directory _ = format_ionosphere_for_gunw(isce_data_directory, gunw_path) - if 'ionosphereBurstRamps' in additional_2d_layers: + if "ionosphereBurstRamps" in additional_2d_layers: # current working directory is ISCE directory _ = format_iono_burst_ramps(isce_data_directory, gunw_path) # Assumes ionosphere raster is written to specific path - additional_attributes_lst = [additional_attributes.get(layer_name, None) - for layer_name in additional_2d_layers] + additional_attributes_lst = [ + additional_attributes.get(layer_name, None) + for layer_name in additional_2d_layers + ] zipped_data = zip(additional_2d_layers, additional_attributes_lst) - [add_2d_layer(layer, - gunw_path, - additional_attrs=add_attrs) - for (layer, add_attrs) in zipped_data] + [ + add_2d_layer(layer, gunw_path, additional_attrs=add_attrs) + for (layer, add_attrs) in zipped_data + ] # Update - with h5py.File(gunw_path, mode='a') as file: - file.attrs.modify('version', '1c') + with h5py.File(gunw_path, mode="a") as file: + file.attrs.modify("version", "1c") return gunw_path -def get_layer_mean(netcdf_path: Path, - groups: list, - layers: list) -> dict: - if len(groups) != len(layers): - raise ValueError('groups and layers must have same length') - data = {} - for group, layer in zip(groups, layers): - with rasterio.open(f'netcdf:{netcdf_path}:{group}/{layer}') as ds: - X = ds.read() - if np.isnan(ds.nodata): - mask = np.isnan(X) - else: - mask = ds.nodata == X - raster_data = X[~mask] - data[f'mean_{layer}'] = np.mean(raster_data) - return data - - -def record_stats(*, - netcdf_path: Path) -> Path: - groups = ['science/grids/imagingGeometry'] * 2 + ['science/grids/data'] - layers = ['perpendicularBaseline', 'parallelBaseline', 'coherence'] - mean_attrs = get_layer_mean(netcdf_path, - groups, - layers) - with h5py.File(netcdf_path, mode='a') as file: - file.attrs.update(**mean_attrs) - return netcdf_path - - -def record_params(*, - netcdf_path: Path, - cmd_line_str: str, - topsapp_params: dict) -> Path: - - with h5py.File(netcdf_path, mode='a') as file: - file.attrs.update(aria_frame_id=topsapp_params['frame_id']) +def get_layer_mean( + merged_dir: Union[Path, str], layer_name: str, apply_water_mask: bool = False +) -> float: + log = f"Extracting mean value from {layer_name}" + if apply_water_mask: + log += " with water mask" + print(log) + merged_dir = Path(merged_dir) + layer_path = merged_dir / LAYER2PATH[layer_name]["file_name"] + band_num = LAYER2PATH[layer_name]["band"] + + with rasterio.open(layer_path) as ds: + X = ds.read(band_num) + X_nodata = ~(ds.read_masks(band_num).astype(bool)) + if apply_water_mask: + p = ds.profile + water_mask = get_water_mask_raster_for_browse_image(p) + X_nodata = water_mask | X_nodata + + mean_val = np.mean(X[~X_nodata]) + return mean_val + + +def get_layer_stats(*, merged_dir: Union[Path, str] = None) -> Path: + if merged_dir is None: + cwd = Path.cwd() + merged_dir = f"{cwd}/merged" + + def get_layer_mean_p(layer_name: str, apply_water_mask: bool = False) -> float: + return get_layer_mean(merged_dir, layer_name, apply_water_mask=apply_water_mask) + + mean_vals = { + "mean_filtered_coherence_without_water_mask": get_layer_mean_p( + "filtered_coherence", apply_water_mask=False + ), + "mean_filtered_coherence_with_water_mask": get_layer_mean_p( + "filtered_coherence", apply_water_mask=True + ), + "mean_incidence_angle": get_layer_mean_p("incidence_angle"), + "mean_azimuth_angle": get_layer_mean_p("azimuth_angle") + 90, + } + return mean_vals + + +def record_params( + *, netcdf_path: Path, cmd_line_str: str, topsapp_params: dict +) -> Path: + with h5py.File(netcdf_path, mode="a") as file: + file.attrs.update(aria_frame_id=topsapp_params["frame_id"]) file.attrs.update(topsapp_command_line_string=cmd_line_str) - file.attrs.update(isce2_topsapp_version=f'{isce2_topsapp.__version__}') - file['science/grids'].attrs.update(**topsapp_params) + file.attrs.update(isce2_topsapp_version=f"{isce2_topsapp.__version__}") + file["science/grids"].attrs.update(**topsapp_params) return netcdf_path -def record_wkt_geometry(*, - netcdf_path: Path, - product_geometry_wkt: str - ) -> Path: - - with h5py.File(netcdf_path, mode='a') as file: +def record_wkt_geometry(*, netcdf_path: Path, product_geometry_wkt: str) -> Path: + with h5py.File(netcdf_path, mode="a") as file: file.attrs.update(product_geometry_wkt=product_geometry_wkt) return netcdf_path -def package_gunw_product(*, - isce_data_directory: Union[str, Path], - reference_properties: list, - secondary_properties: list, - extent: list, - topaspp_params: dict, - cmd_line_str: str, - product_geometry_wkt: str, - additional_2d_layers: list = None, - standard_product: bool = True, - additional_attributes: dict = None, - ) -> Path: +def record_stats(*, netcdf_path: Union[Path, str]) -> Path: + return Path() + + +def package_gunw_product( + *, + isce_data_directory: Union[str, Path], + reference_properties: list, + secondary_properties: list, + extent: list, + topaspp_params: dict, + cmd_line_str: str, + product_geometry_wkt: str, + additional_2d_layers: list = None, + standard_product: bool = True, + additional_attributes: dict = None, +) -> Path: """Creates a GUNW standard product netcdf from the ISCE outputs and some initial metadata. @@ -380,24 +415,32 @@ def package_gunw_product(*, """ make_geocube(isce_data_directory) - gunw_id = get_gunw_id(reference_properties=reference_properties, - secondary_properties=secondary_properties, - extent=extent, - standard_product=standard_product) + gunw_id = get_gunw_id( + reference_properties=reference_properties, + secondary_properties=secondary_properties, + extent=extent, + standard_product=standard_product, + ) - out_nc_file = perform_netcdf_packaging(isce_data_dir=isce_data_directory, - gunw_id=gunw_id) + out_nc_file = perform_netcdf_packaging( + isce_data_dir=isce_data_directory, gunw_id=gunw_id + ) if additional_2d_layers is not None: isce_data_directory = Path(isce_data_directory) - package_additional_layers_into_gunw(out_nc_file, - isce_data_directory, - additional_2d_layers, - additional_attributes) - out_nc_file = record_params(netcdf_path=out_nc_file, - topsapp_params=topaspp_params, - cmd_line_str=cmd_line_str) + package_additional_layers_into_gunw( + out_nc_file, + isce_data_directory, + additional_2d_layers, + additional_attributes, + ) + out_nc_file = record_params( + netcdf_path=out_nc_file, + topsapp_params=topaspp_params, + cmd_line_str=cmd_line_str, + ) out_nc_file = record_stats(netcdf_path=out_nc_file) - out_nc_file = record_wkt_geometry(netcdf_path=out_nc_file, - product_geometry_wkt=product_geometry_wkt) + out_nc_file = record_wkt_geometry( + netcdf_path=out_nc_file, product_geometry_wkt=product_geometry_wkt + ) return out_nc_file diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 4774642..020d2d2 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,7 +1,14 @@ import json from pathlib import Path -from isce2_topsapp.packaging import DATASET_VERSION, get_gunw_id, read_baselines +from numpy.testing import assert_almost_equal + +from isce2_topsapp.packaging import ( + DATASET_VERSION, + get_gunw_id, + get_layer_stats, + read_baselines, +) test_dir = Path(__file__).parent @@ -44,3 +51,19 @@ def test_extract_baselines(tops_proc_xml_path): } assert baseline_dict == expected_output + + +def test_mean_of_geocoded_isce_outputs(): + """This uses the data in the public bucket to demonstrate and verify that the layer means are correct""" + out = get_layer_stats( + merged_dir="https://gunw-development-testing.s3.us-west-2.amazonaws.com/sample_merged_data" + ) + expected_out = { + "mean_filtered_coherence_without_water_mask": 0.3126283, + "mean_filtered_coherence_with_water_mask": 0.32342613, + "mean_incidence_angle": 28.281992, + "mean_azimuth_angle": -99.18769836425781, + } + + for key in expected_out: + assert_almost_equal(expected_out[key], out[key], decimal=5) From 19eaf403a9ce2a129f4a0426f1fd7c305ca7b495 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 18:26:24 -0800 Subject: [PATCH 08/19] rename fxns and finalize record --- isce2_topsapp/packaging.py | 28 +++++++++++++++++++--------- tests/test_packaging.py | 4 ++-- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index 5139614..f6a86ad 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -72,7 +72,7 @@ def read_baselines(tops_proc_xml: str) -> dict: def get_mean_baseline_data(tops_proc_xml: str) -> dict: baseline_data = read_baselines(tops_proc_xml) - mean_baseline_data = {key: np.mean(val) for (key, val) in baseline_data.items()} + mean_baseline_data = {f'mean_{key[:-1]}': np.mean(val) for (key, val) in baseline_data.items()} return mean_baseline_data @@ -330,7 +330,7 @@ def get_layer_mean( return mean_val -def get_layer_stats(*, merged_dir: Union[Path, str] = None) -> Path: +def get_geocoded_layer_means(*, merged_dir: Union[Path, str] = None) -> Path: if merged_dir is None: cwd = Path.cwd() merged_dir = f"{cwd}/merged" @@ -351,7 +351,7 @@ def get_layer_mean_p(layer_name: str, apply_water_mask: bool = False) -> float: return mean_vals -def record_params( +def record_params_as_global_attrs( *, netcdf_path: Path, cmd_line_str: str, topsapp_params: dict ) -> Path: with h5py.File(netcdf_path, mode="a") as file: @@ -362,14 +362,24 @@ def record_params( return netcdf_path -def record_wkt_geometry(*, netcdf_path: Path, product_geometry_wkt: str) -> Path: +def record_wkt_geometry_as_global_attrs(*, netcdf_path: Path, product_geometry_wkt: str) -> Path: with h5py.File(netcdf_path, mode="a") as file: file.attrs.update(product_geometry_wkt=product_geometry_wkt) return netcdf_path -def record_stats(*, netcdf_path: Union[Path, str]) -> Path: - return Path() +def record_stats_as_global_attrs(*, netcdf_path: Union[Path, str], isce_data_dir: Union[Path, str]) -> Path: + """Records the mean coherence (with and without water mask), mean incidence angle, mean azimuth angle, and + mean baselines (parallel and perp)""" + merged_dir = Path(isce_data_dir) / 'merged' + layer_means_from_geocoded_isce_files = get_geocoded_layer_means(merged_dir=merged_dir) + + tops_proc_xml = Path(isce_data_dir) / 'topsProc.xml' + mean_baseline_data = get_mean_baseline_data(tops_proc_xml) + with h5py.File(netcdf_path, mode='a') as file: + file.attrs.update(**layer_means_from_geocoded_isce_files) + file.attrs.update(**mean_baseline_data) + return netcdf_path def package_gunw_product( @@ -434,13 +444,13 @@ def package_gunw_product( additional_2d_layers, additional_attributes, ) - out_nc_file = record_params( + out_nc_file = record_params_as_global_attrs( netcdf_path=out_nc_file, topsapp_params=topaspp_params, cmd_line_str=cmd_line_str, ) - out_nc_file = record_stats(netcdf_path=out_nc_file) - out_nc_file = record_wkt_geometry( + out_nc_file = record_stats_as_global_attrs(netcdf_path=out_nc_file, isce_data_dir=isce_data_directory) + out_nc_file = record_wkt_geometry_as_global_attrs( netcdf_path=out_nc_file, product_geometry_wkt=product_geometry_wkt ) return out_nc_file diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 020d2d2..f45c4a7 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -6,7 +6,7 @@ from isce2_topsapp.packaging import ( DATASET_VERSION, get_gunw_id, - get_layer_stats, + get_geocoded_layer_means, read_baselines, ) @@ -55,7 +55,7 @@ def test_extract_baselines(tops_proc_xml_path): def test_mean_of_geocoded_isce_outputs(): """This uses the data in the public bucket to demonstrate and verify that the layer means are correct""" - out = get_layer_stats( + out = get_geocoded_layer_means( merged_dir="https://gunw-development-testing.s3.us-west-2.amazonaws.com/sample_merged_data" ) expected_out = { From 3dd788f52c729300a3f37566270feb7aba74bac2 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 29 Jan 2024 18:28:27 -0800 Subject: [PATCH 09/19] ruff --- tests/test_packaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index f45c4a7..3d288b9 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -5,8 +5,8 @@ from isce2_topsapp.packaging import ( DATASET_VERSION, - get_gunw_id, get_geocoded_layer_means, + get_gunw_id, read_baselines, ) From 520523eb9be07ea7652404f99d066e4c86886d4d Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 30 Jan 2024 08:49:36 -0800 Subject: [PATCH 10/19] changelog fix version --- CHANGELOG.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1d12c..6b00f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,15 +6,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.3] -* Uses ESA World Cover 2021 Permanent Water Class for ionosphere water mask and brose imagery water mask -* Finalizes global attributes:.... - -## [0.3.2] - -### Changed -* Upgraded to `hyp3lib=>3,<4` from `>=2,<3` - ## [0.3.1] ### Added @@ -35,6 +26,9 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) +* Water mask now uses `tile-mate` to download and merge ESA World cover tiles on +* All water masks applied to processing/packaging use ESA 10 meter world cover: ionosphere processing, browse imagery, and global attributes associate with mean coherence +* Some function names associated to writing global attributes in the netcdf file were renamed to be more descriptive e.g. `record_stats` became `record_stats_as_global_attrs` ## [0.3.0] From 3c293752e9075227486bbd1712bba43d3b500f86 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 30 Jan 2024 08:59:04 -0800 Subject: [PATCH 11/19] add tiff to water mask for iono --- isce2_topsapp/localize_mask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/localize_mask.py b/isce2_topsapp/localize_mask.py index 3b6faf2..86cb85d 100644 --- a/isce2_topsapp/localize_mask.py +++ b/isce2_topsapp/localize_mask.py @@ -25,9 +25,9 @@ def download_water_mask( X, p = get_raster_from_tiles(extent_buffered, tile_shortname='esa_world_cover_2021') mask = (X == 80).astype(np.uint8) mask[mask.astype(bool)] = 255 - mask_filename = 'water_mask_derived_from_esa_world_cover_2021_10m' + mask_filename = 'water_mask_derived_from_esa_world_cover_2021_10m.tif' with rasterio.open(mask_filename, 'w', **p) as ds: - ds.write(X) + ds.write(mask) elif water_mask_name == "SWBD": # Download SRTM-SWDB water mask From df8182f390a9bef8d8bdf8cf066f87fb0da39c6e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 30 Jan 2024 10:41:40 -0800 Subject: [PATCH 12/19] ISCE2-ify mask --- isce2_topsapp/localize_mask.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/localize_mask.py b/isce2_topsapp/localize_mask.py index 86cb85d..0c99a4f 100644 --- a/isce2_topsapp/localize_mask.py +++ b/isce2_topsapp/localize_mask.py @@ -6,6 +6,8 @@ from shapely.geometry import box from tile_mate import get_raster_from_tiles +from .localize_dem import fix_image_xml + def download_water_mask( extent: list, water_mask_name: str = "esa_world_cover_2021_10m", buffer: float = 0.1 @@ -25,10 +27,18 @@ def download_water_mask( X, p = get_raster_from_tiles(extent_buffered, tile_shortname='esa_world_cover_2021') mask = (X == 80).astype(np.uint8) mask[mask.astype(bool)] = 255 - mask_filename = 'water_mask_derived_from_esa_world_cover_2021_10m.tif' - with rasterio.open(mask_filename, 'w', **p) as ds: + mask_filename = 'water_mask_derived_from_esa_world_cover_2021_10m.geo' + + # Remove esa nodata, change gdal driver to ISCE, and generate VRT as in localize DEM + p_isce = p.copy() + p_isce['nodata'] = None + p_isce['driver'] = 'ISCE' + + with rasterio.open(mask_filename, 'w', **p_isce) as ds: ds.write(mask) + mask_filename = fix_image_xml(mask_filename) + elif water_mask_name == "SWBD": # Download SRTM-SWDB water mask # Water mask dataset extent From ce2d6dea3a048c12e03d8d3e12bdb4898891bc6d Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 30 Jan 2024 11:16:15 -0800 Subject: [PATCH 13/19] update env file for conda tile-mate --- environment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 83114ab..8d61944 100644 --- a/environment.yml +++ b/environment.yml @@ -42,7 +42,6 @@ dependencies: - setuptools_scm - shapely - tqdm - - dem_stitcher>=2.4.0 + - dem_stitcher>=2.5.5 + - tile_mate - aiohttp # only needed for manifest and swath download - - pip: - - tile-mate From 5036fc32592dc4798cfeec09d00c9f2ec891e066 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 30 Jan 2024 11:19:24 -0800 Subject: [PATCH 14/19] revert to pip --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8d61944..be3737d 100644 --- a/environment.yml +++ b/environment.yml @@ -43,5 +43,6 @@ dependencies: - shapely - tqdm - dem_stitcher>=2.5.5 - - tile_mate - aiohttp # only needed for manifest and swath download + - pip: + - tile-mate From 979a31fd0d0d0dd12ce02a94ebad6db0c49e9932 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 30 Jan 2024 16:32:17 -0800 Subject: [PATCH 15/19] update tile_mate to conda-forge --- environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index be3737d..d2431c0 100644 --- a/environment.yml +++ b/environment.yml @@ -44,5 +44,4 @@ dependencies: - tqdm - dem_stitcher>=2.5.5 - aiohttp # only needed for manifest and swath download - - pip: - - tile-mate + - tile_mate>=0.0.7 From ff122b2ae70727b1feed436d8cc5708cf4e74491 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 31 Jan 2024 17:14:11 -0800 Subject: [PATCH 16/19] update packaging so that unfiltered coherence is added as global variable --- isce2_topsapp/packaging.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index f6a86ad..622cb04 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -44,6 +44,7 @@ "incidence_angle": {"file_name": "los.rdr.geo", "band": 1}, "azimuth_angle": {"file_name": "los.rdr.geo", "band": 2}, "filtered_coherence": {"file_name": "phsig.cor.geo", "band": 1}, + "unfiltered_coherence": {"file_name": "topophase.cor.geo", "band": 2} } @@ -308,7 +309,7 @@ def package_additional_layers_into_gunw( def get_layer_mean( - merged_dir: Union[Path, str], layer_name: str, apply_water_mask: bool = False + merged_dir: Union[Path, str], layer_name: str, apply_water_mask: bool = False, default_isce_nodata: float = 0., ) -> float: log = f"Extracting mean value from {layer_name}" if apply_water_mask: @@ -320,7 +321,7 @@ def get_layer_mean( with rasterio.open(layer_path) as ds: X = ds.read(band_num) - X_nodata = ~(ds.read_masks(band_num).astype(bool)) + X_nodata = (X == default_isce_nodata) if apply_water_mask: p = ds.profile water_mask = get_water_mask_raster_for_browse_image(p) @@ -345,6 +346,12 @@ def get_layer_mean_p(layer_name: str, apply_water_mask: bool = False) -> float: "mean_filtered_coherence_with_water_mask": get_layer_mean_p( "filtered_coherence", apply_water_mask=True ), + "mean_unfiltered_coherence_without_water_mask": get_layer_mean_p( + "unfiltered_coherence", apply_water_mask=False + ), + "mean_unfiltered_coherence_with_water_mask": get_layer_mean_p( + "unfiltered_coherence", apply_water_mask=True + ), "mean_incidence_angle": get_layer_mean_p("incidence_angle"), "mean_azimuth_angle": get_layer_mean_p("azimuth_angle") + 90, } From 288fcdcc842e9af0e5e9a0ad4702ceae9d3c876a Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 31 Jan 2024 17:14:38 -0800 Subject: [PATCH 17/19] compare means with GUNW cubes/rasters representing similar items --- tests/test_packaging.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 3d288b9..8b9f07e 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,6 +1,8 @@ import json from pathlib import Path +import numpy as np +import rasterio from numpy.testing import assert_almost_equal from isce2_topsapp.packaging import ( @@ -59,11 +61,41 @@ def test_mean_of_geocoded_isce_outputs(): merged_dir="https://gunw-development-testing.s3.us-west-2.amazonaws.com/sample_merged_data" ) expected_out = { - "mean_filtered_coherence_without_water_mask": 0.3126283, - "mean_filtered_coherence_with_water_mask": 0.32342613, - "mean_incidence_angle": 28.281992, - "mean_azimuth_angle": -99.18769836425781, + "mean_filtered_coherence_without_water_mask": 0.4395995, + "mean_filtered_coherence_with_water_mask": 0.4649538, + "mean_unfiltered_coherence_without_water_mask": 0.33125195, + "mean_unfiltered_coherence_with_water_mask": 0.33531728, + "mean_incidence_angle": 38.845047, + "mean_azimuth_angle": -169.84756469726562, } for key in expected_out: assert_almost_equal(expected_out[key], out[key], decimal=5) + + # Use output GUNW and compare means + gunw_s3_path = ( + "/vsis3/gunw-development-testing/sample_merged_data" + "/S1-GUNW-A-R-164-tops-20220212_20220131-222829-00071W_00047N-PP-3d6c-v3_0_0.nc" + ) + + vars_data_0 = {var: f"//science/grids/data/{var}" for var in ["unfilteredCoherence", "coherence"]} + vars_data_1 = {var: f"//science/grids/imagingGeometry/{var}" for var in ["azimuthAngle", "incidenceAngle"]} + vars_data = {**vars_data_0, **vars_data_1} + + gunw_mean_data = {} + for var, var_path in vars_data.items(): + with rasterio.open(f"HDF5:{gunw_s3_path}:{var_path}") as ds: + X = ds.read() + # 0 is the nodata for all the variables we consider + gunw_mean_data[var] = np.mean(X[X != 0]) + + # Ensure the GUNW cubes and rasters (with lower resolution than geocoded ISCE2 data) are within a reasonable range + # of the geocoded rasters that ICE2 produces + assert np.abs(out['mean_filtered_coherence_without_water_mask'] - gunw_mean_data['coherence']) < .01 + assert np.abs(out['mean_unfiltered_coherence_without_water_mask'] - gunw_mean_data['unfilteredCoherence']) < .01 + assert np.abs(out['mean_azimuth_angle'] - gunw_mean_data['azimuthAngle']) < 1 + assert np.abs(out['mean_incidence_angle'] - gunw_mean_data['incidenceAngle']) < 1 + + # Sanity checks - make sure that applying a water mask *increases* the coherence as water has low coherence + assert out['mean_filtered_coherence_without_water_mask'] < out['mean_filtered_coherence_with_water_mask'] + assert out['mean_unfiltered_coherence_without_water_mask'] < out['mean_unfiltered_coherence_with_water_mask'] From f096d287908164fb29eb853728e4434bb54f4451 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 31 Jan 2024 17:14:46 -0800 Subject: [PATCH 18/19] delete --- Untitled.ipynb | 368 ------------------------------------------------- 1 file changed, 368 deletions(-) delete mode 100644 Untitled.ipynb diff --git a/Untitled.ipynb b/Untitled.ipynb deleted file mode 100644 index 2725ca2..0000000 --- a/Untitled.ipynb +++ /dev/null @@ -1,368 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "43a70bf0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is the Open Source version of ISCE.\n", - "Some of the workflows depend on a separate licensed package.\n", - "To obtain the licensed package, please make a request for ISCE\n", - "through the website: https://download.jpl.nasa.gov/ops/request/index.cfm.\n", - "Alternatively, if you are a member, or can become a member of WinSAR\n", - "you may be able to obtain access to a version of the licensed sofware at\n", - "https://winsar.unavco.org/software/isce\n", - "2023-11-08 12:49:09,028 - matplotlib.font_manager - DEBUG - Using fontManager instance from /Users/cmarshak/.matplotlib/fontlist-v330.json\n" - ] - } - ], - "source": [ - "from isce2_topsapp.__main__ import get_slc_parser, true_false_string_argument\n", - "from argparse import ArgumentParser\n", - "import xarray as xr\n", - "import h5py" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "49e167bb", - "metadata": {}, - "outputs": [], - "source": [ - "path = 'S1-GUNW-A-R-064-tops-20210723_20210711-015000-00119W_00033N-PP-6267-v2_0_6.nc'" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "486971e2", - "metadata": {}, - "outputs": [], - "source": [ - "with h5py.File(path, mode='a') as file:\n", - " \n", - " file.attrs.update(dict(oy_vey=1))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "55d6a081", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['product_type',\n", - " 'Conventions',\n", - " 'title',\n", - " 'version',\n", - " 'author',\n", - " 'institution',\n", - " 'references',\n", - " '_NCProperties',\n", - " 'ogr_geometry_field',\n", - " 'ogr_layer_name',\n", - " 'ogr_layer_type',\n", - " 'source',\n", - " 'testing',\n", - " 'test',\n", - " 'oy_vey']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f2 = h5py.File(path)\n", - "list(f2.attrs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ef90ce1", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "407841d2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "root group (NETCDF4 data model, file format HDF5):\n", - " product_type: UNW GEO IFG\n", - " Conventions: CF-1.6\n", - " title: ARIA standard product UNW GEO IFG\n", - " version: 1\n", - " author: David Bekaert, Charlie Marshak, Simran Sangha, Joe Kennedy, Andrew Johnston\n", - " institution: Jet Propulsion Laboratory\n", - " references: https://aria.jpl.nasa.gov/\n", - " ogr_geometry_field: productBoundingBox\n", - " ogr_layer_name: productBoundingBox\n", - " ogr_layer_type: POLYGON\n", - " source: Contains modified Copernicus Sentinel data processed by ESA and ARIA NASA/JPL using the DockerizedTopsApp HyP3 plugin version 0.2.2.dev40+g679dfc6\n", - " testing: 1\n", - " dimensions(sizes): matchup(2), wkt_length(464), wkt_count(1)\n", - " variables(dimensions): |S1 productBoundingBox(wkt_count, wkt_length), int32 crs_polygon()\n", - " groups: science" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds = netCDF4.Dataset(path, mode='a')\n", - "ds" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "cf18ec99", - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'setncattr' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43msetncattr\u001b[49m\n", - "\u001b[0;31mNameError\u001b[0m: name 'setncattr' is not defined" - ] - } - ], - "source": [ - "ds.setncattr()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b68feb0", - "metadata": {}, - "outputs": [], - "source": [ - "ds.setncattr('testing', '1')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d558deb", - "metadata": {}, - "outputs": [], - "source": [ - "#ds.close()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "a1e09885", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Conventions',\n", - " '__class__',\n", - " '__delattr__',\n", - " '__dir__',\n", - " '__doc__',\n", - " '__enter__',\n", - " '__eq__',\n", - " '__exit__',\n", - " '__format__',\n", - " '__ge__',\n", - " '__getattr__',\n", - " '__getattribute__',\n", - " '__getitem__',\n", - " '__gt__',\n", - " '__hash__',\n", - " '__init__',\n", - " '__init_subclass__',\n", - " '__le__',\n", - " '__lt__',\n", - " '__ne__',\n", - " '__new__',\n", - " '__orthogonal_indexing__',\n", - " '__reduce__',\n", - " '__reduce_ex__',\n", - " '__repr__',\n", - " '__setattr__',\n", - " '__sizeof__',\n", - " '__str__',\n", - " '__subclasshook__',\n", - " '_close',\n", - " '_close_mem',\n", - " '_enddef',\n", - " '_getname',\n", - " '_grpid',\n", - " '_isopen',\n", - " '_ncstring_attrs__',\n", - " '_redef',\n", - " 'author',\n", - " 'close',\n", - " 'cmptypes',\n", - " 'createCompoundType',\n", - " 'createDimension',\n", - " 'createEnumType',\n", - " 'createGroup',\n", - " 'createVLType',\n", - " 'createVariable',\n", - " 'data_model',\n", - " 'delncattr',\n", - " 'dimensions',\n", - " 'disk_format',\n", - " 'enumtypes',\n", - " 'file_format',\n", - " 'filepath',\n", - " 'fromcdl',\n", - " 'get_variables_by_attributes',\n", - " 'getncattr',\n", - " 'groups',\n", - " 'has_blosc_filter',\n", - " 'has_bzip2_filter',\n", - " 'has_szip_filter',\n", - " 'has_zstd_filter',\n", - " 'institution',\n", - " 'isopen',\n", - " 'keepweakref',\n", - " 'name',\n", - " 'ncattrs',\n", - " 'ogr_geometry_field',\n", - " 'ogr_layer_name',\n", - " 'ogr_layer_type',\n", - " 'parent',\n", - " 'path',\n", - " 'product_type',\n", - " 'references',\n", - " 'renameAttribute',\n", - " 'renameDimension',\n", - " 'renameGroup',\n", - " 'renameVariable',\n", - " 'set_always_mask',\n", - " 'set_auto_chartostring',\n", - " 'set_auto_mask',\n", - " 'set_auto_maskandscale',\n", - " 'set_auto_scale',\n", - " 'set_fill_off',\n", - " 'set_fill_on',\n", - " 'set_ncstring_attrs',\n", - " 'setncattr',\n", - " 'setncattr_string',\n", - " 'setncatts',\n", - " 'source',\n", - " 'sync',\n", - " 'title',\n", - " 'tocdl',\n", - " 'variables',\n", - " 'version',\n", - " 'vltypes']" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dir(ds)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "f37e82b2", - "metadata": {}, - "outputs": [], - "source": [ - "with xr.open_dataset(path, group='input_parameters') as ds:\n", - " ds = ds.assign_attrs(test_1 = 1)\n", - "ds.to_netcdf(path)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "89023a90", - "metadata": {}, - "outputs": [ - { - "ename": "OSError", - "evalue": "[Errno group not found: input_parameters] 'input_parameters'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:187\u001b[0m, in \u001b[0;36m_nc4_require_group\u001b[0;34m(ds, group, mode, create_group)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 187\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 188\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "\u001b[0;31mKeyError\u001b[0m: 'input_parameters'", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43minput_parameters\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/api.py:566\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 554\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 555\u001b[0m decode_cf,\n\u001b[1;32m 556\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 562\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 563\u001b[0m )\n\u001b[1;32m 565\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 566\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 567\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 568\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 569\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 570\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 571\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 572\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 573\u001b[0m backend_ds,\n\u001b[1;32m 574\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 584\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 585\u001b[0m )\n\u001b[1;32m 586\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:590\u001b[0m, in \u001b[0;36mNetCDF4BackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, lock, autoclose)\u001b[0m\n\u001b[1;32m 569\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mopen_dataset\u001b[39m( \u001b[38;5;66;03m# type: ignore[override] # allow LSP violation, not supporting **kwargs\u001b[39;00m\n\u001b[1;32m 570\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 571\u001b[0m filename_or_obj: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m os\u001b[38;5;241m.\u001b[39mPathLike[Any] \u001b[38;5;241m|\u001b[39m BufferedIOBase \u001b[38;5;241m|\u001b[39m AbstractDataStore,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 587\u001b[0m autoclose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 588\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dataset:\n\u001b[1;32m 589\u001b[0m filename_or_obj \u001b[38;5;241m=\u001b[39m _normalize_path(filename_or_obj)\n\u001b[0;32m--> 590\u001b[0m store \u001b[38;5;241m=\u001b[39m \u001b[43mNetCDF4DataStore\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 592\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 593\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 594\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 595\u001b[0m \u001b[43m \u001b[49m\u001b[43mclobber\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclobber\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 596\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiskless\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiskless\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 597\u001b[0m \u001b[43m \u001b[49m\u001b[43mpersist\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpersist\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 598\u001b[0m \u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 599\u001b[0m \u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 600\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 602\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 603\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:391\u001b[0m, in \u001b[0;36mNetCDF4DataStore.open\u001b[0;34m(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)\u001b[0m\n\u001b[1;32m 385\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(\n\u001b[1;32m 386\u001b[0m clobber\u001b[38;5;241m=\u001b[39mclobber, diskless\u001b[38;5;241m=\u001b[39mdiskless, persist\u001b[38;5;241m=\u001b[39mpersist, \u001b[38;5;28mformat\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mformat\u001b[39m\n\u001b[1;32m 387\u001b[0m )\n\u001b[1;32m 388\u001b[0m manager \u001b[38;5;241m=\u001b[39m CachingFileManager(\n\u001b[1;32m 389\u001b[0m netCDF4\u001b[38;5;241m.\u001b[39mDataset, filename, mode\u001b[38;5;241m=\u001b[39mmode, kwargs\u001b[38;5;241m=\u001b[39mkwargs\n\u001b[1;32m 390\u001b[0m )\n\u001b[0;32m--> 391\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:338\u001b[0m, in \u001b[0;36mNetCDF4DataStore.__init__\u001b[0;34m(self, manager, group, mode, lock, autoclose)\u001b[0m\n\u001b[1;32m 336\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_group \u001b[38;5;241m=\u001b[39m group\n\u001b[1;32m 337\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m mode\n\u001b[0;32m--> 338\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mds\u001b[49m\u001b[38;5;241m.\u001b[39mdata_model\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mds\u001b[38;5;241m.\u001b[39mfilepath()\n\u001b[1;32m 340\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_remote \u001b[38;5;241m=\u001b[39m is_remote_uri(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename)\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:400\u001b[0m, in \u001b[0;36mNetCDF4DataStore.ds\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 398\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mds\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 400\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:395\u001b[0m, in \u001b[0;36mNetCDF4DataStore._acquire\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_acquire\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_manager\u001b[38;5;241m.\u001b[39macquire_context(needs_lock) \u001b[38;5;28;01mas\u001b[39;00m root:\n\u001b[0;32m--> 395\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43m_nc4_require_group\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_group\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 396\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/mambaforge/envs/topsapp_env/lib/python3.9/site-packages/xarray/backends/netCDF4_.py:193\u001b[0m, in \u001b[0;36m_nc4_require_group\u001b[0;34m(ds, group, mode, create_group)\u001b[0m\n\u001b[1;32m 190\u001b[0m ds \u001b[38;5;241m=\u001b[39m create_group(ds, key)\n\u001b[1;32m 191\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 192\u001b[0m \u001b[38;5;66;03m# wrap error to provide slightly more helpful message\u001b[39;00m\n\u001b[0;32m--> 193\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgroup not found: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "\u001b[0;31mOSError\u001b[0m: [Errno group not found: input_parameters] 'input_parameters'" - ] - } - ], - "source": [ - "ds = xr.open_dataset(path, group='input_parameters')\n", - "ds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "48688cd3", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "topsapp_env", - "language": "python", - "name": "topsapp_env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 04a592fbed6b071d6fa8e121e355e25ecddd276c Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 31 Jan 2024 18:16:15 -0800 Subject: [PATCH 19/19] add no sign request for test suite --- tests/test_packaging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 8b9f07e..4a5c2af 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path import numpy as np @@ -57,6 +58,7 @@ def test_extract_baselines(tops_proc_xml_path): def test_mean_of_geocoded_isce_outputs(): """This uses the data in the public bucket to demonstrate and verify that the layer means are correct""" + os.environ['AWS_NO_SIGN_REQUEST'] = 'YES' out = get_geocoded_layer_means( merged_dir="https://gunw-development-testing.s3.us-west-2.amazonaws.com/sample_merged_data" )