Merge pull request #102 from ACCESS-Cloud-Based-InSAR/frame-metadata

Update metadata with `frame_id`, `version` and `temporal_baseline_days`
ACCESS-Cloud-Based-InSAR · May 27, 2023 · 1a140d4 · 1a140d4
2 parents 0f74f88 + 98dadaf
commit 1a140d4
Show file tree

Hide file tree

Showing 7 changed files with 91 additions and 70 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -38,6 +38,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 * Exposes `frame-id` parameter for fixed frame cropping. Discussion, references, and examples in README.
 * Latitude aligned frames and their expected extents are added as geojson in repository as zip file.
 * Pins ISCE2 version to 2.6.1 and numpy / scipy to previous versions (see environment.yml) - to be amended when newest ISCE2 build is sorted out
+* Includes `frame_id` and `temporal_baseline_days` in json metadata for CMR handshake. The former is the fixed frame id and the latter is the number of days between images (will be multiple of 6).
 * Added support to compute and embed solid earth tide correction layers into GUNW products (see PR #91) - reference and secondary have own groups
 * Raises warning if there is at least 80% of water in the IFG area using Natural Earth Land mask.
 

diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py
@@ -62,15 +62,14 @@ def localize_data(
 
         out_aux_cal = download_aux_cal()
 
-    out = {
-        "reference_scenes": reference_scenes,
-        "secondary_scenes": secondary_scenes,
-        **out_slc,
-        **out_dem,
-        **out_water_mask,
-        **out_aux_cal,
-        **out_orbits,
-    }
+    out = {'reference_scenes': reference_scenes,
+           'secondary_scenes': secondary_scenes,
+           'frame_id': frame_id,
+           **out_slc,
+           **out_dem,
+           **out_water_mask,
+           **out_aux_cal,
+           **out_orbits}
     return out
 
 

diff --git a/isce2_topsapp/delivery_prep.py b/isce2_topsapp/delivery_prep.py
@@ -5,6 +5,7 @@
 import numpy as np
 import rasterio
 from PIL import Image
+from dateparser import parse
 from matplotlib import cm
 
 from isce2_topsapp.packaging import DATASET_VERSION
@@ -113,37 +114,36 @@ def gen_browse_imagery(nc_path: Path,
     return out_path
 
 
-def format_time_string(time_str: str) -> str:
-    """Generates formatted string for ingest schema using built in python manipulations.
-    New ASF API provides time string as 2022-05-20T07:11:01.000Z and we need 6 decimal places
-    at end of string, specifically, 2022-05-20T07:11:01.000000Z
-    """
-    assert time_str[-1] == 'Z'
-
-    # removes Z at end and focuses in on decimal
-    temp = time_str[:-1].split('.')
-
-    # update the split string with required decimals
-    temp[-1] = f'{int(temp[-1]):06d}'
-
-    # rejoin and return
-    formatted_time_str = '.'.join(temp)
-    return f'{formatted_time_str}Z'
-
-
 def format_metadata(nc_path: Path,
                     all_metadata: dict) -> dict:
 
     now = datetime.datetime.now()
     label = nc_path.name[:-3]  # removes suffix .nc
     geojson = all_metadata['gunw_geo'].__geo_interface__
 
-    ref_props = all_metadata['reference_properties'][0]
-    sec_props = all_metadata['secondary_properties'][0]
+    ref_props_all = sorted(all_metadata['reference_properties'],
+                           key=lambda prop: prop['startTime'])
+    ref_props_first = ref_props_all[0]
+    sec_props_all = sorted(all_metadata['secondary_properties'],
+                           key=lambda prop: prop['startTime'])
+    sec_props_first = sec_props_all[0]
     b_perp = read_baseline_perp(nc_path).mean()
 
-    startTime_f = format_time_string(ref_props["startTime"])
-    stopTime_f = format_time_string(ref_props["stopTime"])
+    ref_start_times = [parse(props['startTime']) for props in ref_props_all]
+    ref_stop_times = [parse(props['stopTime']) for props in ref_props_all]
+    sec_start_times = [parse(props['startTime']) for props in sec_props_all]
+
+    ref_start_time = ref_start_times[0]
+    ref_stop_time = ref_stop_times[-1]
+    sec_start_time = sec_start_times[0]
+
+    # The %f is miliseconds zero-padded with 6 decimals - just as we need!
+    ref_start_time_formatted = ref_start_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+    ref_stop_time_formatted = ref_stop_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+
+    # We want the nearest day (dt.days takes a floor) so we use total seconds and then round
+    temporal_baseline_seconds = (ref_start_time - sec_start_time).total_seconds()
+    temporal_baseline_days = round(temporal_baseline_seconds / 60 / 60 / 24)
 
     metadata = {}
     # get 4 corners of bounding box of the geometry; default is 5 returning
@@ -152,18 +152,20 @@ def format_metadata(nc_path: Path,
     metadata.update({"ogr_bbox": ogr_bbox,
                      "reference_scenes": all_metadata['reference_scenes'],
                      "secondary_scenes": all_metadata['secondary_scenes'],
-                     "sensing_start": startTime_f,
-                     "sensing_stop": stopTime_f,
-                     "orbit_number": [int(ref_props['orbit']),
-                                      int(sec_props['orbit'])],
-                     "platform": [ref_props['platform'], sec_props['platform']],
-                     "beam_mode": ref_props['beamModeType'],
-                     "orbit_direction": ref_props['flightDirection'].lower(),
+                     "sensing_start": ref_start_time_formatted,
+                     "sensing_stop": ref_stop_time_formatted,
+                     "version": DATASET_VERSION,
+                     "temporal_baseline_days": temporal_baseline_days,
+                     "orbit_number": [int(ref_props_first['orbit']),
+                                      int(sec_props_first['orbit'])],
+                     "platform": [ref_props_first['platform'], sec_props_first['platform']],
+                     "beam_mode": ref_props_first['beamModeType'],
+                     "orbit_direction": ref_props_first['flightDirection'].lower(),
                      "dataset_type": 'slc',
                      "product_type": 'interferogram',
-                     "polarization": "HH",
+                     "polarization": "VV",
                      "look_direction": 'right',
-                     "track_number": int(ref_props['pathNumber']),
+                     "track_number": int(ref_props_first['pathNumber']),
                      "perpendicular_baseline":  round(float(b_perp), 4)
                      })
 
@@ -173,6 +175,9 @@ def format_metadata(nc_path: Path,
             "version": DATASET_VERSION,
             "metadata": metadata}
 
+    if all_metadata['frame_id'] != -1:
+        metadata['frame_number'] = all_metadata['frame_id']
+
     return data
 
 

diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py
@@ -14,7 +14,7 @@
 from isce2_topsapp.packaging_utils.ionosphere import format_ionosphere_for_gunw
 from isce2_topsapp.templates import read_netcdf_packaging_template
 
-DATASET_VERSION = '2.0.6'
+DATASET_VERSION = '3.0.0'
 
 
 PERMISSIBLE_2D_LAYERS = ['ionosphere']

diff --git a/tests/prepare-for-delivery.ipynb b/tests/prepare-for-delivery.ipynb
@@ -6,8 +6,8 @@
    "id": "a9077005",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:48:46.776007Z",
-     "start_time": "2023-02-23T02:48:46.772841Z"
+     "end_time": "2023-03-06T18:45:58.069411Z",
+     "start_time": "2023-03-06T18:45:56.318950Z"
     }
    },
    "outputs": [],
@@ -37,18 +37,31 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "6e4826bf",
+   "id": "2b69cf1f",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:49:14.320081Z",
-     "start_time": "2023-02-23T02:48:47.604876Z"
+     "end_time": "2023-03-06T18:45:58.079450Z",
+     "start_time": "2023-03-06T18:45:58.076986Z"
     }
    },
    "outputs": [],
    "source": [
     "url = 'https://grfn.asf.alaska.edu/door/download/S1-GUNW-A-R-064-tops-20210723_20210711-015001-35393N_33512N-PP-6267-v2_0_4.nc'\n",
-    "nc_path = Path(url.split('/')[-1])\n",
-    "\n",
+    "nc_path = Path(url.split('/')[-1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e4826bf",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-03-06T18:46:20.000997Z",
+     "start_time": "2023-03-06T18:45:58.082664Z"
+    }
+   },
+   "outputs": [],
+   "source": [
     "resp = requests.get(url)\n",
     "with open(nc_path, 'wb') as file:\n",
     "    file.write(resp.content)"
@@ -70,8 +83,8 @@
    "id": "c95b6dd0",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-03T23:48:51.230827Z",
-     "start_time": "2023-02-03T23:48:51.228003Z"
+     "end_time": "2023-03-06T18:46:20.005455Z",
+     "start_time": "2023-03-06T18:46:20.002858Z"
     }
    },
    "outputs": [],
@@ -93,8 +106,8 @@
    "id": "239d2148",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-03T23:48:51.891722Z",
-     "start_time": "2023-02-03T23:48:51.884415Z"
+     "end_time": "2023-03-06T18:46:20.012570Z",
+     "start_time": "2023-03-06T18:46:20.007779Z"
     }
    },
    "outputs": [],
@@ -125,8 +138,8 @@
    "id": "7fd297cb",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-03T23:49:56.049768Z",
-     "start_time": "2023-02-03T23:49:56.043410Z"
+     "end_time": "2023-03-06T18:46:20.023256Z",
+     "start_time": "2023-03-06T18:46:20.014577Z"
     },
     "scrolled": true
    },
@@ -144,8 +157,8 @@
    "id": "aa743e32",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-03T23:53:11.392483Z",
-     "start_time": "2023-02-03T23:53:10.279477Z"
+     "end_time": "2023-03-06T18:46:20.312023Z",
+     "start_time": "2023-03-06T18:46:20.026340Z"
     }
    },
    "outputs": [],
@@ -169,8 +182,8 @@
    "id": "841686cd",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:49:14.327580Z",
-     "start_time": "2023-02-23T02:49:14.322297Z"
+     "end_time": "2023-03-06T18:46:20.318743Z",
+     "start_time": "2023-03-06T18:46:20.314194Z"
     }
    },
    "outputs": [],
@@ -200,8 +213,8 @@
    "id": "982d838b",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:52:00.568443Z",
-     "start_time": "2023-02-23T02:49:14.328921Z"
+     "end_time": "2023-03-06T18:47:55.911191Z",
+     "start_time": "2023-03-06T18:46:20.320764Z"
     }
    },
    "outputs": [],
@@ -215,8 +228,8 @@
    "id": "6b07ba73",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:52:00.584374Z",
-     "start_time": "2023-02-23T02:52:00.571171Z"
+     "end_time": "2023-03-06T18:47:55.917180Z",
+     "start_time": "2023-03-06T18:47:55.912988Z"
     }
    },
    "outputs": [],
@@ -240,8 +253,8 @@
    "id": "81b12194",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:52:10.225364Z",
-     "start_time": "2023-02-23T02:52:10.221367Z"
+     "end_time": "2023-03-06T18:47:55.923057Z",
+     "start_time": "2023-03-06T18:47:55.919248Z"
     }
    },
    "outputs": [],
@@ -256,8 +269,8 @@
    "id": "c863ebae",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2023-02-23T02:53:23.482724Z",
-     "start_time": "2023-02-23T02:53:23.474497Z"
+     "end_time": "2023-03-06T18:47:55.933284Z",
+     "start_time": "2023-03-06T18:47:55.924987Z"
     }
    },
    "outputs": [],
@@ -285,8 +298,8 @@
    "id": "34891199",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2022-01-14T02:33:23.109747Z",
-     "start_time": "2022-01-14T02:33:23.103792Z"
+     "end_time": "2023-03-06T18:47:55.941238Z",
+     "start_time": "2023-03-06T18:47:55.935324Z"
     }
    },
    "outputs": [],

diff --git a/tests/sample_loc_metadata.json b/tests/sample_loc_metadata.json
@@ -15,6 +15,8 @@
     "S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404.zip",
     "S1B_IW_SLC__1SDV_20210711T014922_20210711T014949_027740_034F80_859D.zip"
   ],
+  "frame_id": -1,
+  "temporal_baseline_days": 12,
   "extent": [
     -119.080994,
     33.405682,

diff --git a/tests/test_packaging.py b/tests/test_packaging.py
@@ -1,7 +1,7 @@
 import json
 from pathlib import Path
 
-from isce2_topsapp.packaging import get_gunw_id
+from isce2_topsapp.packaging import DATASET_VERSION, get_gunw_id
 
 test_dir = Path(__file__).parent
 
@@ -12,4 +12,5 @@ def test_gunw_id_generation_crossing_dateline():
     gunw_id = get_gunw_id(metadata['reference_properties'],
                           metadata['secondary_properties'],
                           metadata['extent'])
-    assert gunw_id == 'S1-GUNW-D-R-048-tops-20230106_20221214-235959-00090E_00040N-PP-c254-v2_0_6'
+    version_str = DATASET_VERSION.replace(".", "_")
+    assert gunw_id == f'S1-GUNW-D-R-048-tops-20230106_20221214-235959-00090E_00040N-PP-c254-v{version_str}'