Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update metadata with frame_id, version and temporal_baseline_days #102

Merged
merged 21 commits into from
May 27, 2023
Merged

Update metadata with frame_id, version and temporal_baseline_days #102

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
6a72fce
update metadata w/ frame_id and secondary dt
cmarshak Feb 13, 2023
9b68415
update changelog; fix version num
cmarshak Feb 13, 2023
70ccb38
add frame-id to test data
cmarshak Feb 13, 2023
07ddf91
Update delivery_prep.py
cmarshak Feb 14, 2023
5a4617b
temporal baseline and frame id
cmarshak Feb 27, 2023
add751c
Merge branch 'dev' into frame-metadata
cmarshak Feb 28, 2023
dff9012
Merge branch 'fix_variable_name_error' into frame-metadata
cmarshak Feb 28, 2023
f28b198
fix datetime types
cmarshak Feb 28, 2023
8bf78a7
remove duplicate SET line
cmarshak Feb 28, 2023
772abf2
merge dev
cmarshak Mar 6, 2023
257c7b8
ensure start time is earliest
cmarshak Mar 6, 2023
6e1cc3b
split url and download cells
cmarshak Mar 6, 2023
47a52d7
ensure midnight crossing is correctly handled per DB
cmarshak Mar 6, 2023
a989999
Merge branch 'dev' into frame-metadata
cmarshak Mar 23, 2023
7c72c95
increase dataset version 3.0.0, include version in cmr metadata, and …
cmarshak Mar 23, 2023
cf387a7
dynamically update dataset version for test
cmarshak Mar 23, 2023
fd73e06
lint
cmarshak Mar 23, 2023
4b8019a
andrews suggestion
cmarshak Mar 29, 2023
7ccb526
revert localized data
cmarshak Mar 29, 2023
65f6c9d
Merge branch 'dev' into frame-metadata
cmarshak May 25, 2023
98dadaf
Update __main__.py with water mask
cmarshak May 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
* Exposes `frame-id` parameter for fixed frame cropping. Discussion, references, and examples in README.
* Latitude aligned frames and their expected extents are added as geojson in repository as zip file.
* Pins ISCE2 version to 2.6.1 and numpy / scipy to previous versions (see environment.yml) - to be amended when newest ISCE2 build is sorted out
* Includes `frame_id` and `temporal_baseline_days` in json metadata for CMR handshake. The former is the fixed frame id and the latter is the number of days between images (will be multiple of 6).
* Added support to compute and embed solid earth tide correction layers into GUNW products (see PR #91) - reference and secondary have own groups
* Raises warning if there is at least 80% of water in the IFG area using Natural Earth Land mask.

Expand Down
1 change: 1 addition & 0 deletions isce2_topsapp/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def localize_data(reference_scenes: list,

out = {'reference_scenes': reference_scenes,
'secondary_scenes': secondary_scenes,
'frame_id': frame_id,
**out_slc,
**out_dem,
**out_aux_cal,
Expand Down
64 changes: 33 additions & 31 deletions isce2_topsapp/delivery_prep.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import numpy as np
import rasterio
from PIL import Image
from dateparser import parse
from matplotlib import cm

from isce2_topsapp.packaging import DATASET_VERSION
Expand Down Expand Up @@ -113,37 +114,36 @@ def gen_browse_imagery(nc_path: Path,
return out_path


def format_time_string(time_str: str) -> str:
"""Generates formatted string for ingest schema using built in python manipulations.
New ASF API provides time string as 2022-05-20T07:11:01.000Z and we need 6 decimal places
at end of string, specifically, 2022-05-20T07:11:01.000000Z
"""
assert time_str[-1] == 'Z'

# removes Z at end and focuses in on decimal
temp = time_str[:-1].split('.')

# update the split string with required decimals
temp[-1] = f'{int(temp[-1]):06d}'

# rejoin and return
formatted_time_str = '.'.join(temp)
return f'{formatted_time_str}Z'


def format_metadata(nc_path: Path,
all_metadata: dict) -> dict:

now = datetime.datetime.now()
label = nc_path.name[:-3] # removes suffix .nc
geojson = all_metadata['gunw_geo'].__geo_interface__

ref_props = all_metadata['reference_properties'][0]
sec_props = all_metadata['secondary_properties'][0]
ref_props_all = sorted(all_metadata['reference_properties'],
key=lambda prop: prop['startTime'])
ref_props_first = ref_props_all[0]
sec_props_all = sorted(all_metadata['secondary_properties'],
key=lambda prop: prop['startTime'])
sec_props_first = sec_props_all[0]
b_perp = read_baseline_perp(nc_path).mean()

startTime_f = format_time_string(ref_props["startTime"])
stopTime_f = format_time_string(ref_props["stopTime"])
ref_start_times = [parse(props['startTime']) for props in ref_props_all]
ref_stop_times = [parse(props['stopTime']) for props in ref_props_all]
sec_start_times = [parse(props['startTime']) for props in sec_props_all]

ref_start_time = ref_start_times[0]
ref_stop_time = ref_stop_times[-1]
sec_start_time = sec_start_times[0]

# The %f is miliseconds zero-padded with 6 decimals - just as we need!
ref_start_time_formatted = ref_start_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
ref_stop_time_formatted = ref_stop_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')

# We want the nearest day (dt.days takes a floor) so we use total seconds and then round
temporal_baseline_seconds = (ref_start_time - sec_start_time).total_seconds()
temporal_baseline_days = round(temporal_baseline_seconds / 60 / 60 / 24)
Comment on lines +143 to +146
Copy link
Collaborator Author

@cmarshak cmarshak Mar 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Providing some context for @dbekaert important comments about obtaining an accurate temporal baseline in days. We expect this temporal baseline in days to be a multiple of 6 based on S1's repeat pass frequency.

Suppose the temporal baseline (in days) is for example 5.999999. If delta is a datetime.timedelta object, then delta.days will be 5 (not 6). Therefore, we want to do the rounding using total_seconds(). This resolves David's concerns.

Here is an example:

import datetime
d0 = datetime.datetime(2021, 1, 1, 0, 0, 0)
d1 = datetime.datetime(2021, 1, 6, 23, 59, 59)
(d1 - d0).days

Will be 5.

To be clear, this concern can occur anywhere and anytime, not just at pairs in which acquisitions are close to midnight in UTC.


metadata = {}
# get 4 corners of bounding box of the geometry; default is 5 returning
Expand All @@ -152,18 +152,20 @@ def format_metadata(nc_path: Path,
metadata.update({"ogr_bbox": ogr_bbox,
"reference_scenes": all_metadata['reference_scenes'],
"secondary_scenes": all_metadata['secondary_scenes'],
"sensing_start": startTime_f,
"sensing_stop": stopTime_f,
"orbit_number": [int(ref_props['orbit']),
int(sec_props['orbit'])],
"platform": [ref_props['platform'], sec_props['platform']],
"beam_mode": ref_props['beamModeType'],
"orbit_direction": ref_props['flightDirection'].lower(),
"sensing_start": ref_start_time_formatted,
"sensing_stop": ref_stop_time_formatted,
"frame_id": all_metadata['frame_id'],
"temporal_baseline_days": temporal_baseline_days,
"orbit_number": [int(ref_props_first['orbit']),
int(sec_props_first['orbit'])],
"platform": [ref_props_first['platform'], sec_props_first['platform']],
"beam_mode": ref_props_first['beamModeType'],
"orbit_direction": ref_props_first['flightDirection'].lower(),
"dataset_type": 'slc',
"product_type": 'interferogram',
"polarization": "HH",
"polarization": "VV",
"look_direction": 'right',
"track_number": int(ref_props['pathNumber']),
"track_number": int(ref_props_first['pathNumber']),
"perpendicular_baseline": round(float(b_perp), 4)
})

Expand Down
67 changes: 40 additions & 27 deletions tests/prepare-for-delivery.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
"id": "a9077005",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:48:46.776007Z",
"start_time": "2023-02-23T02:48:46.772841Z"
"end_time": "2023-03-06T18:45:58.069411Z",
"start_time": "2023-03-06T18:45:56.318950Z"
}
},
"outputs": [],
Expand Down Expand Up @@ -37,18 +37,31 @@
{
"cell_type": "code",
"execution_count": null,
"id": "6e4826bf",
"id": "2b69cf1f",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:49:14.320081Z",
"start_time": "2023-02-23T02:48:47.604876Z"
"end_time": "2023-03-06T18:45:58.079450Z",
"start_time": "2023-03-06T18:45:58.076986Z"
}
},
"outputs": [],
"source": [
"url = 'https://grfn.asf.alaska.edu/door/download/S1-GUNW-A-R-064-tops-20210723_20210711-015001-35393N_33512N-PP-6267-v2_0_4.nc'\n",
"nc_path = Path(url.split('/')[-1])\n",
"\n",
"nc_path = Path(url.split('/')[-1])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6e4826bf",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-06T18:46:20.000997Z",
"start_time": "2023-03-06T18:45:58.082664Z"
}
},
"outputs": [],
"source": [
"resp = requests.get(url)\n",
"with open(nc_path, 'wb') as file:\n",
" file.write(resp.content)"
Expand All @@ -70,8 +83,8 @@
"id": "c95b6dd0",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-03T23:48:51.230827Z",
"start_time": "2023-02-03T23:48:51.228003Z"
"end_time": "2023-03-06T18:46:20.005455Z",
"start_time": "2023-03-06T18:46:20.002858Z"
}
},
"outputs": [],
Expand All @@ -93,8 +106,8 @@
"id": "239d2148",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-03T23:48:51.891722Z",
"start_time": "2023-02-03T23:48:51.884415Z"
"end_time": "2023-03-06T18:46:20.012570Z",
"start_time": "2023-03-06T18:46:20.007779Z"
}
},
"outputs": [],
Expand Down Expand Up @@ -125,8 +138,8 @@
"id": "7fd297cb",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-03T23:49:56.049768Z",
"start_time": "2023-02-03T23:49:56.043410Z"
"end_time": "2023-03-06T18:46:20.023256Z",
"start_time": "2023-03-06T18:46:20.014577Z"
},
"scrolled": true
},
Expand All @@ -144,8 +157,8 @@
"id": "aa743e32",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-03T23:53:11.392483Z",
"start_time": "2023-02-03T23:53:10.279477Z"
"end_time": "2023-03-06T18:46:20.312023Z",
"start_time": "2023-03-06T18:46:20.026340Z"
}
},
"outputs": [],
Expand All @@ -169,8 +182,8 @@
"id": "841686cd",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:49:14.327580Z",
"start_time": "2023-02-23T02:49:14.322297Z"
"end_time": "2023-03-06T18:46:20.318743Z",
"start_time": "2023-03-06T18:46:20.314194Z"
}
},
"outputs": [],
Expand Down Expand Up @@ -200,8 +213,8 @@
"id": "982d838b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:52:00.568443Z",
"start_time": "2023-02-23T02:49:14.328921Z"
"end_time": "2023-03-06T18:47:55.911191Z",
"start_time": "2023-03-06T18:46:20.320764Z"
}
},
"outputs": [],
Expand All @@ -215,8 +228,8 @@
"id": "6b07ba73",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:52:00.584374Z",
"start_time": "2023-02-23T02:52:00.571171Z"
"end_time": "2023-03-06T18:47:55.917180Z",
"start_time": "2023-03-06T18:47:55.912988Z"
}
},
"outputs": [],
Expand All @@ -240,8 +253,8 @@
"id": "81b12194",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:52:10.225364Z",
"start_time": "2023-02-23T02:52:10.221367Z"
"end_time": "2023-03-06T18:47:55.923057Z",
"start_time": "2023-03-06T18:47:55.919248Z"
}
},
"outputs": [],
Expand All @@ -256,8 +269,8 @@
"id": "c863ebae",
"metadata": {
"ExecuteTime": {
"end_time": "2023-02-23T02:53:23.482724Z",
"start_time": "2023-02-23T02:53:23.474497Z"
"end_time": "2023-03-06T18:47:55.933284Z",
"start_time": "2023-03-06T18:47:55.924987Z"
}
},
"outputs": [],
Expand Down Expand Up @@ -285,8 +298,8 @@
"id": "34891199",
"metadata": {
"ExecuteTime": {
"end_time": "2022-01-14T02:33:23.109747Z",
"start_time": "2022-01-14T02:33:23.103792Z"
"end_time": "2023-03-06T18:47:55.941238Z",
"start_time": "2023-03-06T18:47:55.935324Z"
}
},
"outputs": [],
Expand Down
2 changes: 2 additions & 0 deletions tests/sample_loc_metadata.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
"S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404.zip",
"S1B_IW_SLC__1SDV_20210711T014922_20210711T014949_027740_034F80_859D.zip"
],
"frame_id": -1,
"temporal_baseline_days": 12,
"extent": [
-119.080994,
33.405682,
Expand Down