Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP - search and download S1 and HLS DSWx files #2

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+1,084 −0
Open

WIP - search and download S1 and HLS DSWx files #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
672b3b2
WIP - search and download S1 and HLS DSWx files
seyeonjeon Nov 13, 2024
fcaac62
WIP - Fill cloud pixels using Naive Bayes paper
seyeonjeon Nov 27, 2024
55896ca
WIP - modify 3d operations into 2d, functionalize
seyeonjeon Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
323 changes: 323 additions & 0 deletions notebook/dswx_search.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,323 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"import datetime\n",
"import logging\n",
"import os\n",
"import glob \n",
"from datetime import datetime, timedelta\n",
"\n",
"from pathlib import Path\n",
"from osgeo import gdal, osr\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib.colors import ListedColormap\n",
"import numpy as np\n",
"from pyproj import Transformer\n",
"from sklearn.metrics import cohen_kappa_score\n",
"from shapely.geometry import LinearRing, Point, Polygon, box\n",
"import rasterio\n",
"import requests\n",
"from pprint import pprint"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"def download_dswx_hls(polys, \n",
" datetime_start_str, \n",
" datetime_end_str, \n",
" MGRS_tile,\n",
" download_folder,\n",
" download_flag=True):\n",
" CMR_OPS = 'https://cmr.earthdata.nasa.gov/search'\n",
" url = f'{CMR_OPS}/{\"granules\"}'\n",
" boundind_box = polys.bounds\n",
" provider = 'POCLOUD'\n",
" parameters = {'temporal': f'{datetime_start_str},{datetime_end_str}',\n",
" 'concept_id': 'C2617126679-POCLOUD', #HLS\n",
" #'concept_id': 'C2949811996-POCLOUD', #S1\n",
" 'provider': provider,\n",
" #'bounding_box': f'{boundind_box[1]+0.2},{boundind_box[0]+0.2},{boundind_box[3]-0.2},{boundind_box[2]-0.2}',\n",
" 'bounding_box': f'{boundind_box[1]},{boundind_box[0]},{boundind_box[3]},{boundind_box[2]}',\n",
" 'page_size': 200,}\n",
" print(parameters)\n",
" response = requests.get(url,\n",
" params=parameters,\n",
" headers={\n",
" 'Accept': 'application/json'\n",
" }\n",
" )\n",
" print(response.status_code)\n",
" print(response.headers['CMR-Hits'])\n",
" downloaded_list = []\n",
" num_search_data = response.headers['CMR-Hits']\n",
" number_hls_data = 0\n",
" if num_search_data:\n",
" collections = response.json()['feed']['entry']\n",
" # print('collection', collections)\n",
" for collection in collections:\n",
" dswx_hls_file_id = collection['producer_granule_id']\n",
" if MGRS_tile in dswx_hls_file_id:\n",
" print('url', f'{collection[\"links\"][4][\"href\"]}')\n",
" print('s3', f'{collection[\"links\"][3][\"href\"]}')\n",
" if download_flag:\n",
" for layer in [2, 4, 6]: # 2: WTR, 4: BWTR, 6:CONF\n",
" dswx_hls_url = collection[\"links\"][layer][\"href\"]\n",
" dswx_hls_filename = os.path.basename(dswx_hls_url)\n",
" download_file = f'{download_folder}/{dswx_hls_filename}'\n",
"\n",
" if not os.path.isfile(download_file):\n",
" response = requests.get(dswx_hls_url, stream=True)\n",
"\n",
" # Check if the request was successful\n",
" if response.status_code == 200:\n",
" # Open a local file with wb (write binary) permission.\n",
" with open(f'{download_file}', 'wb') as file:\n",
" print('downloading')\n",
" for chunk in response.iter_content(chunk_size=128):\n",
" file.write(chunk)\n",
" else:\n",
" print('under dev.')\n",
" downloaded_list.append(download_file)\n",
" number_hls_data += 1\n",
"\n",
" else:\n",
" print('MGRS tile id does not match')\n",
"\n",
" return number_hls_data, downloaded_list"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'temporal': '2024-10-01T00:00:00Z,2024-10-13T00:00:00Z', 'concept_id': 'C2949811996-POCLOUD', 'provider': 'POCLOUD', 'bounding_box': '-123.1,37.9,-121.9,38.0', 'page_size': 200}\n",
"200\n",
"24\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"url https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241003T020832Z_20241003T103817Z_S1A_30_v1.0_B02_BWTR.tif\n",
"s3 s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241003T020832Z_20241003T103817Z_S1A_30_v1.0_B01_WTR.tif\n",
"downloading\n",
"downloading\n",
"downloading\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"url https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241003T140755Z_20241003T211205Z_S1A_30_v1.0_B02_BWTR.tif\n",
"s3 s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241003T140755Z_20241003T211205Z_S1A_30_v1.0_B01_WTR.tif\n",
"downloading\n",
"downloading\n",
"downloading\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"url https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241008T141619Z_20241008T201644Z_S1A_30_v1.0_B02_BWTR.tif\n",
"s3 s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241008T141619Z_20241008T201644Z_S1A_30_v1.0_B01_WTR.tif\n",
"downloading\n",
"downloading\n",
"downloading\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"MGRS tile id does not match\n",
"url https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241010T020037Z_20241010T184256Z_S1A_30_v1.0_B02_BWTR.tif\n",
"s3 s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T10SEH_20241010T020037Z_20241010T184256Z_S1A_30_v1.0_B01_WTR.tif\n",
"downloading\n",
"downloading\n",
"downloading\n"
]
},
{
"data": {
"text/plain": [
"(4,\n",
" ['/u/aurora-r0/jeon/tools/DSWx-fusion/DSWx-Enhanced-Analysis/download_10SEH/OPERA_L3_DSWx-S1_T10SEH_20241003T020832Z_20241003T103817Z_S1A_30_v1.0_B03_CONF.tif',\n",
" '/u/aurora-r0/jeon/tools/DSWx-fusion/DSWx-Enhanced-Analysis/download_10SEH/OPERA_L3_DSWx-S1_T10SEH_20241003T140755Z_20241003T211205Z_S1A_30_v1.0_B03_CONF.tif',\n",
" '/u/aurora-r0/jeon/tools/DSWx-fusion/DSWx-Enhanced-Analysis/download_10SEH/OPERA_L3_DSWx-S1_T10SEH_20241008T141619Z_20241008T201644Z_S1A_30_v1.0_B03_CONF.tif',\n",
" '/u/aurora-r0/jeon/tools/DSWx-fusion/DSWx-Enhanced-Analysis/download_10SEH/OPERA_L3_DSWx-S1_T10SEH_20241010T020037Z_20241010T184256Z_S1A_30_v1.0_B03_CONF.tif'])"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#poly_cand = box(64.90, -148, 65,-147)\n",
"poly_cand = box(37.9,-123.1, 38.0,-121.9)\n",
"\n",
"datetime_start_str = '2024-10-01T00:00:00Z'\n",
"datetime_end_str = '2024-10-13T00:00:00Z'\n",
"#mgrs_tile = '06WWT'\n",
"mgrs_tile = '10SEH'\n",
"download_folder = '/u/aurora-r0/jeon/tools/DSWx-fusion/DSWx-Enhanced-Analysis/download_' + mgrs_tile\n",
"# formats_output = '%Y-%m-%dT%H:%M:%SZ'\n",
"\n",
"download_dswx_hls(poly_cand, datetime_start_str, datetime_end_str, mgrs_tile, download_folder) "
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"def download_dswx_s1(polys, \n",
" datetime_start_str, \n",
" datetime_end_str, \n",
" MGRS_tile,\n",
" download_folder,\n",
" download_flag=True):\n",
" CMR_OPS = 'https://cmr.earthdata.nasa.gov/search'\n",
" url = f'{CMR_OPS}/{\"granules\"}'\n",
" boundind_box = polys.bounds\n",
" provider = 'POCLOUD'\n",
" parameters = {'temporal': f'{datetime_start_str},{datetime_end_str}',\n",
" #'concept_id': 'C2617126679-POCLOUD', #HLS\n",
" 'concept_id': 'C2949811996-POCLOUD', #S1\n",
" 'provider': provider,\n",
" #'bounding_box': f'{boundind_box[1]+0.2},{boundind_box[0]+0.2},{boundind_box[3]-0.2},{boundind_box[2]-0.2}',\n",
" 'bounding_box': f'{boundind_box[1]},{boundind_box[0]},{boundind_box[3]},{boundind_box[2]}',\n",
" 'page_size': 200,}\n",
" print(parameters)\n",
" response = requests.get(url,\n",
" params=parameters,\n",
" headers={\n",
" 'Accept': 'application/json'\n",
" }\n",
" )\n",
" print(response.status_code)\n",
" print(response.headers['CMR-Hits'])\n",
" downloaded_list = []\n",
" num_search_data = response.headers['CMR-Hits']\n",
" number_hls_data = 0\n",
" if num_search_data:\n",
" collections = response.json()['feed']['entry']\n",
" # print('collection', collections)\n",
" for collection in collections:\n",
" dswx_hls_file_id = collection['producer_granule_id']\n",
" if MGRS_tile in dswx_hls_file_id:\n",
" print('url', f'{collection[\"links\"][4][\"href\"]}')\n",
" print('s3', f'{collection[\"links\"][3][\"href\"]}')\n",
" if download_flag:\n",
" for layer in [2, 4, 6]: # 2: WTR, 4: BWTR, 6:CONF\n",
" dswx_hls_url = collection[\"links\"][layer][\"href\"]\n",
" dswx_hls_filename = os.path.basename(dswx_hls_url)\n",
" download_file = f'{download_folder}/{dswx_hls_filename}'\n",
"\n",
" if not os.path.isfile(download_file):\n",
" response = requests.get(dswx_hls_url, stream=True)\n",
"\n",
" # Check if the request was successful\n",
" if response.status_code == 200:\n",
" # Open a local file with wb (write binary) permission.\n",
" with open(f'{download_file}', 'wb') as file:\n",
" print('downloading')\n",
" for chunk in response.iter_content(chunk_size=128):\n",
" file.write(chunk)\n",
" else:\n",
" print('under dev.')\n",
" downloaded_list.append(download_file)\n",
" number_hls_data += 1\n",
"\n",
" else:\n",
" print('MGRS tile id does not match')\n",
"\n",
" return number_hls_data, downloaded_list"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'temporal': '2024-10-01T00:00:00Z,2024-10-13T00:00:00Z', 'concept_id': 'C2949811996-POCLOUD', 'provider': 'POCLOUD', 'bounding_box': '-148.0,64.9,-147.0,65.0', 'page_size': 200}\n",
"200\n",
"11\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WWS_20241007T032104Z_20241007T102813Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WVS_20241007T032104Z_20241007T102813Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WWT_20241007T032203Z_20241007T113244Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WVT_20241007T032203Z_20241007T113244Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WVT_20241009T162821Z_20241010T002418Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WWT_20241009T162821Z_20241010T002418Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WVS_20241009T162854Z_20241010T002539Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WWT_20241011T161201Z_20241011T222258Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WVT_20241011T161201Z_20241011T222258Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WWS_20241011T161301Z_20241011T230713Z_S1A_30_v1.0_B02_BWTR.tif\n",
"https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-S1_V1/OPERA_L3_DSWx-S1_T06WVS_20241011T161301Z_20241011T230713Z_S1A_30_v1.0_B02_BWTR.tif\n"
]
},
{
"data": {
"text/plain": [
"'11'"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#poly_cand = box(64.90, -148, 65,-147)\n",
"poly_cand = box(37.9,-123.1, 38.0,-121.9)\n",
"\n",
"datetime_start_str = '2024-10-01T00:00:00Z'\n",
"datetime_end_str = '2024-10-13T00:00:00Z'\n",
"#mgrs_tile = '06WWT'\n",
"mgrs_tile = '10SEH'\n",
"download_folder = '/u/aurora-r0/jeon/tools/DSWx-fusion/DSWx-Enhanced-Analysis/download_' + mgrs_tile\n",
"# formats_output = '%Y-%m-%dT%H:%M:%SZ'\n",
"\n",
"download_dswx_s1(poly_cand, datetime_start_str, datetime_end_str, mgrs_tile, download_folder) "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "dswx-env",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Loading