From fadc7a38745efeedd200adc9ef61986199b6cd5f Mon Sep 17 00:00:00 2001 From: MDavidson Date: Thu, 23 Feb 2023 15:31:44 +1300 Subject: [PATCH] feat: update to run for s3://linz-raster-data-store --- tools/generate-argo-cli-commands.py | 362 +++++++++++----------------- tools/test-workflows.sh | 31 --- 2 files changed, 146 insertions(+), 247 deletions(-) delete mode 100755 tools/test-workflows.sh diff --git a/tools/generate-argo-cli-commands.py b/tools/generate-argo-cli-commands.py index c1c4e1df9..3f63914eb 100644 --- a/tools/generate-argo-cli-commands.py +++ b/tools/generate-argo-cli-commands.py @@ -5,120 +5,22 @@ from linz_logger import get_log # nb: CHANGE if working from a different source -SOURCE = "s3://linz-data-lake-raster-prod/" +# SOURCE = "s3://linz-data-lake-raster-prod/" +SOURCE = "s3://linz-raster-data-store/" + PARAMETERS_CSV = "./imagery-standardising-parameters-bulk-process.csv" -PRODUCERS = [ - "Unknown", - "AAM NZ", - "Aerial Surveys", - "Beca", - "GeoSmart", - "Landpro", - "Maxar", - "NZ Aerial Mapping", - "Recon", - "Terralink International", - "UAV Mapping NZ", -] -LICENSORS = [ - "Ashburton District Council", - "Auckland Council", - "BOPLASS", - "Bay of Plenty Regional Council", - "Buller District Council", - "Canterbury Aerial Imagery Consortium (CAI)", - "Carterton District Council", - "Central Hawke's Bay District Council", - "Central Otago District Council", - "Chatham Islands Council", - "Christchurch City Council", - "Clutha District Council", - "CoLAB", - "Department of Conservation", - "Dunedin City Council", - "Environment Canterbury", - "Environment Southland", - "Far North District Council", - "Gisborne District Council", - "Gore District Council", - "Greater Wellington Regional Council", - "Grey District Council", - "Hamilton City Council", - "Hastings District Council", - "Hauraki District Council", - "Hawke's Bay Local Authority Shared Services (HB LASS)", - "Hawke's Bay Regional Council", - "Horizons Regional Council", - "Horowhenua District Council", - "Hurunui District Council", - "Hutt City Council", - "Invercargill City Council", - "Kaikōura District Council", - "Kaipara District Council", - "Kawerau District Council", - "Kāpiti Coast District Council", - "Mackenzie District Council", - "Manawatū District Council", - "Manawatū-Whanganui LASS", - "Marlborough District Council", - "Masterton District Council", - "Matamata-Piako District Council", - "Maxar Technologies", - "Ministry of Primary Industries", - "NZ Aerial Mapping", - "Napier City Council", - "Nelson City Council", - "New Plymouth District Council", - "Northland Regional Council", - "Northland Aerial Imagery Consortium (NAIC)", - "Ōpōtiki District Council", - "Ōtorohanga District Council", - "Otago Regional Council", - "Palmerston North City Council", - "Porirua City Council", - "Queenstown-Lakes District Council", - "Rangitīkei District Council", - "Rotorua District Council", - "Ruapehu District Council", - "Selwyn District Council", - "Sinergise", - "South Taranaki District Council", - "South Waikato District Council", - "South Wairarapa District Council", - "Southland District Council", - "Stratford District Council", - "Taranaki Regional Council", - "Tararua District Council", - "Tasman District Council", - "Taupō District Council", - "Tauranga City Council", - "Terralink International", - "Thames-Coromandel District Council", - "The Southland Consortium", - "Timaru District Council", - "Toitū Te Whenua Land Information New Zealand", - "Upper Hutt City Council", - "Waikato District Council", - "Waikato Regional Aerial Photography Service (WRAPS)", - "Waikato Regional Council", - "Waimakariri District Council", - "Waimate District Council", - "Waipā District Council", - "Wairoa District Council", - "Waitaki District Council", - "Waitomo District Council", - "Waka Kotahi", - "Wellington City Council", - "West Coast Regional Council", - "Western Bay of Plenty District Council", - "Westland District Council", - "Whakatāne District Council", - "Whanganui District Council", - "Whangārei District Council", -] -SCALES = ["500", "1000", "2000", "5000", "10000"] +with open("../workflows/imagery/standardising.yaml", "r") as f: + workflow = yaml.load(f, Loader=yaml.loader.SafeLoader) + for parameter in workflow["spec"]["arguments"]["parameters"]: + if parameter["name"] == "producer": + PRODUCERS = parameter["enum"] + if parameter["name"] == "licensor": + LICENSORS = parameter["enum"] + if parameter["name"] == "scale": + SCALES = parameter["enum"] -command_list = [] +spi_list = [] +sp_list = [] def _format_date(date: str) -> str: @@ -134,12 +36,20 @@ def _validate_licensor(licensor: str) -> Optional[str]: return licensor elif licensor == "BOPLASS Limited": return "BOPLASS" - elif licensor == "Kapiti Coast District Council": + elif licensor == "Kapiti Coast District Council" or licensor == "Kapiti District Council": return "Kāpiti Coast District Council" elif licensor == "The Canterbury Aerial Imagery (CAI) Consortium": return "Canterbury Aerial Imagery Consortium (CAI)" elif licensor == "Hawke's Bay Local Authority Shared Services (HBLASS)": return "Hawke's Bay Local Authority Shared Services (HB LASS)" + elif licensor == "Central Hawkes Bay District Council": + return "Central Hawke's Bay District Council" + elif licensor == "Thames Coromandel District Council": + return "Thames-Coromandel District Council" + elif licensor == "Waikato Regional Aerial Photography Service (WRAPS) 2017-2019": + return "Waikato Regional Aerial Photography Service (WRAPS)" + elif licensor == "Northland Aerial Imagery Consortium (NAIC)": + return "Northland Aerial Imagery Consortium (NAIC)" elif " and " in licensor: return licensor.replace(" and ", ";") return None @@ -156,6 +66,8 @@ def _validate_producer(producer: str) -> Optional[str]: return "AAM NZ" elif producer == "Landpro Ltd": return "Landpro" + elif producer == "UAV Mapping NZ Ltd": + return "UAV Mapping NZ" return None @@ -164,106 +76,124 @@ def _validate_scale(scale: str) -> Optional[str]: return scale return None - -with open(PARAMETERS_CSV, "r") as csv_file: - reader = csv.reader(csv_file) - header = next(reader) - - ind_comment = header.index("Comment") - ind_source = header.index("source") - ind_target = header.index("target") - ind_scale = header.index("scale") - ind_title = header.index("Title") - ind_licensor = header.index("licensor(s)") - ind_producer = header.index("producer(s)") - ind_description = header.index("description") - ind_startdate = header.index("start_datetime") - ind_enddate = header.index("end_datetime") - ind_basemaps = header.index("basemaps s3 path") - - command = "argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./{0}.yaml --generate-name ispi-{1}-\n" - - for row in reader: - if not row[ind_source].startswith(SOURCE): - continue - - if row[ind_comment] != "": - get_log().warning( - "skipped: comment", - comment=row[ind_comment], - source=row[ind_source], - title=row[ind_title], - ) - continue - - if row[ind_basemaps] == "": - get_log().info( - "basemaps import required", - source=row[ind_source], - title=row[ind_title], - ) - params = { - "source": row[ind_source].rstrip("/") + "/", - "target": row[ind_target], - "version-basemaps-cli": "latest", - "version-topo-imagery":"v1.1.0-3-g8fe8211", - "scale": _validate_scale(row[ind_scale]), - "title": row[ind_title], - "description": row[ind_description], - "producer": _validate_producer(row[ind_producer]), - "start-datetime": _format_date(row[ind_startdate]), - "end-datetime": _format_date(row[ind_enddate]), - } - - licensor = _validate_licensor(row[ind_licensor]) - if licensor and ";" in licensor: - params["licensor-list"] = licensor - params["licensor"] = "" - else: - params["licensor"] = licensor - params["licensor-list"] = "" - - if not params["licensor"] and params["licensor-list"] == "": - get_log().warning( - "skipped: invalid licensor", - licensor=row[ind_licensor], - source=row[ind_source], - title=row[ind_title], - ) - continue - - if not params["producer"]: - get_log().warning( - "skipped: invalid producer", - producer=row[ind_producer], - source=row[ind_source], - title=row[ind_title], - ) - continue - - if not params["scale"]: - get_log().warning( - "skipped: invalid scale", - scale=f"{row[ind_scale]}", - source=row[ind_source], - title=row[ind_title], - ) - continue - - file_name = row[ind_target].rstrip("/rgb/2193/").split("/")[-1] - command_list.append(command.format(file_name, file_name.replace("_", "-").replace(".", "-"))) - - with open(f"./{file_name}.yaml", "w", encoding="utf-8") as output: - yaml.dump( - params, - output, - default_flow_style=False, - default_style='"', - sort_keys=False, - allow_unicode=True, - width=1000, - ) - -with open("standardise-publish.sh", "w") as script: - script.write("#!/bin/bash\n\n") - script.writelines(command_list) +def main() -> None: + with open(PARAMETERS_CSV, "r") as csv_file: + reader = csv.reader(csv_file) + header = next(reader) + + ind_comment = header.index("Comment") + ind_source = header.index("source") + ind_target = header.index("target") + ind_scale = header.index("scale") + ind_title = header.index("Title") + ind_licensor = header.index("licensor(s)") + ind_producer = header.index("producer(s)") + ind_description = header.index("description") + ind_startdate = header.index("start_datetime") + ind_enddate = header.index("end_datetime") + ind_basemaps = header.index("basemaps s3 path") + + command = "argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./{0}.yaml --generate-name ispi-{1}-\n" + + for row in reader: + if not row[ind_source].startswith(SOURCE): + continue + + if row[ind_comment] != "": + get_log().warning( + "skipped: comment", + comment=row[ind_comment], + source=row[ind_source], + title=row[ind_title], + ) + continue + + params = { + "source": row[ind_source].rstrip("/") + "/", + "target": row[ind_target], + "version-basemaps-cli": "latest", + "version-topo-imagery":"v1.1.0-3-g8fe8211", + "scale": _validate_scale(row[ind_scale]), + "title": row[ind_title], + "description": row[ind_description], + "producer": _validate_producer(row[ind_producer]), + "start-datetime": _format_date(row[ind_startdate]), + "end-datetime": _format_date(row[ind_enddate]), + } + + licensor = _validate_licensor(row[ind_licensor]) + if licensor and ";" in licensor: + params["licensor-list"] = licensor + params["licensor"] = "" + else: + params["licensor"] = licensor + params["licensor-list"] = "" + + if not params["licensor"] and params["licensor-list"] == "": + get_log().warning( + "skipped: invalid licensor", + licensor=row[ind_licensor], + source=row[ind_source], + title=row[ind_title], + ) + continue + + if not params["producer"]: + get_log().warning( + "skipped: invalid producer", + producer=row[ind_producer], + source=row[ind_source], + title=row[ind_title], + ) + continue + + if not params["scale"]: + get_log().warning( + "skipped: invalid scale", + scale=f"{row[ind_scale]}", + source=row[ind_source], + title=row[ind_title], + ) + continue + + file_name = row[ind_target].rstrip("/rgb/2193/").split("/")[-1] + + if row[ind_basemaps] == "": + get_log().info( + "basemaps import required", + source=row[ind_source], + title=row[ind_title], + ) + bm_params = { + "category": "Urban Aerial Photos", + "name": params["target"].rstrip("/rgb/2193/").split("/")[-1], + "tile-matrix": "NZTM2000Quad/WebMercatorQuad", + "blend": "20", + "aligned-level": "6", + "create-pull-request": "true" + } + params = {**params, **bm_params} + spi_list.append(command.format(file_name, file_name.replace("_", "-").replace(".", "-"))) + else: + sp_list.append(command.format(file_name, file_name.replace("_", "-").replace(".", "-"))) + + with open(f"./{file_name}.yaml", "w", encoding="utf-8") as output: + yaml.dump( + params, + output, + default_flow_style=False, + default_style='"', + sort_keys=False, + allow_unicode=True, + width=1000, + ) + + with open("standardise-publish.sh", "w") as script: + script.write("#!/bin/bash\n\n") + script.writelines(sp_list) + + with open("standardise-publish-import.sh", "w") as script: + script.write("#!/bin/bash\n\n") + script.writelines(spi_list) + +main() diff --git a/tools/test-workflows.sh b/tools/test-workflows.sh deleted file mode 100755 index c73224a71..000000000 --- a/tools/test-workflows.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./christchurch_2018-2019_0.075m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./gisborne_2012-2013_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./bay-of-plenty_2011-2012_0.25m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./auckland_2010-2012_0.5m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./waikato_2012-2013_0.5m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./tasman_2001-2002_1m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./canterbury_2012-2013_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./selwyn_2012-2013_0.125m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./timaru_2012-2013_0.075m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./southland_2005-2011_0.75m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./waimakariri_2013-2014_0.075m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./dunedin_2013_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./otago_2013-2014_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./tasman_2009-2010_0.5m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./tasman_2004-2005_1m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./tasman_2003_1m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./hurunui_2013_0.125m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./canterbury_2013-2014_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./southland-central-otago_2013-2014_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./southland-central-otago_2015-2017_0.4m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./canterbury_2014-2015_0.3m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./waikato-district_2014_0.1m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./northland_2014-2015_0.1m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./wairoa_2014-2015_0.1m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./hawkes-bay_2014-2015_0.3m.yaml -# argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./christchurch_2015-2016_0.075m.yaml -argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./waimakariri_2015-2016_0.075m.yaml --generate-name ispi-waimakariri-2015-2016-0-075m- -argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./canterbury_2015-2016_0.3m.yaml --generate-name ispi-canterbury-2015-2016-0-3m- -argo submit ~/dev/topo-workflows/workflows/imagery/standardising-publish-import.yaml -n argo -f ./marlborough_2015-2016_0.2m.yaml --generate-name ispi-marlborough-2015-2016-0-2m-