From 0c0ccc074f9d00e5d06c8cb92e173020b9dbf868 Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Thu, 2 Nov 2023 09:19:20 +0100 Subject: [PATCH] update cwl-workflow --- .../app-water-bodies-cloud-native.cwl | 8 ++--- docs/cwl-workflow/cloud-native.md | 6 ++-- docs/cwl-workflow/scatter-cloud-native.md | 5 ++- docs/cwl-workflow/stage-out.md | 4 ++- docs/cwl-workflow/staged.md | 34 +++++++++++++++++++ scripts/exec-app-water-body.sh | 2 +- 6 files changed, 49 insertions(+), 10 deletions(-) diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index 4b9e1c8..ab50004 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -120,7 +120,7 @@ $graph: ramMax: 512 hints: DockerRequirement: - dockerPull: ghcr.io/terradue/ogc-eo-application-package-hands-on/crop:bids23 + dockerPull: localhost/crop:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: @@ -158,7 +158,7 @@ $graph: ramMax: 512 hints: DockerRequirement: - dockerPull: ghcr.io/terradue/ogc-eo-application-package-hands-on/norm_diff:bids23 + dockerPull: localhost/norm_diff:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: @@ -184,7 +184,7 @@ $graph: ramMax: 512 hints: DockerRequirement: - dockerPull: ghcr.io/terradue/ogc-eo-application-package-hands-on/otsu:bids23 + dockerPull: localhost/otsu:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: @@ -210,7 +210,7 @@ $graph: ramMax: 512 hints: DockerRequirement: - dockerPull: ghcr.io/terradue/ogc-eo-application-package-hands-on/stac:bids23 + dockerPull: localhost/stac:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: diff --git a/docs/cwl-workflow/cloud-native.md b/docs/cwl-workflow/cloud-native.md index 090c47c..56d1722 100644 --- a/docs/cwl-workflow/cloud-native.md +++ b/docs/cwl-workflow/cloud-native.md @@ -19,12 +19,12 @@ subgraph scatter on bands P --> B(("crop(green)")) P --> C(("crop(nir)")) Q --> B(("crop(green)")) - Q --> C(("crop(green)")) + Q --> C(("crop(nir)")) R --> B(("crop(green)")) - R --> C(("crop(green)")) + R --> C(("crop(nir)")) end B(("crop(green)")) --> D -C(("crop(green)")) --> D +C(("crop(nir)")) --> D D(("`Normalized difference`")); D --> E(("`Otsu diff --git a/docs/cwl-workflow/scatter-cloud-native.md b/docs/cwl-workflow/scatter-cloud-native.md index 8f30f5d..a0a50d0 100644 --- a/docs/cwl-workflow/scatter-cloud-native.md +++ b/docs/cwl-workflow/scatter-cloud-native.md @@ -1,10 +1,12 @@ +CWL can run sub-workflows in a step. Below a CWL Workflow created to process a list of STAC Items and then generate a STAC catalog with several detected water bodies. -```yaml linenums="1" hl_lines="33-43" +```yaml linenums="1" hl_lines="18-21 33-43" --8<-- cwl-workflow/app-water-bodies-cloud-native.cwl:8:58 --8<-- ``` +The `stac` CommandLineTool is updated to manage arrays: ```yaml linenums="200" hl_lines="19-23 25-29" --8<-- @@ -12,6 +14,7 @@ cwl-workflow/app-water-bodies-cloud-native.cwl:200:234 --8<-- ``` +To run this CWL document, one would do: ```bash --8<-- diff --git a/docs/cwl-workflow/stage-out.md b/docs/cwl-workflow/stage-out.md index 0237da5..afb34d9 100644 --- a/docs/cwl-workflow/stage-out.md +++ b/docs/cwl-workflow/stage-out.md @@ -43,13 +43,15 @@ It uses a custom STAC I/O to facilitate S3 interactions. The AWS credentials and endpoint URL need to be properly configured in the environment variables for the script to work. A - ```bash linenums="1" hl_lines="8-71" --8<-- scripts/build-stage-container.sh --8<-- + ``` +To run the stage-out step, one would run: + ```bash linenums="1" hl_lines="8-71" --8<-- scripts/cwl-cli-stage-out.sh diff --git a/docs/cwl-workflow/staged.md b/docs/cwl-workflow/staged.md index a0cb121..ad587a6 100644 --- a/docs/cwl-workflow/staged.md +++ b/docs/cwl-workflow/staged.md @@ -1,4 +1,38 @@ +The non-Cloud native Workflow chains the `crop`, `norm_diff`, `otsu` and `stac` steps taking a staged EO acquisition as a STAC Catalog in a directory as input parameters: +* a local STAC Catalog in a directory +* a bounding box area of interest (AOI) +* the EPSG code of the bounding box area of interest +* a list of common band names (["green", "nir08"]) + +``` mermaid +graph TB +S[(local storage)] +S -- STAC Catalog --> B(("crop(green)")); +S -- STAC Catalog --> C(("crop(nir)")); +S -- STAC Catalog --> F +P[bands] +Q[EPSG code] +R[AOI] +subgraph scatter on bands + P --> B(("crop(green)")) + P --> C(("crop(nir08)")) + Q --> B(("crop(green)")) + Q --> C(("crop(nir08)")) + R --> B(("crop(green)")) + R --> C(("crop(nir08)")) +end +B(("crop(green)")) --> D +C(("crop(nir08)")) --> D +D(("`Normalized +difference`")); +D --> E(("`Otsu + threshold`")) +E --> F(("`Create +STAC Catalog`")) +``` + +The CWL Workflow is shown below and the lines highlighted show the changes to manage a staged STAC Catalog instead of an URL to a STAC Item: ```yaml linenums="1" hl_lines="32 90 180" title="app-water-body.cwl" --8<-- diff --git a/scripts/exec-app-water-body.sh b/scripts/exec-app-water-body.sh index b5272bc..caae5c7 100644 --- a/scripts/exec-app-water-body.sh +++ b/scripts/exec-app-water-body.sh @@ -8,4 +8,4 @@ cwltool \ --aoi="-118.985,38.432,-118.183,38.938" \ --epsg "EPSG:4326" \ --bands green \ - --bands nir08 > results.json \ No newline at end of file + --bands nir08 \ No newline at end of file