Merge pull request #16 from SergioLangaritaBenitez/main

update OSCAR workflow

DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs.yaml

@@ -7,6 +7,7 @@ functions:
       image: deltares/sfincs-cpu:sfincs-v2.1.1-Dollerup-Release
       script: sfincs_script.sh
       log_level: DEBUG   # To avoid supervisor logs in response
+      vo: dev.intertwin.eu
       input:
       - storage_provider: minio
         path: sfincs/in

DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs_script.sh

@@ -1,4 +1,6 @@
-OUTPUT_FILE="$TMP_OUTPUT_DIR/sfincs_output.tar"
+ID=`basename "$INPUT_FILE_PATH" | cut -d'_' -f1`
+OUTPUT_FILE="$TMP_OUTPUT_DIR"/"$ID"_sfincs_output.tar
+echo $OUTPUT_FILE
 tar -xvf  "$INPUT_FILE_PATH" -C /data/
 sfincs
 tar -cf sfincs_output.tar sfincs_map.nc sfincs_his.nc sfincs.log

DT_flood/workflows/cwl_OSCAR/oscar_services/wflow.yaml

@@ -7,6 +7,7 @@ functions:
       image: deltares/wflow:v0.7.3
       script: wflow_script.sh
       log_level: DEBUG   # To avoid supervisor logs in response
+      vo: dev.intertwin.eu
       input:
       - storage_provider: minio
         path: wflow/in

DT_flood/workflows/cwl_OSCAR/oscar_services/wflow_script.sh

@@ -1,7 +1,9 @@
 FILE_NAME=`basename "$INPUT_FILE_PATH"`
-OUTPUT_FILE="$TMP_OUTPUT_DIR/wflow_output.tar"
-mkdir -p /model
-tar -xvf  "$INPUT_FILE_PATH" -C /model/
-/app/build/create_binaries/wflow_bundle/bin/wflow_cli /model/wflow_warmup/wflow_sbm.toml
-tar -cf wflow_output.tar /model/wflow_warmup/run_default/
+ID=`basename "$INPUT_FILE_PATH" | cut -d'_' -f1`
+OUTPUT_FILE="$TMP_OUTPUT_DIR"/"$ID"_wflow_output.tar
+echo $OUTPUT_FILE
+mkdir -p model
+tar -xvf  "$INPUT_FILE_PATH" -C model/
+/app/build/create_binaries/wflow_bundle/bin/wflow_cli model/wflow_warmup/wflow_sbm.toml
+tar -cf wflow_output.tar model/wflow_warmup/run_default/
 mv wflow_output.tar  $OUTPUT_FILE

DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.cwl

@@ -3,17 +3,15 @@
 cwlVersion: v1.0
 class: CommandLineTool
 baseCommand: python
-arguments: [$(inputs.oscar_script),"--endpoint",$(inputs.endpoint), "--user",$(inputs.user), "--password",$(inputs.password), "--service",$(inputs.service), "--filename", $(inputs.filename),"--service_directory",$(inputs.oscar_service),"--output",$(inputs.output)]
+arguments: [$(inputs.oscar_script),"--endpoint",$(inputs.endpoint), "--token",$(inputs.token), "--service",$(inputs.service), "--filename", $(inputs.filename),"--service_directory",$(inputs.oscar_service),"--output",$(inputs.output)]
 
 
 inputs:
   oscar_script:
     type: File
   endpoint:
     type: string
-  user:
-    type: string
-  password:
+  token:
     type: string
   service:
     type: string

DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.py

@@ -4,13 +4,13 @@
 from minio import Minio
 from oscar_python.client import Client
 import tarfile
+import uuid
 
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--endpoint")
 parser.add_argument("--filename")
-parser.add_argument("--user")
-parser.add_argument("--password")
+parser.add_argument("--token")
 parser.add_argument("--service")
 parser.add_argument("--service_directory")
 parser.add_argument("--output")
@@ -21,8 +21,7 @@
 
 endpoint = variables['endpoint']
 filename = variables['filename']
-user = variables['user']
-password = variables['password']
+token = variables['token']
 service = variables['service']
 service_directory = variables['service_directory']
 output = variables['output']
@@ -33,14 +32,14 @@ def check_oscar_connection():
     print("Checking OSCAR connection status")
     options_basic_auth = {'cluster_id':'cluster-id',
                     'endpoint': endpoint,
-                    'user':user,
-                    'password':password,
-                    'ssl':'True'}
+                    'oidc_token': token,                
+                    'ssl': 'True'}
 
     client = Client(options = options_basic_auth)
     try:
         info = client.get_cluster_info()
     except Exception as err:
+        print(err)
         print("OSCAR cluster not Found")
         exit(1)
     return client
@@ -89,14 +88,16 @@ def connect_minio(minio_info):
 def upload_file_minio(client, input_info, input_file):
     #Upload the file into input bucket
     print("Uploading the file into input bucket")
+    random= uuid.uuid4().hex + "_" + input_file.split("/")[-1]
     result = client.fput_object(
         input_info["path"].split("/")[0],
-        '/'.join(input_info["path"].split("/")[1:]) + "/" + input_file.split("/")[-1],
+        '/'.join(input_info["path"].split("/")[1:]) + "/" + random,
         input_file,
     )
+    return random.split("_")[0]
 
 
-def wait_output_and_download(client, output_info):
+def wait_output_and_download(client, output_info,execution_id):
     #Wait the output 
     print("Waiting the output")
     with client.listen_bucket_notification(
@@ -107,7 +108,9 @@ def wait_output_and_download(client, output_info):
         for event in events:
             outputfile = event["Records"][0]["s3"]["object"]["key"]
             print(event["Records"][0]["s3"]["object"]["key"])
-            break
+            if (execution_id in outputfile) :
+                print(event["Records"][0]["s3"]["object"]["key"])
+                break
     #Download the file
     print("Downloading the file")
     client.fget_object(output_info["path"].split("/")[0], 
@@ -138,6 +141,7 @@ def decompress(output_file):
 client = check_oscar_connection()
 minio_info, input_info, output_info = check_service(client, service, service_directory)
 minio_client = connect_minio(minio_info)
-upload_file_minio(minio_client, input_info, input_file)
-output_file = wait_output_and_download(minio_client, output_info)
+execution_id = upload_file_minio(minio_client, input_info, input_file)
+print(execution_id)
+output_file = wait_output_and_download(minio_client, output_info, execution_id)
 decompress(output_file)

DT_flood/workflows/cwl_OSCAR/workflow.cwl

@@ -11,9 +11,7 @@ inputs:
     type: File
   endpoint:
     type: string
-  user:
-    type: string
-  password:
+  token:
     type: string
   service_wflow:
     type: string
@@ -40,8 +38,7 @@ steps:
     in:
       oscar_script: oscar_script
       endpoint: endpoint
-      user: user
-      password: password
+      token: token
       service: service_wflow
       filename: filename_wflow
       oscar_service: oscar_service
@@ -52,8 +49,7 @@ steps:
     in:
       oscar_script: oscar_script
       endpoint: endpoint
-      user: user
-      password: password
+      token: token
       service: service_sfincs
       filename: filename_sfincs
       oscar_service: oscar_service

DT_flood/workflows/cwl_OSCAR/workflow_config.yaml

@@ -2,8 +2,7 @@ oscar_script:
   class: File
   path: oscar_workflow/oscar.py
 endpoint: 
-user: 
-password: 
+token: 
 service_wflow: wflow
 service_sfincs: sfincs
 filename_wflow: