Merge branch 'Predizioni-Epidemiologiche-Italia:main' into main

.github/data-storage/changes_db.json

@@ -1 +1,10 @@
-{}
+{
+    "TestTeam": [
+        {
+            "model": "TestModel",
+            "changes": [
+                "previsioni/TestTeam_TestModel/2017_50.csv"
+            ]
+        }
+    ]
+}

.github/workflows/forecasts_upload.yml

@@ -0,0 +1,109 @@
+name: Forecast Upload
+
+on:
+  # scheduled submission every Wednesday at 1.30 a.m. 
+  # schedule:
+  #   - cron: '30 5 * * 3'  
+
+  workflow_dispatch:
+
+
+jobs:
+  scheduled-submit:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      # Checkout the python tools repo
+      # used to authenticate and validate the PR
+      # -------------------------------------------      
+      - name: checkout python tools repo
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: 'Predizioni-Epidemiologiche-Italia/hub-tools'
+          ref: 'main'
+          path: './tools/'
+
+
+      - name: checkout data repo
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: 'Predizioni-Epidemiologiche-Italia/Influcast'
+          ref: 'main'
+          path: './repo/'
+
+
+      # Get the list of changes from a json file 
+      - name: Get changes list
+        id: get_changes
+        run: |
+          content=`cat ./repo/.github/data-storage/changes_db.json`
+          content="${content//'%'/'%25'}"
+          content="${content//$'\n'/'%0A'}"
+          content="${content//$'\r'/'%0D'}"
+          echo "::set-output name=packageJson::$content"
+          
+      - run: |
+          echo "${{ steps.get_changes.outputs.packageJson }}"
+
+              
+      # Trigger server-side updating and UI deploy
+      # --------------------------------------------
+      # WEBHOOK
+      - name: Invoke deployment hook
+        id: invoke_webhook
+        env:
+          webhook_url: ${{ secrets.WEB_HOOK_URL }}
+          webhook_secret: ${{ secrets.WEB_HOOK_SECRET }}
+          data: ${{ steps.get_changes.outputs.packageJson }}
+          data_type: "forecast"
+          season: "NA"
+
+        run: |
+          pip install requests
+          python ./tools/code/workflow_webhook.py
+
+      - name: trace only
+        run: |
+          echo "webhook results: ${{ steps.invoke_webhook.outputs.run_results }}"
+
+      # SUCCESSFUL RUN
+      # --------------------------------------------
+      - name: Successful run
+        if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'success'}}
+        env:
+          wh_response:  ${{ steps.invoke_webhook.outputs.run_results }}
+        run: |
+          echo "UPLOAD SUCCEEDED -  clear djson db"
+          python ./tools/code/clear_data_storage.py --storage_type "previsioni"
+
+      # RUN FAILURE
+      # --------------------------------------------
+      - name: Some ingestion failed
+        if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'failure'}}
+        env:
+          wh_response:  ${{ steps.invoke_webhook.outputs.run_results }}
+
+        run: |
+          echo "UPLOAD FAILED - Some ingestion failed"
+          python ./tools/code/clear_data_storage.py --storage_type "previsioni" 
+
+      # ERROR RUN
+      # --------------------------------------------
+      - name: Error
+        if: ${{ steps.invoke_webhook.outputs.run_results.status == 'error'}}
+        run: |
+          echo "UPLOAD FAILED -  ignore djson db cleaning"
+
+      # Final commit
+      # --------------------------------------------
+      - name: Commit changes
+        if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'failure' || fromJSON(steps.invoke_webhook.outputs.run_results).status == 'success' }}
+        uses: EndBug/add-and-commit@v7
+        with: 
+          cwd: './repo'
+          message: "Update model-output db json"
+          default_author: github_actions
+          push: true

.github/workflows/validate_submission.yml

@@ -14,7 +14,6 @@ jobs:
   validate_request:
     runs-on: ubuntu-latest
     outputs: 
-      # output_data: ${{ steps.compile_output.outputs.output_data }}
       validate_error_msg: ${{ steps.validate.outputs.message }}
       authenticate_error_msg: ${{ steps.authenticate.outputs.message }}
       changed_files:  ${{ steps.get_changed_files.outputs.all_changed_files }}

.github/workflows/validate_surveillance.yml

@@ -0,0 +1,155 @@
+# merge submission on the main branch after a pullrequest approval
+name: ValidateSurveillance
+on: 
+  pull_request_target:
+    branches: [ main ]
+    paths:
+      - 'sorveglianza/**'
+      - '!**README**'
+
+
+jobs:
+    validate_request:
+        runs-on: ubuntu-latest
+        outputs: 
+          is_valid: ${{ steps.authenticate.outputs.authenticate == 'success' }}
+          changed_files:  ${{ steps.get_changed_files.outputs.all_changed_files }}
+          authenticate_error_msg: ${{ steps.authenticate.outputs.message }}
+
+
+        steps:    
+
+          # Check out the submission repo
+          # -------------------------------------------      
+          - name: Checkout code
+            uses: actions/checkout@v3
+            with:
+              token: ${{ secrets.GITHUB_TOKEN }}
+              ref: ${{ github.event.pull_request.head.sha }}
+
+
+          # Checkout the python tools repo
+          # used to authenticate and validate the PR
+          # -------------------------------------------      
+          - name: checkout python tools repo
+            uses: actions/checkout@v3
+            with:
+              token: ${{ secrets.GITHUB_TOKEN }}
+              repository: 'Predizioni-Epidemiologiche-Italia/hub-tools'
+              ref: 'main'
+              path: './tools'
+
+
+          # Get changes from pull request 
+          # --------------------------------      
+          - name: Get changes
+            id: get_changed_files
+            uses: tj-actions/changed-files@v36
+
+          # Authenticate the pull_request 
+          # returns as output authentication = { True / False }
+          # -------------------------------      
+          - name: Execute Authentication script
+            id: authenticate
+            env:
+              calling_actor: ${{ github.actor }}
+              changed_files: ${{ steps.get_changed_files.outputs.all_changed_files }}
+
+            run: python ./tools/.github/scripts/request_authentication/authenticate_surveillance.py
+
+
+    # -------------------------------------------
+    # SUCCESS
+    # -------------------------------------------
+    on_successful_validation:
+        runs-on: ubuntu-latest
+        needs: validate_request
+        if: needs.validate_request.outputs.is_valid == 'true'
+
+        steps:
+          - name: DEBUG
+            run: | 
+              echo "Is_Valid: ${{ needs.validate_request.outputs.is_valid }}"
+          
+          # Checkout the forecast repo
+          # ---------------------------
+          - name: Checkout code
+            uses: actions/checkout@v3
+            with:
+              token: ${{ secrets.GITHUB_TOKEN }}
+              ref: "${{ github.event.pull_request.head.sha }}"
+
+          # Approve pull request 
+          # ---------------------------
+          - name: Approve Pull Request
+            uses: juliangruber/[email protected]
+            with:
+              github-token: ${{ secrets.GITHUB_TOKEN }}
+              number: ${{ github.event.pull_request.number }}
+
+          # Eventually comment on it
+          - name: Comment PR
+            uses: thollander/actions-comment-pull-request@v2
+            with:
+              message: |
+                All checks completed successfully ! :wave:
+              pr_number: ${{ github.event.pull_request.number }}
+
+          # Merge changes  
+          # ---------------------------    
+          - name: Merge the pull request
+            id: merge_pr
+            run: |
+              gh pr merge --auto --squash $PR_URL
+              echo "status=true" >> $GITHUB_OUTPUT
+            env:
+              PR_URL: ${{ github.event.pull_request.html_url }}
+              GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+          # Save changes to artifact for later persisting 
+          - name: Save changes to artifact
+            env:
+              changed_files: ${{ needs.validate_request.outputs.changed_files }}
+            run: |
+              echo "{ \"pr-changes\" : \"$changed_files\" }" > pr_changes.json
+            
+          - name: Upload changes
+            uses: actions/upload-artifact@v2
+            with:
+              name: pr_changes
+              path: ./pr_changes.json
+
+    # -------------------------------------------
+    # FAILED
+    # -------------------------------------------
+    on_validation_failed:
+        runs-on: ubuntu-latest
+        needs: validate_request
+        if: needs.validate_request.outputs.is_valid == 'false'
+
+        steps:
+
+          # DEBUG ONLY
+          - name: DEBUG - DUMP INFO
+            env:
+              changed-files: ${{ needs.validate_request.outputs.changed_files }}
+
+            run: |
+              echo ">>> Is_Valid: ${{ needs.validate_request.outputs.is_valid }}"
+              echo ">>> FAILED JOB "
+              echo ">>> Changes: $changed-files"
+              echo ">>> Repo: ${{ github.repository }}"
+              echo ">>> SrcPath: $GITHUB_WORKSPACE"
+              echo ">>> Pull request number: ${{ github.event.pull_request.number }}"
+              echo ">>> Git hub actor: ${{ github.actor }}"
+              echo ">>> Authenticate Error Message ${{ needs.validate_request.outputs.authenticate_error_msg  }}"
+    
+
+          # Inform the user about what went wrong
+          - name: Comment PR
+            uses: thollander/actions-comment-pull-request@v2
+            with:
+              message: |
+                An error occurred during authentication: ${{  needs.validate_request.outputs.authenticate_error_msg }}
+              pr_number: ${{ github.event.pull_request.number }}
+