Merge branch 'Predizioni-Epidemiologiche-Italia:main' into main

Predizioni-Epidemiologiche-Italia · Nov 27, 2023 · 070f939 · 070f939
2 parents db9e275 + 61a7ea7
commit 070f939
Show file tree

Hide file tree

Showing 5 changed files with 164 additions and 10 deletions.
diff --git a/.github/data-storage/changes_db.json b/.github/data-storage/changes_db.json
@@ -1,10 +1 @@
-{
-    "TestTeam": [
-        {
-            "model": "TestModel",
-            "changes": [
-                "previsioni/TestTeam-TestModel/2023_31.csv"
-            ]
-        }
-    ]
-}
+{}
diff --git a/.github/workflows/cleanup_wf_runs.yml b/.github/workflows/cleanup_wf_runs.yml
@@ -0,0 +1,21 @@
+name: Delete old workflow runs
+on:
+  #schedule:
+  ## Run monthly, at 00:00 on the 1st day of month.
+  #  - cron: '0 0 1 * *'
+
+  workflow_dispatch:
+
+jobs:
+  del_runs:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: Delete workflow runs
+        uses: Mattraks/delete-workflow-runs@v2
+        with:
+          token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          retain_days: 7
+          keep_minimum_runs: 6
diff --git a/.github/workflows/forecasts_upload.yml → .github/workflows/upload_forecasts.yml b/.github/workflows/forecasts_upload.yml → .github/workflows/upload_forecasts.yml
diff --git a/.github/workflows/upload_surveillance.yml b/.github/workflows/upload_surveillance.yml
@@ -0,0 +1,109 @@
+name: Surveillance Upload
+
+on:
+  # scheduled submission every Wednesday at 1.30 a.m. 
+  # schedule:
+  #   - cron: '30 5 * * 3'  
+
+  workflow_dispatch:
+
+
+jobs:
+  scheduled-submit:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      # Checkout the python tools repo
+      # used to authenticate and validate the PR
+      # -------------------------------------------      
+      - name: checkout python tools repo
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: 'Predizioni-Epidemiologiche-Italia/hub-tools'
+          ref: 'main'
+          path: './tools/'
+
+
+      - name: checkout data repo
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: 'Predizioni-Epidemiologiche-Italia/Influcast'
+          ref: 'main'
+          path: './repo/'
+
+
+      # Get the list of changes from a json file 
+      - name: Get changes list
+        id: get_changes
+        run: |
+          content=`cat ./repo/.github/data-storage/target_db.json`
+          content="${content//'%'/'%25'}"
+          content="${content//$'\n'/'%0A'}"
+          content="${content//$'\r'/'%0D'}"
+          echo "::set-output name=packageJson::$content"
+          
+      - run: |
+          echo "${{ steps.get_changes.outputs.packageJson }}"
+
+              
+      # Trigger server-side updating and UI deploy
+      # --------------------------------------------
+      # WEBHOOK
+      - name: Invoke deployment hook
+        id: invoke_webhook
+        env:
+          webhook_url: ${{ secrets.WEB_HOOK_URL }}
+          webhook_secret: ${{ secrets.WEB_HOOK_SECRET }}
+          data: ${{ steps.get_changes.outputs.packageJson }}
+          data_type: "surveillance"
+          season: "2023-2024"
+
+        run: |
+          pip install requests
+          python ./tools/code/workflow_webhook.py
+
+      - name: trace only
+        run: |
+          echo "webhook results: ${{ steps.invoke_webhook.outputs.run_results }}"
+
+      # SUCCESSFUL RUN
+      # --------------------------------------------
+      - name: Successful run
+        if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'success'}}
+        env:
+          wh_response:  ${{ steps.invoke_webhook.outputs.run_results }}
+        run: |
+          echo "UPLOAD SUCCEEDED -  clear djson db"
+          python ./tools/code/clear_data_storage.py --storage_type "target"
+
+      # RUN FAILURE
+      # --------------------------------------------
+      - name: Some ingestion failed
+        if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'failure'}}
+        env:
+          wh_response:  ${{ steps.invoke_webhook.outputs.run_results }}
+
+        run: |
+          echo "UPLOAD FAILED - Some ingestion failed"
+          python ./tools/code/clear_data_storage.py --storage_type "target" 
+
+      # ERROR RUN
+      # --------------------------------------------
+      - name: Error
+        if: ${{ steps.invoke_webhook.outputs.run_results.status == 'error'}}
+        run: |
+          echo "UPLOAD FAILED -  ignore djson db cleaning"
+
+      # Final commit
+      # --------------------------------------------
+      - name: Commit changes
+        if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'failure' || fromJSON(steps.invoke_webhook.outputs.run_results).status == 'success' }}
+        uses: EndBug/add-and-commit@v7
+        with: 
+          cwd: './repo'
+          message: "Update model-output db json"
+          default_author: github_actions
+          push: true
diff --git a/previsioni/TestTeam-TestModel/2023_45.csv b/previsioni/TestTeam-TestModel/2023_45.csv
@@ -0,0 +1,33 @@
+anno,settimana,luogo,tipo_valore,id_valore,orizzonte,valore
+2023,45,01,quantile,0.01,1,0.9786677558027155
+2023,45,01,quantile,0.99,1,0.986332243955226
+2023,45,01,quantile,0.05,1,0.9729141146128889
+2023,45,01,quantile,0.95,1,0.9920858851450526
+2023,45,01,quantile,0.01,2,0.9886677558027155
+2023,45,01,quantile,0.99,2,0.996332243955226
+2023,45,01,quantile,0.05,2,0.9929141146128889
+2023,45,01,quantile,0.95,2,0.9990858851450526
+2023,45,01,quantile,0.01,3,0.9986677558027155
+2023,45,01,quantile,0.99,3,1.086332243955226
+2023,45,01,quantile,0.05,3,1.09729141146128889
+2023,45,01,quantile,0.95,3,1.09920858851450526
+2023,45,01,quantile,0.01,4,1.3786677558027155
+2023,45,01,quantile,0.99,4,1.486332243955226
+2023,45,01,quantile,0.05,4,1.5729141146128889
+2023,45,01,quantile,0.95,4,1.6920858851450526
+2023,45,02,quantile,0.1,1,0.8632903499361065
+2023,45,02,quantile,0.9,1,0.6865912532837774
+2023,45,02,quantile,0.15,1,1.078408746474164
+2023,45,02,quantile,0.85,1,0.5310538458167399
+2023,45,02,quantile,0.1,2,0.9632903499361065
+2023,45,02,quantile,0.9,2,0.7865912532837774
+2023,45,02,quantile,0.15,2,1.178408746474164
+2023,45,02,quantile,0.85,2,0.7310538458167399
+2023,45,02,quantile,0.1,3,0.9832903499361065
+2023,45,02,quantile,0.9,3,0.7965912532837774
+2023,45,02,quantile,0.15,3,1.278408746474164
+2023,45,02,quantile,0.85,3,0.7710538458167399
+2023,45,02,quantile,0.1,4,1.09632903499361065
+2023,45,02,quantile,0.9,4,0.8865912532837774
+2023,45,02,quantile,0.15,4,1.378408746474164
+2023,45,02,quantile,0.85,4,0.9610538458167399