From 6db592c34f9a2568c2501fd9c8294af763fc2f74 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 23 Nov 2023 17:34:01 +0000 Subject: [PATCH 1/4] Update data storage --- .github/data-storage/changes_db.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/data-storage/changes_db.json b/.github/data-storage/changes_db.json index 0967ef42..2639b058 100644 --- a/.github/data-storage/changes_db.json +++ b/.github/data-storage/changes_db.json @@ -1 +1,10 @@ -{} +{ + "TestTeam": [ + { + "model": "TestModel", + "changes": [ + "previsioni/TestTeam_TestModel/2017_50.csv" + ] + } + ] +} \ No newline at end of file From 3b806992040d4b99c3603ac4b5c655b224d324e6 Mon Sep 17 00:00:00 2001 From: PMi74 Date: Thu, 23 Nov 2023 18:35:28 +0100 Subject: [PATCH 2/4] Update 2017_50.csv (#14) --- previsioni/TestTeam_TestModel/2017_50.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/previsioni/TestTeam_TestModel/2017_50.csv b/previsioni/TestTeam_TestModel/2017_50.csv index b1b18823..80015d91 100644 --- a/previsioni/TestTeam_TestModel/2017_50.csv +++ b/previsioni/TestTeam_TestModel/2017_50.csv @@ -7,5 +7,5 @@ anno,settimana,luogo,tipo_valore,id_valore,orizzonte,valore 2017,45,02,quantile,0.9,2,0.7865912532837774 2017,45,02,quantile,0.15,2,1.178408746474164 2017,45,02,quantile,0.85,2,0.7310538458167399 -2017,45,03,quantile,0.2,3,1.2339461539412016 +2017,45,03,quantile,0.2,3,1.2339461539412017 2017,45,03,quantile,0.8,3,0.9824999998789737 From 940917e98d035d147ef89f445bd00d6d559d82df Mon Sep 17 00:00:00 2001 From: PMi74 Date: Fri, 24 Nov 2023 13:14:37 +0100 Subject: [PATCH 3/4] implementing upload --- .github/workflows/forecasts_upload.yml | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/forecasts_upload.yml diff --git a/.github/workflows/forecasts_upload.yml b/.github/workflows/forecasts_upload.yml new file mode 100644 index 00000000..31e77dd0 --- /dev/null +++ b/.github/workflows/forecasts_upload.yml @@ -0,0 +1,109 @@ +name: Forecast Upload + +on: + # scheduled submission every Wednesday at 1.30 a.m. + # schedule: + # - cron: '30 5 * * 3' + + workflow_dispatch: + + +jobs: + scheduled-submit: + runs-on: ubuntu-latest + + steps: + + # Checkout the python tools repo + # used to authenticate and validate the PR + # ------------------------------------------- + - name: checkout python tools repo + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + repository: 'Predizioni-Epidemiologiche-Italia/hub-tools' + ref: 'main' + path: './tools/' + + + - name: checkout data repo + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + repository: 'Predizioni-Epidemiologiche-Italia/Influcast' + ref: 'main' + path: './repo/' + + + # Get the list of changes from a json file + - name: Get changes list + id: get_changes + run: | + content=`cat ./repo/.github/data-storage/changes_db.json` + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + echo "::set-output name=packageJson::$content" + + - run: | + echo "${{ steps.get_changes.outputs.packageJson }}" + + + # Trigger server-side updating and UI deploy + # -------------------------------------------- + # WEBHOOK + - name: Invoke deployment hook + id: invoke_webhook + env: + webhook_url: ${{ secrets.WEB_HOOK_URL }} + webhook_secret: ${{ secrets.WEB_HOOK_SECRET }} + data: ${{ steps.get_changes.outputs.packageJson }} + data_type: "forecast" + season: "NA" + + run: | + pip install requests + python ./tools/code/workflow_webhook.py + + - name: trace only + run: | + echo "webhook results: ${{ steps.invoke_webhook.outputs.run_results }}" + + # SUCCESSFUL RUN + # -------------------------------------------- + - name: Successful run + if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'success'}} + env: + wh_response: ${{ steps.invoke_webhook.outputs.run_results }} + run: | + echo "UPLOAD SUCCEEDED - clear djson db" + python ./tools/code/clear_data_storage.py --storage_type "previsioni" + + # RUN FAILURE + # -------------------------------------------- + - name: Some ingestion failed + if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'failure'}} + env: + wh_response: ${{ steps.invoke_webhook.outputs.run_results }} + + run: | + echo "UPLOAD FAILED - Some ingestion failed" + python ./tools/code/clear_data_storage.py --storage_type "previsioni" + + # ERROR RUN + # -------------------------------------------- + - name: Error + if: ${{ steps.invoke_webhook.outputs.run_results.status == 'error'}} + run: | + echo "UPLOAD FAILED - ignore djson db cleaning" + + # Final commit + # -------------------------------------------- + - name: Commit changes + if: ${{ fromJSON(steps.invoke_webhook.outputs.run_results).status == 'failure' || fromJSON(steps.invoke_webhook.outputs.run_results).status == 'success' }} + uses: EndBug/add-and-commit@v7 + with: + cwd: './repo' + message: "Update model-output db json" + default_author: github_actions + push: true \ No newline at end of file From f9d5f8aa66a596fd191c1fb4dea1acb774d0615f Mon Sep 17 00:00:00 2001 From: PMi74 Date: Fri, 24 Nov 2023 16:48:33 +0100 Subject: [PATCH 4/4] handling surveillance submission --- .github/workflows/validate_submission.yml | 1 - .github/workflows/validate_surveillance.yml | 155 ++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/validate_surveillance.yml diff --git a/.github/workflows/validate_submission.yml b/.github/workflows/validate_submission.yml index 15dbb1fb..b458f3ed 100644 --- a/.github/workflows/validate_submission.yml +++ b/.github/workflows/validate_submission.yml @@ -14,7 +14,6 @@ jobs: validate_request: runs-on: ubuntu-latest outputs: - # output_data: ${{ steps.compile_output.outputs.output_data }} validate_error_msg: ${{ steps.validate.outputs.message }} authenticate_error_msg: ${{ steps.authenticate.outputs.message }} changed_files: ${{ steps.get_changed_files.outputs.all_changed_files }} diff --git a/.github/workflows/validate_surveillance.yml b/.github/workflows/validate_surveillance.yml new file mode 100644 index 00000000..c74f4618 --- /dev/null +++ b/.github/workflows/validate_surveillance.yml @@ -0,0 +1,155 @@ +# merge submission on the main branch after a pullrequest approval +name: ValidateSurveillance +on: + pull_request_target: + branches: [ main ] + paths: + - 'sorveglianza/**' + - '!**README**' + + +jobs: + validate_request: + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.authenticate.outputs.authenticate == 'success' }} + changed_files: ${{ steps.get_changed_files.outputs.all_changed_files }} + authenticate_error_msg: ${{ steps.authenticate.outputs.message }} + + + steps: + + # Check out the submission repo + # ------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ github.event.pull_request.head.sha }} + + + # Checkout the python tools repo + # used to authenticate and validate the PR + # ------------------------------------------- + - name: checkout python tools repo + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + repository: 'Predizioni-Epidemiologiche-Italia/hub-tools' + ref: 'main' + path: './tools' + + + # Get changes from pull request + # -------------------------------- + - name: Get changes + id: get_changed_files + uses: tj-actions/changed-files@v36 + + # Authenticate the pull_request + # returns as output authentication = { True / False } + # ------------------------------- + - name: Execute Authentication script + id: authenticate + env: + calling_actor: ${{ github.actor }} + changed_files: ${{ steps.get_changed_files.outputs.all_changed_files }} + + run: python ./tools/.github/scripts/request_authentication/authenticate_surveillance.py + + + # ------------------------------------------- + # SUCCESS + # ------------------------------------------- + on_successful_validation: + runs-on: ubuntu-latest + needs: validate_request + if: needs.validate_request.outputs.is_valid == 'true' + + steps: + - name: DEBUG + run: | + echo "Is_Valid: ${{ needs.validate_request.outputs.is_valid }}" + + # Checkout the forecast repo + # --------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: "${{ github.event.pull_request.head.sha }}" + + # Approve pull request + # --------------------------- + - name: Approve Pull Request + uses: juliangruber/approve-pull-request-action@v2.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} + + # Eventually comment on it + - name: Comment PR + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + All checks completed successfully ! :wave: + pr_number: ${{ github.event.pull_request.number }} + + # Merge changes + # --------------------------- + - name: Merge the pull request + id: merge_pr + run: | + gh pr merge --auto --squash $PR_URL + echo "status=true" >> $GITHUB_OUTPUT + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Save changes to artifact for later persisting + - name: Save changes to artifact + env: + changed_files: ${{ needs.validate_request.outputs.changed_files }} + run: | + echo "{ \"pr-changes\" : \"$changed_files\" }" > pr_changes.json + + - name: Upload changes + uses: actions/upload-artifact@v2 + with: + name: pr_changes + path: ./pr_changes.json + + # ------------------------------------------- + # FAILED + # ------------------------------------------- + on_validation_failed: + runs-on: ubuntu-latest + needs: validate_request + if: needs.validate_request.outputs.is_valid == 'false' + + steps: + + # DEBUG ONLY + - name: DEBUG - DUMP INFO + env: + changed-files: ${{ needs.validate_request.outputs.changed_files }} + + run: | + echo ">>> Is_Valid: ${{ needs.validate_request.outputs.is_valid }}" + echo ">>> FAILED JOB " + echo ">>> Changes: $changed-files" + echo ">>> Repo: ${{ github.repository }}" + echo ">>> SrcPath: $GITHUB_WORKSPACE" + echo ">>> Pull request number: ${{ github.event.pull_request.number }}" + echo ">>> Git hub actor: ${{ github.actor }}" + echo ">>> Authenticate Error Message ${{ needs.validate_request.outputs.authenticate_error_msg }}" + + + # Inform the user about what went wrong + - name: Comment PR + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + An error occurred during authentication: ${{ needs.validate_request.outputs.authenticate_error_msg }} + pr_number: ${{ github.event.pull_request.number }} + \ No newline at end of file