From 8b1c5d609ede9ffb1989456c60dd8484e385dc51 Mon Sep 17 00:00:00 2001 From: Meet Agrawal Date: Sun, 16 Jun 2024 12:06:32 +0530 Subject: [PATCH 1/8] added base framework --- .github/workflows/download-data.yml | 56 +++++++++++++++++++++++++++++ tests/docker.AmerifluxLBL.xml | 32 +++++++++++++++++ tests/docker.CRUNCEP.xml | 32 +++++++++++++++++ web/test.R | 4 +++ 4 files changed, 124 insertions(+) create mode 100644 .github/workflows/download-data.yml create mode 100644 tests/docker.AmerifluxLBL.xml create mode 100644 tests/docker.CRUNCEP.xml create mode 100644 web/test.R diff --git a/.github/workflows/download-data.yml b/.github/workflows/download-data.yml new file mode 100644 index 00000000000..c58e95caefc --- /dev/null +++ b/.github/workflows/download-data.yml @@ -0,0 +1,56 @@ +name : Test Data Download +on : + workflow_call: + inputs: + R-version: + required: true + type: string + +env: + R_LIBS_USER: /usr/local/lib/R/site-library + LC_ALL: en_US.UTF-8 + NCPUS: 2 + PGHOST: postgres + CI: true + +jobs: + data-download: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + services: + postgres: + image: mdillon/postgis:9.5 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + container: + image: pecan/depends:R${{ inputs.R-version }} + + steps: + # checkout source code + - name: work around https://github.com/actions/checkout/issues/766 + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/checkout@v4 + with: + set-safe-directory: false + + # install additional tools needed + - name: install utils + run: apt-get update && apt-get install -y postgresql-client qpdf + - name: install new dependencies + run: Rscript scripts/generate_dependencies.R && cd docker/depends && Rscript pecan.depends.R + + # initialize database + - name: db setup + uses: docker://pecan/db:ci + - name: add models to db + run: ./scripts/add.models.sh + + - name: CRUNCEP Test + run: | + Rscript ./web/test.R --settings ./tests/docker.CRUNCEP.xml + + - name: CRUNCEP Test + run: | + Rscript ./web/test.R --settings ./tests/docker.AmerifluxLBL.xml \ No newline at end of file diff --git a/tests/docker.AmerifluxLBL.xml b/tests/docker.AmerifluxLBL.xml new file mode 100644 index 00000000000..23309670212 --- /dev/null +++ b/tests/docker.AmerifluxLBL.xml @@ -0,0 +1,32 @@ + + + /data/tests/sipnet + + + PostgreSQL + bety + bety + postgres + bety + FALSE + + + + SIPNET + + + + 1000000057 + + + + AmerifluxLBL + + + 2011-01-01 + 2011-12-31 + + + localhost + + \ No newline at end of file diff --git a/tests/docker.CRUNCEP.xml b/tests/docker.CRUNCEP.xml new file mode 100644 index 00000000000..06b15ac8bda --- /dev/null +++ b/tests/docker.CRUNCEP.xml @@ -0,0 +1,32 @@ + + + /data/tests/sipnet + + + PostgreSQL + bety + bety + postgres + bety + FALSE + + + + SIPNET + + + + 1000000057 + + + + CRUNCEP + + + 2001-01-01 + 2001-12-31 + + + localhost + + \ No newline at end of file diff --git a/web/test.R b/web/test.R new file mode 100644 index 00000000000..49970960a20 --- /dev/null +++ b/web/test.R @@ -0,0 +1,4 @@ +args <- PEcAn.settings::get_args() +settings <- PEcAn.settings::read.settings(args$settings) +settings <- PEcAn.settings::prepare.settings(settings, force = FALSE) +settings <- PEcAn.workflow::do_conversions(settings) \ No newline at end of file From fca8f2f8dd70eefa661edc8e506dd88b48f10fca Mon Sep 17 00:00:00 2001 From: Meet Agrawal Date: Sun, 16 Jun 2024 12:13:18 +0530 Subject: [PATCH 2/8] updated trigger --- .github/workflows/download-data.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/download-data.yml b/.github/workflows/download-data.yml index c58e95caefc..58bdb980f2e 100644 --- a/.github/workflows/download-data.yml +++ b/.github/workflows/download-data.yml @@ -1,10 +1,13 @@ name : Test Data Download on : - workflow_call: - inputs: - R-version: - required: true - type: string + pull_request: + + # allow manual triggering + workflow_dispatch: + + schedule: + # run Thursday 4:30 AM UTC + - cron: '30 4 * * 4' env: R_LIBS_USER: /usr/local/lib/R/site-library @@ -25,7 +28,7 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 container: - image: pecan/depends:R${{ inputs.R-version }} + image: pecan/depends:R4.1 steps: # checkout source code From 7b9d4d95b5549b1e88f4ee1d12bfaeeaf3161844 Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Sun, 16 Jun 2024 12:18:20 +0530 Subject: [PATCH 3/8] bug fix --- .github/workflows/download-data.yml | 6 +++++- web/test.R | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/download-data.yml b/.github/workflows/download-data.yml index 58bdb980f2e..ffdad170452 100644 --- a/.github/workflows/download-data.yml +++ b/.github/workflows/download-data.yml @@ -50,10 +50,14 @@ jobs: - name: add models to db run: ./scripts/add.models.sh + # compile PEcAn code + - name: build + run: make -j1 + - name: CRUNCEP Test run: | Rscript ./web/test.R --settings ./tests/docker.CRUNCEP.xml - - name: CRUNCEP Test + - name: AmerifluxLBL Test run: | Rscript ./web/test.R --settings ./tests/docker.AmerifluxLBL.xml \ No newline at end of file diff --git a/web/test.R b/web/test.R index 49970960a20..0e7bbaffe09 100644 --- a/web/test.R +++ b/web/test.R @@ -1,3 +1,5 @@ +library("PEcAn.all") + args <- PEcAn.settings::get_args() settings <- PEcAn.settings::read.settings(args$settings) settings <- PEcAn.settings::prepare.settings(settings, force = FALSE) From a337153d65d601c074069f5ec83aa825e5f57c48 Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Sun, 16 Jun 2024 12:56:40 +0530 Subject: [PATCH 4/8] added NOAA_GEFS --- .github/workflows/download-data.yml | 10 ++++++--- tests/docker.NOAA_GEFS.xml | 32 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 tests/docker.NOAA_GEFS.xml diff --git a/.github/workflows/download-data.yml b/.github/workflows/download-data.yml index ffdad170452..4ad7c6ffa9c 100644 --- a/.github/workflows/download-data.yml +++ b/.github/workflows/download-data.yml @@ -54,10 +54,14 @@ jobs: - name: build run: make -j1 - - name: CRUNCEP Test + - name: CRUNCEP run: | Rscript ./web/test.R --settings ./tests/docker.CRUNCEP.xml - - name: AmerifluxLBL Test + - name: AmerifluxLBL run: | - Rscript ./web/test.R --settings ./tests/docker.AmerifluxLBL.xml \ No newline at end of file + Rscript ./web/test.R --settings ./tests/docker.AmerifluxLBL.xml + + - name: NOAA_GEFS + run: | + Rscript ./web/test.R --settings ./tests/docker.NOAA_GEFS.xml \ No newline at end of file diff --git a/tests/docker.NOAA_GEFS.xml b/tests/docker.NOAA_GEFS.xml new file mode 100644 index 00000000000..2d4e367ac6b --- /dev/null +++ b/tests/docker.NOAA_GEFS.xml @@ -0,0 +1,32 @@ + + + /data/tests/sipnet + + + PostgreSQL + bety + bety + postgres + bety + FALSE + + + + SIPNET + + + + 676 + + + + NOAA_GEFS + + + 2011-01-01 + 2011-12-31 + + + localhost + + \ No newline at end of file From c064722d2eb8bb7866f0c53241bbf73b41b19f8e Mon Sep 17 00:00:00 2001 From: Meet Agrawal Date: Sun, 4 Aug 2024 22:46:49 +0530 Subject: [PATCH 5/8] Accepted suggestions Co-authored-by: Chris Black --- tests/docker.AmerifluxLBL.xml | 3 ++- tests/docker.CRUNCEP.xml | 3 ++- tests/docker.NOAA_GEFS.xml | 3 ++- web/test.R | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/docker.AmerifluxLBL.xml b/tests/docker.AmerifluxLBL.xml index 23309670212..4841d300800 100644 --- a/tests/docker.AmerifluxLBL.xml +++ b/tests/docker.AmerifluxLBL.xml @@ -1,6 +1,6 @@ - /data/tests/sipnet + /data/tests/met_download/AmerifluxLBL PostgreSQL @@ -25,6 +25,7 @@ 2011-01-01 2011-12-31 + /data/dbfiles localhost diff --git a/tests/docker.CRUNCEP.xml b/tests/docker.CRUNCEP.xml index 06b15ac8bda..77e81fc198f 100644 --- a/tests/docker.CRUNCEP.xml +++ b/tests/docker.CRUNCEP.xml @@ -1,6 +1,6 @@ - /data/tests/sipnet + /data/tests/met_download/CRUNCEP PostgreSQL @@ -25,6 +25,7 @@ 2001-01-01 2001-12-31 + /data/dbfiles localhost diff --git a/tests/docker.NOAA_GEFS.xml b/tests/docker.NOAA_GEFS.xml index 2d4e367ac6b..4c7cb44df60 100644 --- a/tests/docker.NOAA_GEFS.xml +++ b/tests/docker.NOAA_GEFS.xml @@ -1,6 +1,6 @@ - /data/tests/sipnet + /data/tests/met_download/NOAA_GEFS PostgreSQL @@ -25,6 +25,7 @@ 2011-01-01 2011-12-31 + /data/dbfiles localhost diff --git a/web/test.R b/web/test.R index 0e7bbaffe09..58d21b161fb 100644 --- a/web/test.R +++ b/web/test.R @@ -3,4 +3,4 @@ library("PEcAn.all") args <- PEcAn.settings::get_args() settings <- PEcAn.settings::read.settings(args$settings) settings <- PEcAn.settings::prepare.settings(settings, force = FALSE) -settings <- PEcAn.workflow::do_conversions(settings) \ No newline at end of file +settings <- PEcAn.workflow::do_conversions(settings) From ab232a3c7e5af35a20bcde2de619d16ff1993dbc Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Thu, 5 Sep 2024 20:56:40 +0530 Subject: [PATCH 6/8] clean-up --- .github/workflows/download-data.yml | 12 ++----- tests/docker.AmerifluxLBL.xml | 33 ------------------- tests/docker.NOAA_GEFS.xml | 33 ------------------- .../docker.CRUNCEP.xml | 0 web/test.R => tests/test_met_downloads.R | 0 5 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 tests/docker.AmerifluxLBL.xml delete mode 100644 tests/docker.NOAA_GEFS.xml rename tests/{ => met_download_settings}/docker.CRUNCEP.xml (100%) rename web/test.R => tests/test_met_downloads.R (100%) diff --git a/.github/workflows/download-data.yml b/.github/workflows/download-data.yml index 4ad7c6ffa9c..deb13065f5c 100644 --- a/.github/workflows/download-data.yml +++ b/.github/workflows/download-data.yml @@ -17,7 +17,7 @@ env: CI: true jobs: - data-download: + met-data-download: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} @@ -56,12 +56,4 @@ jobs: - name: CRUNCEP run: | - Rscript ./web/test.R --settings ./tests/docker.CRUNCEP.xml - - - name: AmerifluxLBL - run: | - Rscript ./web/test.R --settings ./tests/docker.AmerifluxLBL.xml - - - name: NOAA_GEFS - run: | - Rscript ./web/test.R --settings ./tests/docker.NOAA_GEFS.xml \ No newline at end of file + Rscript ./tests/test_met_download.R --settings ./tests/met_download_settings/docker.CRUNCEP.xml \ No newline at end of file diff --git a/tests/docker.AmerifluxLBL.xml b/tests/docker.AmerifluxLBL.xml deleted file mode 100644 index 4841d300800..00000000000 --- a/tests/docker.AmerifluxLBL.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - /data/tests/met_download/AmerifluxLBL - - - PostgreSQL - bety - bety - postgres - bety - FALSE - - - - SIPNET - - - - 1000000057 - - - - AmerifluxLBL - - - 2011-01-01 - 2011-12-31 - /data/dbfiles - - - localhost - - \ No newline at end of file diff --git a/tests/docker.NOAA_GEFS.xml b/tests/docker.NOAA_GEFS.xml deleted file mode 100644 index 4c7cb44df60..00000000000 --- a/tests/docker.NOAA_GEFS.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - /data/tests/met_download/NOAA_GEFS - - - PostgreSQL - bety - bety - postgres - bety - FALSE - - - - SIPNET - - - - 676 - - - - NOAA_GEFS - - - 2011-01-01 - 2011-12-31 - /data/dbfiles - - - localhost - - \ No newline at end of file diff --git a/tests/docker.CRUNCEP.xml b/tests/met_download_settings/docker.CRUNCEP.xml similarity index 100% rename from tests/docker.CRUNCEP.xml rename to tests/met_download_settings/docker.CRUNCEP.xml diff --git a/web/test.R b/tests/test_met_downloads.R similarity index 100% rename from web/test.R rename to tests/test_met_downloads.R From 5d10e6d5339d3fc36b4b199476dce9e9a01604f0 Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Thu, 5 Sep 2024 21:21:12 +0530 Subject: [PATCH 7/8] bug fix --- .github/workflows/{download-data.yml => download-met-data.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{download-data.yml => download-met-data.yml} (93%) diff --git a/.github/workflows/download-data.yml b/.github/workflows/download-met-data.yml similarity index 93% rename from .github/workflows/download-data.yml rename to .github/workflows/download-met-data.yml index deb13065f5c..f72a8b8529a 100644 --- a/.github/workflows/download-data.yml +++ b/.github/workflows/download-met-data.yml @@ -56,4 +56,4 @@ jobs: - name: CRUNCEP run: | - Rscript ./tests/test_met_download.R --settings ./tests/met_download_settings/docker.CRUNCEP.xml \ No newline at end of file + Rscript ./tests/test_met_downloads.R --settings ./tests/met_download_settings/docker.CRUNCEP.xml \ No newline at end of file From 28a75e1b2aa1c32f95e4e2037a1f776fa6d9e91b Mon Sep 17 00:00:00 2001 From: Meet Agrawal Date: Thu, 5 Sep 2024 21:40:53 +0530 Subject: [PATCH 8/8] Removed pull-request option --- .github/workflows/download-met-data.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/download-met-data.yml b/.github/workflows/download-met-data.yml index f72a8b8529a..1871176e0c1 100644 --- a/.github/workflows/download-met-data.yml +++ b/.github/workflows/download-met-data.yml @@ -1,7 +1,5 @@ name : Test Data Download on : - pull_request: - # allow manual triggering workflow_dispatch: @@ -56,4 +54,4 @@ jobs: - name: CRUNCEP run: | - Rscript ./tests/test_met_downloads.R --settings ./tests/met_download_settings/docker.CRUNCEP.xml \ No newline at end of file + Rscript ./tests/test_met_downloads.R --settings ./tests/met_download_settings/docker.CRUNCEP.xml