Merge branch 'develop' into dependabot/pip/docker/monitor/requests-2.…

…31.0

.github/workflows/ci.yml

@@ -82,7 +82,7 @@ jobs:
   # ----------------------------------------------------------------------
   # R CHECK
   # ----------------------------------------------------------------------
-  check:
+  check_base:
     if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build')
     runs-on: ubuntu-latest
 
@@ -131,7 +131,121 @@ jobs:
 
     # run PEcAn checks
     - name: check
-      run: make -j1 check
+      run: make -j1 check_base
+      env:
+        REBUILD_DOCS: "FALSE"
+        RUN_TESTS: "FALSE"
+
+    - name: check for out-of-date files
+      uses: infotroph/tree-is-clean@v1
+
+  check_modules:
+    if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build')
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        R:
+          - "4.0"
+          - "4.1"
+
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      _R_CHECK_LENGTH_1_CONDITION_: true
+      _R_CHECK_LENGTH_1_LOGIC2_: true
+      # Avoid compilation check warnings that come from the system Makevars
+      # See https://stat.ethz.ch/pipermail/r-package-devel/2019q2/003898.html
+      _R_CHECK_COMPILATION_FLAGS_KNOWN_: -Wformat -Werror=format-security -Wdate-time
+      # Keep R checks from trying to consult the very flaky worldclockapi.com
+      _R_CHECK_SYSTEM_CLOCK_: 0
+
+    container: 
+      image: pecan/depends:R${{ matrix.R }}
+
+    steps:
+    # checkout source code
+    - name: work around https://github.com/actions/checkout/issues/766
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - uses: actions/checkout@v3
+      with:
+        set-safe-directory: false
+
+    # Forbid spaces in names. Yes, *we* know it's not 1980 anymore, but Make doesn't.
+    - name: check for filenames that would confuse Make
+      run: |
+        SPACENAMES=`find . -name '* *'`
+        if [ -n "$SPACENAMES" ]; then
+          echo "::error file=${SPACENAMES}::Spaces in filename(s): ${SPACENAMES}. Please rename these files by converting spaces to underscores."
+          exit 1
+        fi
+
+    # install additional tools needed
+    - name: install utils
+      run: apt-get update && apt-get install -y postgresql-client qpdf
+    - name: install new dependencies
+      run: Rscript scripts/generate_dependencies.R && Rscript docker/depends/pecan.depends.R
+
+    # run PEcAn checks
+    - name: check
+      run: make -j1 check_modules
+      env:
+        REBUILD_DOCS: "FALSE"
+        RUN_TESTS: "FALSE"
+
+    - name: check for out-of-date files
+      uses: infotroph/tree-is-clean@v1
+
+  check_models:
+    if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build')
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        R:
+          - "4.0"
+          - "4.1"
+
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      _R_CHECK_LENGTH_1_CONDITION_: true
+      _R_CHECK_LENGTH_1_LOGIC2_: true
+      # Avoid compilation check warnings that come from the system Makevars
+      # See https://stat.ethz.ch/pipermail/r-package-devel/2019q2/003898.html
+      _R_CHECK_COMPILATION_FLAGS_KNOWN_: -Wformat -Werror=format-security -Wdate-time
+      # Keep R checks from trying to consult the very flaky worldclockapi.com
+      _R_CHECK_SYSTEM_CLOCK_: 0
+
+    container: 
+      image: pecan/depends:R${{ matrix.R }}
+
+    steps:
+    # checkout source code
+    - name: work around https://github.com/actions/checkout/issues/766
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - uses: actions/checkout@v3
+      with:
+        set-safe-directory: false
+
+    # Forbid spaces in names. Yes, *we* know it's not 1980 anymore, but Make doesn't.
+    - name: check for filenames that would confuse Make
+      run: |
+        SPACENAMES=`find . -name '* *'`
+        if [ -n "$SPACENAMES" ]; then
+          echo "::error file=${SPACENAMES}::Spaces in filename(s): ${SPACENAMES}. Please rename these files by converting spaces to underscores."
+          exit 1
+        fi
+
+    # install additional tools needed
+    - name: install utils
+      run: apt-get update && apt-get install -y postgresql-client qpdf
+    - name: install new dependencies
+      run: Rscript scripts/generate_dependencies.R && Rscript docker/depends/pecan.depends.R
+
+    # run PEcAn checks
+    - name: check
+      run: make -j1 check_models
       env:
         REBUILD_DOCS: "FALSE"
         RUN_TESTS: "FALSE"

CHANGELOG.md

@@ -41,6 +41,9 @@ see if you need to change any of these:
 - Added Observation preparation functions into the SDA workflow, which supports AGB, LAI, Soil Carbon, and Soil moisture.
 - We are slowly change the license from NCSA opensource to BSD-3 to help with publishing PEcAn to CRAN.
 - Added an optional `pfts` argument to `PEcAn.uncertainty::run.sensitivity.analysis()` so that sensitivity analysis and variance decomposition can be run on a subset of PFTs defined in `settings` if desired (#3155).
+- Added new features of the SDA function including: 1) allow user-defined free-run mode; 
+2) allow user-defined parallel mode for the qsub submission; 3) allow user-defined email option to report the progress.
+- The analysis function now supports the parallelization of multi-chain MCMC sampling with the fully randomized inits function.
 
 ### Fixed
 
@@ -80,6 +83,9 @@ convert data for a single PFT fixed (#1329, #2974, #2981)
 - DDBH (change in DBH over time) is no longer extracted and summarized from monthly -E- files by `PEcAn.ED2::model2netcdf.ED2()`.  We are not sure it makes sense to summarize this variable across cohorts of different sizes.
 - The `yr` and `yfiles` arguments of `PEcAn.ED2::read_E_files()` are no longer used and the simulation date is extracted from the names of the .h5 files output by ED2.
 - Update Dockerfile for sipnet/maespa/template to use pecan/models:tag to build.
+- Fixed a bug in `PEcAn.utils::ud_convert()` where it failed with objects of class "difftime" introduced by refactoring to use the `units` package instead of `udunits` (#3012)
+- The propagation of aqq and bqq for the SINGLE Q type has been corrected.
+- The issue where the indirect constraints will be increase with the increase of the covariance sizes.
 
 ### Changed
 
@@ -101,6 +107,7 @@ convert data for a single PFT fixed (#1329, #2974, #2981)
 - Internal changes to keep up to date with tidyselect v1.2.0
 - The `PEcAn.utils::download.file()` function has now been renamed to `PEcAn.utils::download_file()`
 - The `regrid()` and `grid2netcdf()` function from `PEcAn.utils` have been moved to `PEcAn.data.remote` package.
+- Web is now build using PHP 8 since PHP 7 is EOL.
 
 ### Removed
 

DEV-INTRO.md

@@ -34,13 +34,13 @@ The use of Docker in PEcAn is described in detail in the [PEcAn documentation](h
 ### Installing Docker
 
 To install Docker and docker-compose, see the docker documentation:
-- Docker Desktop in [Mac OSX](https://docs.docker.com/docker-for-mac/install/) or [Windows](https://docs.docker.com/docker-for-windows/install/)
+- Docker Desktop in [MacOS](https://docs.docker.com/docker-for-mac/install/) or [Windows](https://docs.docker.com/docker-for-windows/install/)
 - Docker (e.g. [Ubuntu](https://docs.docker.com/compose/install/)) and [docker-compose](https://docs.docker.com/compose/install/) on your linux operating system.
 
 _Note for Linux users:_ add your user to the docker group. This will prevent you from having to use `sudo` to start the docker containers, and makes sure that any file that is written to a mounted volume is owned by you. This can be done using
 ```sh
 # for linux users
-sudo adduser ${USER} docker`.
+sudo adduser ${USER} docker.
 ```
 
 ### Deploying PEcAn in Docker
@@ -235,7 +235,7 @@ You can now modify the code on your local machine, or you can use [rstudio](http
 
 The compiled code is written to `/usr/local/lib/R/site-library` which is mapped to `volumes/lib` on your machine. This same folder is mounted in many other containers, allowing you to share the same PEcAn modules in all containers. Now if you change a module, and compile all other containers will see and use this new version of your module.
 
-To compile the PEcAn code you can use the make command in either the rstudio container, or in the executor container. The script [`compile.sh`](sripts/compile.sh) will run make inside the executor container.
+To compile the PEcAn code you can use the make command in either the rstudio container, or in the executor container. The script [`compile.sh`](scripts/compile.sh) will run make inside the executor container.
 
 ### Workflow Submission
 

Makefile

@@ -96,10 +96,19 @@ depends = .doc/$(1) .install/$(1) .check/$(1) .test/$(1)
 
 ### Rules
 
-.PHONY: all install check test document shiny
+.PHONY: all install check test document shiny \
+            check_base check_models check_modules 
 
 all: install document
 
+
+check_base: $(BASE_C) 
+check_models: $(MODELS_C) 
+
+# Install base first as Modules has a circular dependency on base,
+# and then run a check on modules
+check_modules: $(BASE_I) $(MODULES_C) 
+
 document: $(ALL_PKGS_D) .doc/base/all
 install: $(ALL_PKGS_I) .install/base/all
 check: $(ALL_PKGS_C) .check/base/all

README.md

@@ -31,8 +31,8 @@ See our ["Tutorials Page"](https://pecanproject.github.io/tutorials.html) that p
 ### Installation
 
 Complete instructions on how to install PEcAn can be found in the [documentation here](https://pecanproject.github.io/pecan-documentation/develop/pecan-manual-setup.html). To get PEcAn up and running you can use one of three methods:
-1. Run a [Virtual Machine](https://pecanproject.github.io/pecan-documentation/develop/pecan-manual-setup.html#install-vm). This is recommended for students and new users, and provides a consistent, tested environment for each release.
-2. Use [Docker](https://pecanproject.github.io/pecan-documentation/develop/pecan-manual-setup.html#install-docker). This is recommended, especially for development and production deployment.
+1. Run a [Virtual Machine](https://pecanproject.github.io/pecan-documentation/develop/install-vm.html#install-vm). This is recommended for students and new users, and provides a consistent, tested environment for each release.
+2. Use [Docker](https://pecanproject.github.io/pecan-documentation/develop/install-docker.html#install-docker). This is recommended, especially for development and production deployment.
 3. Install all of the PEcAn R packages on your own Linux or MacOS computer or server. This can be done by [installing from r-universe](https://pecanproject.github.io/pecan-documentation/develop/r-universe.html): 
 ``` r
 # Enable repository from pecanproject

apps/api/R/submit.workflow.R

@@ -124,7 +124,7 @@ submit.workflow.list <- function(workflowList, userDetails) {
 
   # Post workflow to RabbitMQ
   message <- list(folder = outdir, workflowid = workflow_id_str)
-  res <- PEcAn.remote::rabbitmq_post_message(workflowList$host$rabbitmq$uri, "pecan", message, "rabbitmq")
+  res <- PEcAn.remote::rabbitmq_post_message(workflowList$host$rabbitmq$uri, "pecan", message)
 
   if(res$routed){
     return(list(workflow_id = workflow_id_str, status = "Submitted successfully"))

base/db/DESCRIPTION

@@ -66,21 +66,24 @@ Suggests:
     data.table,
     here,
     knitr,
+    mockery,
     RPostgreSQL,
     RPostgres,
     RSQLite,
     rcrossref,
     rmarkdown (>= 2.19),
     testthat (>= 2.0.0),
-    tidyverse
+    tidyverse,
+    withr
 X-Comment-Remotes:
     Installing markdown from GitHub because as of 2023-02-05, this is the
     easiest way to get version >= 2.19 onto Docker images that use older
     Rstudio Package Manager snapshots.
     When building on a system that finds a new enough version on CRAN,
     OK to remove the Remotes line and this comment.
 Remotes:
-    github::rstudio/[email protected]
+    github::rstudio/[email protected],
+    github::r-lib/[email protected]
 License: BSD_3_clause + file LICENSE
 VignetteBuilder: knitr
 Copyright: Authors

base/db/R/assign.treatments.R

@@ -13,7 +13,7 @@
 ##' Assigns all control treatments the same value, then assigns unique treatments
 ##' within each site. Each site is required to have a control treatment.
 ##' The algorithm (incorrectly) assumes that each site has a unique set of experimental
-##' treatments. This assumption is required by the data in BETTdb that does not always consistently name treatments or quantity them in the managements table. Also it avoids having the need to estimate treatment by site interactions in the meta analysis model. This model uses data in the control treatment to estimate model parameters so the impact of the assumption is minimal.
+##' treatments. This assumption is required by the data in BETYdb that does not always consistently name treatments or quantity them in the managements table. Also it avoids having the need to estimate treatment by site interactions in the meta analysis model. This model uses data in the control treatment to estimate model parameters so the impact of the assumption is minimal.
 ##' @name assign.treatments
 ##' @title assign.treatments
 ##' @param data input data

base/db/R/dbfiles.R

@@ -71,7 +71,7 @@ dbfile.input.insert <- function(in.path, in.prefix, siteid, startdate, enddate,
       "SELECT * FROM inputs WHERE site_id=", siteid,
       " AND name= '", name,
       "' AND format_id=", formatid,
-      parent
+      parent, ";"
     ),
     con = con
   )
@@ -120,26 +120,26 @@ dbfile.input.insert <- function(in.path, in.prefix, siteid, startdate, enddate,
         "INSERT INTO inputs ",
         "(site_id, format_id, name) VALUES (",
         siteid, ", ", formatid, ", '", name,
-        "'", ") RETURNING id"
+        "'", ") RETURNING id;"
       )
     } else if (parent == "" && !is.null(startdate)) {
       cmd <- paste0(
         "INSERT INTO inputs ",
         "(site_id, format_id, start_date, end_date, name) VALUES (",
         siteid, ", ", formatid, ", '", startdate, "', '", enddate, "','", name,
-        "') RETURNING id"
+        "') RETURNING id;"
       )
     } else if (is.null(startdate)) {
       cmd <- paste0(
         "INSERT INTO inputs ",
         "(site_id, format_id, name, parent_id) VALUES (",
-        siteid, ", ", formatid, ", '", name, "',", parentid, ") RETURNING id"
+        siteid, ", ", formatid, ", '", name, "',", parentid, ") RETURNING id;"
       )
     } else {
       cmd <- paste0(
         "INSERT INTO inputs ",
         "(site_id, format_id, start_date, end_date, name, parent_id) VALUES (",
-        siteid, ", ", formatid, ", '", startdate, "', '", enddate, "','", name, "',", parentid, ") RETURNING id"
+        siteid, ", ", formatid, ", '", startdate, "', '", enddate, "','", name, "',", parentid, ") RETURNING id;"
       )
     }
     # This is the id that we just registered
@@ -150,7 +150,7 @@ dbfile.input.insert <- function(in.path, in.prefix, siteid, startdate, enddate,
       inputid <- db.query(
         query = paste0(
           "SELECT id FROM inputs WHERE site_id=", siteid,
-          " AND format_id=", formatid
+          " AND format_id=", formatid, ";"
         ),
         con = con
       )$id

base/db/R/query.dplyr.R

@@ -61,7 +61,7 @@ dplyr.count <- function(df) {
 #' @param unit string containing CF-style time unit including origin (e.g. "days since 2010-01-01")
 #' @export
 ncdays2date <- function(time, unit) {
-  date    <- lubridate::parse_date_time(unit, c("ymd_hms", "ymd_h", "ymd"))
+  date    <- lubridate::parse_date_time(unit, c("ymd_HMS", "ymd_H", "ymd"))
   days    <- PEcAn.utils::ud_convert(time, unit, paste("days since ", date))
   seconds <- PEcAn.utils::ud_convert(days, "days", "seconds")
   return(as.POSIXct.numeric(seconds, origin = date, tz = "UTC"))
@@ -124,7 +124,7 @@ workflows <- function(bety, ensemble = FALSE) {
 #' @export
 workflow <- function(bety, workflow_id) {
   workflows(bety) %>%
-    dplyr::filter(.data$workflow_id == !!.data$workflow_id)
+    dplyr::filter(.data$workflow_id == !!workflow_id)
 }  # workflow
 
 

base/db/R/query.file.path.R

@@ -8,11 +8,11 @@
 ##' @author Betsy Cowdery
 query.file.path <- function(input.id, host_name, con){
   machine.host <- PEcAn.DB::default_hostname(host_name)
-  machine <- db.query(query = paste0("SELECT * from machines where hostname = '",machine.host,"'"), con = con)
+  machine <- db.query(query = paste0("SELECT * from machines where hostname = '",machine.host,"';"), con = con)
   dbfile <- db.query(
     query = paste(
       "SELECT file_name,file_path from dbfiles where container_id =", input.id,
-      " and container_type = 'Input' and machine_id =", machine$id
+      " and container_type = 'Input' and machine_id =", machine$id, ";"
     ),
     con = con
   )

base/db/R/query.yields.R

@@ -24,21 +24,28 @@ query.yields <- function(trait = 'yield', spstr, extra.columns = '', con = NULL,
                          ids_are_cultivars = FALSE, ...){
 
   member_column <- if (ids_are_cultivars) {"cultivar_id"} else {"specie_id"}
+
+  if(!is.null(extra.columns)) {
+    if(!is.character(extra.columns) || length(extra.columns) != 1) {
+      PEcAn.logger::logger.severe("`extra.columns` must be a string")
+    }
+  }
+
   query <- paste("select
             yields.id, yields.citation_id, yields.site_id, treatments.name,
             yields.date, yields.time, yields.cultivar_id, yields.specie_id,
             yields.mean, yields.statname, yields.stat, yields.n,
             variables.name as vname,
             month(yields.date) as month,",
-                 extra.columns,
+                if(extra.columns != '') { paste(extra.columns, ",", sep = "") } else {""},
                  "treatments.control, sites.greenhouse
           from yields
             left join treatments on  (yields.treatment_id = treatments.id)
             left join sites on (yields.site_id = sites.id)
             left join variables on (yields.variable_id = variables.id)
           where ", member_column, " in (", spstr,");", sep = "")
   if(!trait == 'yield'){
-    query <- gsub(");", paste(" and variables.name in ('", trait,"');", sep = ""), query)
+    query <- gsub(";", paste(" and variables.name in ('", trait,"');", sep = ""), query)
   }
 
   return(fetch.stats2se(connection = con, query = query))