From ea0f9e2a0b6b935dc50e605443052143df4194cf Mon Sep 17 00:00:00 2001 From: Chris Black Date: Mon, 11 Dec 2023 00:59:15 -0800 Subject: [PATCH 1/5] install from newer CRAN if needed to satisfy version spec --- Makefile.depends | 57 ++++---- docker/depends/pecan.depends.R | 49 ++++--- scripts/generate_dependencies.R | 245 ++++++++++++++++++++++---------- 3 files changed, 225 insertions(+), 126 deletions(-) diff --git a/Makefile.depends b/Makefile.depends index 652db3c287a..d5327b65cac 100644 --- a/Makefile.depends +++ b/Makefile.depends @@ -1,44 +1,41 @@ # autogenerated -$(call depends,base/all): | .install/base/db .install/base/settings .install/modules/meta.analysis .install/base/logger .install/base/utils .install/modules/uncertainty .install/modules/data.atmosphere .install/modules/data.land .install/modules/data.remote .install/modules/assim.batch .install/modules/emulator .install/modules/priors .install/modules/benchmark .install/base/remote .install/base/workflow .install/models/ed .install/models/sipnet .install/models/biocro .install/models/dalec .install/models/linkages .install/modules/allometry .install/modules/photosynthesis +$(call depends,base/all): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/base/workflow .install/models/biocro .install/models/dalec .install/models/ed .install/models/linkages .install/models/sipnet .install/modules/allometry .install/modules/assim.batch .install/modules/benchmark .install/modules/data.atmosphere .install/modules/data.land .install/modules/data.remote .install/modules/emulator .install/modules/meta.analysis .install/modules/photosynthesis .install/modules/priors .install/modules/uncertainty $(call depends,base/db): | .install/base/logger .install/base/remote .install/base/utils -$(call depends,base/logger): | -$(call depends,base/qaqc): | .install/base/db .install/base/logger .install/models/biocro .install/models/ed .install/models/sipnet .install/base/utils +$(call depends,base/qaqc): | .install/base/db .install/base/logger .install/base/utils .install/models/biocro .install/models/ed .install/models/sipnet $(call depends,base/remote): | .install/base/logger $(call depends,base/settings): | .install/base/db .install/base/logger .install/base/remote .install/base/utils $(call depends,base/utils): | .install/base/logger $(call depends,base/visualization): | .install/base/db .install/base/logger .install/base/utils -$(call depends,base/workflow): | .install/modules/data.atmosphere .install/modules/data.land .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/modules/uncertainty .install/base/utils -$(call depends,modules/allometry): | .install/base/db -$(call depends,modules/assim.batch): | .install/modules/benchmark .install/base/db .install/modules/emulator .install/base/logger .install/modules/meta.analysis .install/base/remote .install/base/settings .install/modules/uncertainty .install/base/utils .install/base/workflow -$(call depends,modules/assim.sequential): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/modules/uncertainty .install/base/workflow .install/modules/benchmark .install/modules/data.remote -$(call depends,modules/benchmark): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/modules/data.land -$(call depends,modules/data.atmosphere): | .install/base/db .install/base/logger .install/base/remote .install/base/utils -$(call depends,modules/data.hydrology): | .install/base/logger .install/base/utils -$(call depends,modules/data.land): | .install/modules/benchmark .install/modules/data.atmosphere .install/base/db .install/base/logger .install/base/remote .install/base/utils .install/base/visualization .install/base/settings -$(call depends,modules/data.remote): | .install/base/db .install/base/utils .install/base/logger .install/base/remote -$(call depends,modules/emulator): | -$(call depends,modules/meta.analysis): | .install/base/utils .install/base/db .install/base/logger .install/base/settings -$(call depends,modules/photosynthesis): | -$(call depends,modules/priors): | .install/base/logger .install/modules/meta.analysis .install/base/utils .install/base/visualization -$(call depends,modules/rtm): | .install/base/logger .install/modules/assim.batch .install/base/utils .install/models/ed -$(call depends,modules/uncertainty): | .install/base/db .install/modules/emulator .install/base/logger .install/modules/priors .install/base/settings .install/base/utils -$(call depends,models/basgra): | .install/base/logger .install/modules/data.atmosphere .install/base/utils -$(call depends,models/biocro): | .install/base/logger .install/base/remote .install/base/utils .install/base/settings .install/modules/data.atmosphere .install/modules/data.land .install/base/db +$(call depends,base/workflow): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/modules/data.atmosphere .install/modules/data.land .install/modules/uncertainty +$(call depends,models/basgra): | .install/base/logger .install/base/utils .install/modules/data.atmosphere +$(call depends,models/biocro): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/modules/data.atmosphere .install/modules/data.land $(call depends,models/cable): | .install/base/logger .install/base/utils $(call depends,models/clm45): | .install/base/logger .install/base/utils $(call depends,models/dalec): | .install/base/logger .install/base/remote .install/base/utils $(call depends,models/dvmdostem): | .install/base/logger .install/base/utils -$(call depends,models/ed): | .install/modules/data.atmosphere .install/modules/data.land .install/base/logger .install/base/remote .install/base/settings .install/base/utils +$(call depends,models/ed): | .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/modules/data.atmosphere .install/modules/data.land $(call depends,models/fates): | .install/base/logger .install/base/remote .install/base/utils -$(call depends,models/gday): | .install/base/utils .install/base/logger .install/base/remote -$(call depends,models/jules): | .install/modules/data.atmosphere .install/base/logger .install/base/remote .install/base/utils -$(call depends,models/ldndc): | .install/base/db .install/base/logger .install/base/utils .install/base/remote .install/modules/data.atmosphere -$(call depends,models/linkages): | .install/base/utils .install/modules/data.atmosphere .install/base/logger .install/base/remote +$(call depends,models/gday): | .install/base/logger .install/base/remote .install/base/utils +$(call depends,models/jules): | .install/base/logger .install/base/remote .install/base/utils .install/modules/data.atmosphere +$(call depends,models/ldndc): | .install/base/db .install/base/logger .install/base/remote .install/base/utils .install/modules/data.atmosphere +$(call depends,models/linkages): | .install/base/logger .install/base/remote .install/base/utils .install/modules/data.atmosphere $(call depends,models/lpjguess): | .install/base/logger .install/base/remote .install/base/utils -$(call depends,models/maat): | .install/modules/data.atmosphere .install/base/logger .install/base/remote .install/base/settings .install/base/utils -$(call depends,models/maespa): | .install/modules/data.atmosphere .install/base/logger .install/base/remote .install/base/utils -$(call depends,models/preles): | .install/base/utils .install/base/logger .install/modules/data.atmosphere .install/base/utils +$(call depends,models/maat): | .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/modules/data.atmosphere +$(call depends,models/maespa): | .install/base/logger .install/base/remote .install/base/utils .install/modules/data.atmosphere +$(call depends,models/preles): | .install/base/logger .install/base/utils .install/modules/data.atmosphere $(call depends,models/sibcasa): | .install/base/logger .install/base/utils -$(call depends,models/sipnet): | .install/modules/data.atmosphere .install/base/logger .install/base/remote .install/base/utils -$(call depends,models/stics): | .install/base/settings .install/base/db .install/base/logger .install/base/utils .install/base/remote +$(call depends,models/sipnet): | .install/base/logger .install/base/remote .install/base/utils .install/modules/data.atmosphere +$(call depends,models/stics): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils $(call depends,models/template): | .install/base/db .install/base/logger .install/base/utils +$(call depends,modules/allometry): | .install/base/db +$(call depends,modules/assim.batch): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/base/workflow .install/modules/benchmark .install/modules/emulator .install/modules/meta.analysis .install/modules/uncertainty +$(call depends,modules/assim.sequential): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/workflow .install/modules/benchmark .install/modules/data.remote .install/modules/uncertainty +$(call depends,modules/benchmark): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/modules/data.land +$(call depends,modules/data.atmosphere): | .install/base/db .install/base/logger .install/base/remote .install/base/utils +$(call depends,modules/data.hydrology): | .install/base/logger .install/base/utils +$(call depends,modules/data.land): | .install/base/db .install/base/logger .install/base/remote .install/base/settings .install/base/utils .install/base/visualization .install/modules/benchmark .install/modules/data.atmosphere +$(call depends,modules/data.remote): | .install/base/db .install/base/logger .install/base/remote .install/base/utils +$(call depends,modules/meta.analysis): | .install/base/db .install/base/logger .install/base/settings .install/base/utils +$(call depends,modules/priors): | .install/base/logger .install/base/utils .install/base/visualization .install/modules/meta.analysis +$(call depends,modules/rtm): | .install/base/logger .install/base/utils .install/models/ed .install/modules/assim.batch +$(call depends,modules/uncertainty): | .install/base/db .install/base/logger .install/base/settings .install/base/utils .install/modules/emulator .install/modules/priors diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index e49b8fb72fb..12ae384634b 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -22,9 +22,9 @@ remotes::install_github(c( 'rstudio/rmarkdown@v2.20' ), lib = rlib) -# install all packages (depends, imports, suggests) +# install package listed as Depends, Imports, Suggests of any PEcAn package +# that do not have a stated version limit wanted <- c( -'abind', 'amerifluxr', 'arrow', 'assertthat', @@ -32,7 +32,6 @@ wanted <- c( 'BioCro', 'bit64', 'BrownDog', -'coda', 'corrplot', 'curl', 'data.table', @@ -51,7 +50,6 @@ wanted <- c( 'fs', 'furrr', 'future', -'geonames', 'getPass', 'ggmap', 'ggmcmc', @@ -72,9 +70,7 @@ wanted <- c( 'lattice', 'linkages', 'lqmm', -'lubridate', 'Maeswrap', -'magic', 'magrittr', 'maps', 'markdown', @@ -86,11 +82,8 @@ wanted <- c( 'mgcv', 'minpack.lm', 'mlegp', -'mockery', -'MODISTools', 'mvbutils', 'mvtnorm', -'ncdf4', 'neonstore', 'neonUtilities', 'nimble', @@ -98,11 +91,9 @@ wanted <- c( 'optparse', 'parallel', 'plotrix', -'plyr', 'png', 'prodlim', 'progress', -'purrr', 'pwr', 'R.utils', 'randtoolbox', @@ -111,13 +102,10 @@ wanted <- c( 'REddyProc', 'redland', 'reshape', -'reshape2', 'reticulate', 'rjags', 'rjson', -'rlang', 'rlist', -'rmarkdown', 'RPostgres', 'RPostgreSQL', 'Rpreles', @@ -128,11 +116,9 @@ wanted <- c( 'sp', 'stats', 'stringi', -'stringr', 'suntools', 'swfscMisc', 'terra', -'testthat', 'tibble', 'tictoc', 'tidyr', @@ -140,17 +126,42 @@ wanted <- c( 'tidyverse', 'tools', 'traits', -'TruncatedNormal', 'truncnorm', 'units', 'urltools', 'utils', -'vdiffr', 'withr', -'XML', 'xtable', 'xts', 'zoo' ) missing <- wanted[!(wanted %in% installed.packages()[,'Package'])] install.packages(missing, lib = rlib) + +# Install packages listed as Depends, Imports, Suggests +# that list a minimum version. +# When the minimum is not satisfied in the fixed-date CRAN snapshot +# used by our Docker images, we pull it in from an up-to-date mirror. +# (Assumes our CRAN uses the same URL scheme as Posit package manager) +options(repos = c( + getOption('repos'), + sub(r'(\d{4}-\d{2}-\d{2})', 'latest', getOption('repos')) +)) +remotes::install_version('MODISTools', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('TruncatedNormal', '>= 2.2', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('XML', '>= 3.98-1.4', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('abind', '>= 1.4.5', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('coda', '>= 0.18', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('geonames', '> 0.998', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('lubridate', '>= 1.7.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('magic', '>= 1.5.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('mockery', '>= 0.3.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('ncdf4', '>= 1.15', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('plyr', '>= 1.8.4', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('purrr', '>= 0.2.3', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('reshape2', '>= 1.4.2', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('rlang', '>= 0.2.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('rmarkdown', '>= 2.19', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('stringr', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('testthat', '>= 3.0.4', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('vdiffr', '>= 1.0.2', dependencies = TRUE, upgrade = FALSE) diff --git a/scripts/generate_dependencies.R b/scripts/generate_dependencies.R index 832ba7ec92e..a21378ab00f 100755 --- a/scripts/generate_dependencies.R +++ b/scripts/generate_dependencies.R @@ -1,88 +1,167 @@ #!/usr/bin/env Rscript -# force sorting +## Parse the dependencies of each PEcAn package as listed in DESCRIPTION files, +## and update two scripts to match: +## * Makefile.depends, which tracks the dependencies between PEcAn packages +## so that Make can install them in the correct order +## * docker/depends/pecan.depends.R. which preinstalls non-PEcAn dependencies +## on the Docker image (or on other systems if you call it directly). +## Run this script after each time you change a DESCRIPTION, +## and commit any resulting changes to Makefile.depends or pecan.depends.R + + + + + +# Get remote refs from a package description, +# formatted for install_github. +# @param pkg_desc as read by `desc::desc` +# @return vector of remote refs, +# with names set to their (inferred) package names +extract_remotes <- function(pkg_desc) { + remote_sources <- pkg_desc$get_remotes() + non_gh <- !grepl("^github::", remote_sources) + if (any(non_gh)) { + warning( + "Found `Remotes` address pointing to non-Github repo: ", + remote_sources[non_gh], + "pecan.depends.R only supports github remotes so far, ", + "so skipping.") + } + + sub("^github::", "", remote_sources[!non_gh]) +} + + +# Read DESCRIPTION file, return a list of: +# strings `pkgname`, `dirname` +# character vectors `pecan_deps`, `gh_deps` +# data frame `deps`, with columns +# `type`, `package`, `version`, `needed_by_dir`, `is_pecan`, `is_remote` +parse_desc <- function(path) { + d <- desc::desc(file = path) + deps <- d$get_deps() + deps$needed_by_dir <- dirname(path) + + # Find packages listed as remote + remote_sources <- extract_remotes(d) + # deps$is_remote <- deps$package %in% names(remote_sources) + + list( + mapping = data.frame( + package = d$get_field("Package"), + package_dir = dirname(path)), + remotes = remote_sources, + deps = deps) +} + + +# find the latest of several possible minimum package versions +condense_version_requirements <- function(specs) { + if (all(specs == "*")) { + # any version is acceptable + return("*") + } + specs <- unique(specs[specs != "*"]) + if (any(!grepl(">", specs))) { + # we *could* write more to handle this case if needed, but it seems very rare: + # Zero of the 477 packages in my R library declare a `<=` or `==` + stop( + "generate_dependencies only supports minimum versions (e.g. `>= 2.8.1`), ", + "not exact (`==`) or maximum versions (`<=`).", + "Problem seen in (", paste(dQuote(specs), collapse = ", "), ")") + } + versions <- package_version( + gsub("[^[:digit:].-]+", "", specs)) + + specs[versions == max(versions)] +} + + + + + +# force sorting to hopefully write lines in same order on every machine if (capabilities("ICU")) { icuSetCollate(locale = "en_US.UTF-8") } else { print("Can not force sorting, this could result in unpredicted results.") } -# following modules will be ignored +# Find all PEcAn packages... +files <- list.files( + path = c("base", "modules", "models"), + full.names = TRUE, + pattern = "^DESCRIPTION$", + recursive = TRUE +) +# ...but don't do anything with these ones ignore <- c("modules/data.mining") +files <- files[!(dirname(files) %in% ignore)] + + +# Read all DESCRIPTIONS and compile all deps into one list +pkgs_parsed <- purrr::map(files, parse_desc) +deps <- pkgs_parsed |> + purrr:::map_dfr("deps") |> + # ignore R version requirements (e.g. "Depends: R (>= 3.2.0)") + dplyr::filter(package != "R") |> + dplyr::mutate(is_pecan = grepl("^PEcAn", package)) + + + +# Map out dependencies between PEcAn packages, write to makefile +pecan_name_map <- purrr::map_dfr(pkgs_parsed, "mapping") +pecan_deps <- deps |> + dplyr::filter(is_pecan) |> + dplyr::left_join(pecan_name_map, by = "package") |> + dplyr::group_by(needed_by_dir) |> + dplyr::summarize( + dep_list = paste0(".install/", sort(unique(package_dir)), collapse = " "), + call_txt = paste0("$(call depends,", unique(needed_by_dir), "): | ", dep_list), + .groups = "drop") |> + dplyr::pull(call_txt) + +cat( + c("# autogenerated", pecan_deps), + file = "Makefile.depends", + sep = "\n", + append = FALSE) + + +# list all the GitHub remotes mentioned by any package, +# to attempt installation from these at start of Docker build +remote_repos <- sapply(pkgs_parsed, \(x)x$remotes) |> + unlist() |> + sort() + + +# For deps used by multiple packages, find a version that works for all +uniq_deps <- deps |> + dplyr::filter(!is_pecan) |> + dplyr::group_by(package) |> + dplyr::summarize( + version = condense_version_requirements(version), + .groups = "drop") |> + dplyr::arrange(package) + +# Deps without any restriction on version installed. +# We'll install these with one plain old `install.packages()` call. +unversioned_deps <- uniq_deps[uniq_deps$version == "*",]$package + +# Deps that need a set minimum version. +# We'll install these with `remotes::install_version`, +# directing it to look outside our fixed-date CRAN snapshot if +# it can't fill the version req from snapshot versions. +versioned_dep_install_calls <- uniq_deps[uniq_deps$version != "*",] |> + dplyr::mutate( + inst_call = paste0( + "remotes::install_version(", + shQuote(package), ", ", shQuote(version), + ", dependencies = TRUE, upgrade = FALSE)")) |> + dplyr::pull(inst_call) -files <- - c( - list.files( - path = "base", - full.names = TRUE, - pattern = "^DESCRIPTION$", - recursive = TRUE - ), - list.files( - path = "modules", - full.names = TRUE, - pattern = "^DESCRIPTION$", - recursive = TRUE - ), - list.files( - path = "models", - full.names = TRUE, - pattern = "^DESCRIPTION$", - recursive = TRUE - ) - ) - -## Required dependencies -pecan <- c() -depends <- c() -docker <- c() -remotes <- c() -d <- purrr::walk( - files, - function(x) { - f <- dirname(x) - if (f %in% ignore) return() - - # load DESCRIPTION file - d <- desc::desc(file = x) - deps <- d$get_deps()[["package"]] - # ignore R version requirements (e.g. "Depends: R (>= 3.2.0)") - deps <- deps[deps != "R"] - - # PEcAn dependencies - y <- deps[grepl("^PEcAn", deps)] - p <- d$get_field("Package") - pecan[[p]] <<- f - depends[[f]] <<- y - - # Dockerfile dependencies - z <- y["package"] - z <- deps[!grepl("^PEcAn", deps)] - docker <<- unique(c(docker, z)) - - # Dockerfile remote dependencies - if (!purrr::is_empty(d$get_remotes())) { - deps <- d$get_remotes() - github <- gsub("github::", "", deps[grepl("^github::", deps)]) - remotes <<- unique(c(remotes, github)) - - bad <- deps[!grepl("^github::", deps)] - if (!purrr::is_empty(bad)) { - print(bad) - } - } - } -) -# write for makefile -cat("# autogenerated", file = "Makefile.depends", sep = "\n", append = FALSE) -for (name in names(depends)) { - x <- paste0("$(call depends,", name, "): |") - for (p in depends[[name]]) { - x <- paste0(x, " .install/", pecan[[p]]) - } - cat(x, file = "Makefile.depends", sep = "\n", append = TRUE) -} # write for docker dependency image cat("#!/usr/bin/env Rscript", @@ -96,11 +175,23 @@ cat("#!/usr/bin/env Rscript", "", "# install remotes first in case packages are references in dependencies", "remotes::install_github(c(", - paste(shQuote(sort(remotes)), collapse = ",\n"), + paste(shQuote(sort(remote_repos)), collapse = ",\n"), "), lib = rlib)", "", - "# install all packages (depends, imports, suggests)", - "wanted <- c(", paste(shQuote(sort(docker)), sep = "", collapse = ",\n"), ")", + "# install package listed as Depends, Imports, Suggests of any PEcAn package", + "# that do not have a stated version limit", + "wanted <- c(", paste(shQuote(sort(unversioned_deps)), sep = "", collapse = ",\n"), ")", "missing <- wanted[!(wanted %in% installed.packages()[,'Package'])]", "install.packages(missing, lib = rlib)", + "", + "# Install packages listed as Depends, Imports, Suggests", + "# that list a minimum version.", + "# When the minimum is not satisfied in the fixed-date CRAN snapshot", + "# used by our Docker images, we pull it in from an up-to-date mirror.", + "# (Assumes our CRAN uses the same URL scheme as Posit package manager)", + "options(repos = c(", + " getOption('repos'),", + " sub(r'(\\d{4}-\\d{2}-\\d{2})', 'latest', getOption('repos'))", + "))", + paste(versioned_dep_install_calls, collapse = "\n"), file = "docker/depends/pecan.depends.R", sep = "\n", append = FALSE) From 41fcb25e4b22f74bf318ebb341eaae4450619ac6 Mon Sep 17 00:00:00 2001 From: Chris Black Date: Mon, 11 Dec 2023 01:05:13 -0800 Subject: [PATCH 2/5] no longer need to install these from GitHub --- base/db/DESCRIPTION | 11 +---------- base/qaqc/DESCRIPTION | 10 ---------- docker/depends/pecan.depends.R | 8 ++------ 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/base/db/DESCRIPTION b/base/db/DESCRIPTION index e8d4f273b84..f8d67075ad2 100644 --- a/base/db/DESCRIPTION +++ b/base/db/DESCRIPTION @@ -66,7 +66,7 @@ Suggests: data.table, here, knitr, - mockery, + mockery (>= 0.4.3), RPostgreSQL, RPostgres, RSQLite, @@ -75,15 +75,6 @@ Suggests: testthat (>= 2.0.0), tidyverse, withr -X-Comment-Remotes: - Installing markdown from GitHub because as of 2023-02-05, this is the - easiest way to get version >= 2.19 onto Docker images that use older - Rstudio Package Manager snapshots. - When building on a system that finds a new enough version on CRAN, - OK to remove the Remotes line and this comment. -Remotes: - github::rstudio/rmarkdown@v2.20, - github::r-lib/mockery@v0.4.3 License: BSD_3_clause + file LICENSE VignetteBuilder: knitr Copyright: Authors diff --git a/base/qaqc/DESCRIPTION b/base/qaqc/DESCRIPTION index a723730270e..d9018745440 100644 --- a/base/qaqc/DESCRIPTION +++ b/base/qaqc/DESCRIPTION @@ -31,16 +31,6 @@ Suggests: testthat (>= 3.0.4), vdiffr (>= 1.0.2), withr -X-Comment-Remotes: - Installing vdiffr from GitHub because as of 2021-09-23, this is the - easiest way to get version >= 1.0.2 onto Docker images that use older - Rstudio Package Manager snapshots. - Ditto for testthat, because we need >= 3.0.4 for vdiffr compatibility. - When building on a system that finds these versions on CRAN, - OK to remove these Remotes lines and this comment. -Remotes: - github::r-lib/testthat@v3.1.6, - github::r-lib/vdiffr@v1.0.4 License: BSD_3_clause + file LICENSE Copyright: Authors LazyLoad: yes diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index 12ae384634b..cb88d127e48 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -14,12 +14,8 @@ remotes::install_github(c( 'chuhousen/amerifluxr', 'ebimodeling/biocro@0.951', 'MikkoPeltoniemi/Rpreles', -'r-lib/mockery@v0.4.3', -'r-lib/testthat@v3.1.6', -'r-lib/vdiffr@v1.0.4', 'ropensci/geonames', -'ropensci/nneo', -'rstudio/rmarkdown@v2.20' +'ropensci/nneo' ), lib = rlib) # install package listed as Depends, Imports, Suggests of any PEcAn package @@ -155,7 +151,7 @@ remotes::install_version('coda', '>= 0.18', dependencies = TRUE, upgrade = FALSE remotes::install_version('geonames', '> 0.998', dependencies = TRUE, upgrade = FALSE) remotes::install_version('lubridate', '>= 1.7.0', dependencies = TRUE, upgrade = FALSE) remotes::install_version('magic', '>= 1.5.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('mockery', '>= 0.3.0', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('mockery', '>= 0.4.3', dependencies = TRUE, upgrade = FALSE) remotes::install_version('ncdf4', '>= 1.15', dependencies = TRUE, upgrade = FALSE) remotes::install_version('plyr', '>= 1.8.4', dependencies = TRUE, upgrade = FALSE) remotes::install_version('purrr', '>= 0.2.3', dependencies = TRUE, upgrade = FALSE) From 34c49817d01fc92cb2da971cb8bb9e3a1f82223c Mon Sep 17 00:00:00 2001 From: Chris Black Date: Mon, 11 Dec 2023 02:14:19 -0800 Subject: [PATCH 3/5] sort instead of arrange --- docker/depends/pecan.depends.R | 6 +++--- scripts/generate_dependencies.R | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index cb88d127e48..abc8e9f6864 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -143,15 +143,13 @@ options(repos = c( getOption('repos'), sub(r'(\d{4}-\d{2}-\d{2})', 'latest', getOption('repos')) )) -remotes::install_version('MODISTools', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('TruncatedNormal', '>= 2.2', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('XML', '>= 3.98-1.4', dependencies = TRUE, upgrade = FALSE) remotes::install_version('abind', '>= 1.4.5', dependencies = TRUE, upgrade = FALSE) remotes::install_version('coda', '>= 0.18', dependencies = TRUE, upgrade = FALSE) remotes::install_version('geonames', '> 0.998', dependencies = TRUE, upgrade = FALSE) remotes::install_version('lubridate', '>= 1.7.0', dependencies = TRUE, upgrade = FALSE) remotes::install_version('magic', '>= 1.5.0', dependencies = TRUE, upgrade = FALSE) remotes::install_version('mockery', '>= 0.4.3', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('MODISTools', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) remotes::install_version('ncdf4', '>= 1.15', dependencies = TRUE, upgrade = FALSE) remotes::install_version('plyr', '>= 1.8.4', dependencies = TRUE, upgrade = FALSE) remotes::install_version('purrr', '>= 0.2.3', dependencies = TRUE, upgrade = FALSE) @@ -160,4 +158,6 @@ remotes::install_version('rlang', '>= 0.2.0', dependencies = TRUE, upgrade = FAL remotes::install_version('rmarkdown', '>= 2.19', dependencies = TRUE, upgrade = FALSE) remotes::install_version('stringr', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) remotes::install_version('testthat', '>= 3.0.4', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('TruncatedNormal', '>= 2.2', dependencies = TRUE, upgrade = FALSE) remotes::install_version('vdiffr', '>= 1.0.2', dependencies = TRUE, upgrade = FALSE) +remotes::install_version('XML', '>= 3.98-1.4', dependencies = TRUE, upgrade = FALSE) diff --git a/scripts/generate_dependencies.R b/scripts/generate_dependencies.R index a21378ab00f..a6b40a1a10d 100755 --- a/scripts/generate_dependencies.R +++ b/scripts/generate_dependencies.R @@ -142,8 +142,7 @@ uniq_deps <- deps |> dplyr::group_by(package) |> dplyr::summarize( version = condense_version_requirements(version), - .groups = "drop") |> - dplyr::arrange(package) + .groups = "drop") # Deps without any restriction on version installed. # We'll install these with one plain old `install.packages()` call. @@ -159,7 +158,8 @@ versioned_dep_install_calls <- uniq_deps[uniq_deps$version != "*",] |> "remotes::install_version(", shQuote(package), ", ", shQuote(version), ", dependencies = TRUE, upgrade = FALSE)")) |> - dplyr::pull(inst_call) + dplyr::pull(inst_call) |> + sort() From b8a29ae6a730e500ecacf42d245e0bcebcbe0d36 Mon Sep 17 00:00:00 2001 From: Chris Black Date: Mon, 11 Dec 2023 14:50:37 -0800 Subject: [PATCH 4/5] avoid reinstalling (and potentially downgrading) if req already satisfied --- docker/depends/pecan.depends.R | 45 ++++++++++++++++++++------------- scripts/generate_dependencies.R | 13 +++++++--- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index abc8e9f6864..a933e1a9920 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -143,21 +143,30 @@ options(repos = c( getOption('repos'), sub(r'(\d{4}-\d{2}-\d{2})', 'latest', getOption('repos')) )) -remotes::install_version('abind', '>= 1.4.5', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('coda', '>= 0.18', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('geonames', '> 0.998', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('lubridate', '>= 1.7.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('magic', '>= 1.5.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('mockery', '>= 0.4.3', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('MODISTools', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('ncdf4', '>= 1.15', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('plyr', '>= 1.8.4', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('purrr', '>= 0.2.3', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('reshape2', '>= 1.4.2', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('rlang', '>= 0.2.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('rmarkdown', '>= 2.19', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('stringr', '>= 1.1.0', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('testthat', '>= 3.0.4', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('TruncatedNormal', '>= 2.2', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('vdiffr', '>= 1.0.2', dependencies = TRUE, upgrade = FALSE) -remotes::install_version('XML', '>= 3.98-1.4', dependencies = TRUE, upgrade = FALSE) +install_if_version <- function(pkg, version) { + vers <- gsub('[^[:digit:].-]+', '', version) + cmp <- get(gsub('[^<>=]+', '', version)) + ok <- requireNamespace(pkg, quietly = TRUE) && + cmp(packageVersion(pkg), vers) + if (!ok) { + remotes::install_version(pkg, version, dependencies = TRUE, upgrade = FALSE) + } +} +install_if_version('abind', '>= 1.4.5') +install_if_version('coda', '>= 0.18') +install_if_version('geonames', '> 0.998') +install_if_version('lubridate', '>= 1.7.0') +install_if_version('magic', '>= 1.5.0') +install_if_version('mockery', '>= 0.4.3') +install_if_version('MODISTools', '>= 1.1.0') +install_if_version('ncdf4', '>= 1.15') +install_if_version('plyr', '>= 1.8.4') +install_if_version('purrr', '>= 0.2.3') +install_if_version('reshape2', '>= 1.4.2') +install_if_version('rlang', '>= 0.2.0') +install_if_version('rmarkdown', '>= 2.19') +install_if_version('stringr', '>= 1.1.0') +install_if_version('testthat', '>= 3.0.4') +install_if_version('TruncatedNormal', '>= 2.2') +install_if_version('vdiffr', '>= 1.0.2') +install_if_version('XML', '>= 3.98-1.4') diff --git a/scripts/generate_dependencies.R b/scripts/generate_dependencies.R index a6b40a1a10d..06fc56846e6 100755 --- a/scripts/generate_dependencies.R +++ b/scripts/generate_dependencies.R @@ -155,9 +155,7 @@ unversioned_deps <- uniq_deps[uniq_deps$version == "*",]$package versioned_dep_install_calls <- uniq_deps[uniq_deps$version != "*",] |> dplyr::mutate( inst_call = paste0( - "remotes::install_version(", - shQuote(package), ", ", shQuote(version), - ", dependencies = TRUE, upgrade = FALSE)")) |> + "install_if_version(", shQuote(package), ", ", shQuote(version), ")")) |> dplyr::pull(inst_call) |> sort() @@ -193,5 +191,14 @@ cat("#!/usr/bin/env Rscript", " getOption('repos'),", " sub(r'(\\d{4}-\\d{2}-\\d{2})', 'latest', getOption('repos'))", "))", + "install_if_version <- function(pkg, version) {", + " vers <- gsub('[^[:digit:].-]+', '', version)", + " cmp <- get(gsub('[^<>=]+', '', version))", + " ok <- requireNamespace(pkg, quietly = TRUE) &&", + " cmp(packageVersion(pkg), vers)", + " if (!ok) {", + " remotes::install_version(pkg, version, dependencies = TRUE, upgrade = FALSE)", + " }", + "}", paste(versioned_dep_install_calls, collapse = "\n"), file = "docker/depends/pecan.depends.R", sep = "\n", append = FALSE) From 67d6266e7818b49f39f884d7e8f1082b6e0beee9 Mon Sep 17 00:00:00 2001 From: Chris Black Date: Sat, 16 Dec 2023 21:09:53 -0800 Subject: [PATCH 5/5] Clearer function name Seemed too easy to read "install_if_version(foo, >= 1.2.3)" as "if the version of foo is >= 1.2.3, then install [some other version?]" --- docker/depends/pecan.depends.R | 38 ++++++++++++++++----------------- scripts/generate_dependencies.R | 4 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index a933e1a9920..b603a842543 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -143,7 +143,7 @@ options(repos = c( getOption('repos'), sub(r'(\d{4}-\d{2}-\d{2})', 'latest', getOption('repos')) )) -install_if_version <- function(pkg, version) { +ensure_version <- function(pkg, version) { vers <- gsub('[^[:digit:].-]+', '', version) cmp <- get(gsub('[^<>=]+', '', version)) ok <- requireNamespace(pkg, quietly = TRUE) && @@ -152,21 +152,21 @@ install_if_version <- function(pkg, version) { remotes::install_version(pkg, version, dependencies = TRUE, upgrade = FALSE) } } -install_if_version('abind', '>= 1.4.5') -install_if_version('coda', '>= 0.18') -install_if_version('geonames', '> 0.998') -install_if_version('lubridate', '>= 1.7.0') -install_if_version('magic', '>= 1.5.0') -install_if_version('mockery', '>= 0.4.3') -install_if_version('MODISTools', '>= 1.1.0') -install_if_version('ncdf4', '>= 1.15') -install_if_version('plyr', '>= 1.8.4') -install_if_version('purrr', '>= 0.2.3') -install_if_version('reshape2', '>= 1.4.2') -install_if_version('rlang', '>= 0.2.0') -install_if_version('rmarkdown', '>= 2.19') -install_if_version('stringr', '>= 1.1.0') -install_if_version('testthat', '>= 3.0.4') -install_if_version('TruncatedNormal', '>= 2.2') -install_if_version('vdiffr', '>= 1.0.2') -install_if_version('XML', '>= 3.98-1.4') +ensure_version('abind', '>= 1.4.5') +ensure_version('coda', '>= 0.18') +ensure_version('geonames', '> 0.998') +ensure_version('lubridate', '>= 1.7.0') +ensure_version('magic', '>= 1.5.0') +ensure_version('mockery', '>= 0.4.3') +ensure_version('MODISTools', '>= 1.1.0') +ensure_version('ncdf4', '>= 1.15') +ensure_version('plyr', '>= 1.8.4') +ensure_version('purrr', '>= 0.2.3') +ensure_version('reshape2', '>= 1.4.2') +ensure_version('rlang', '>= 0.2.0') +ensure_version('rmarkdown', '>= 2.19') +ensure_version('stringr', '>= 1.1.0') +ensure_version('testthat', '>= 3.0.4') +ensure_version('TruncatedNormal', '>= 2.2') +ensure_version('vdiffr', '>= 1.0.2') +ensure_version('XML', '>= 3.98-1.4') diff --git a/scripts/generate_dependencies.R b/scripts/generate_dependencies.R index 06fc56846e6..8e6fa38f2d8 100755 --- a/scripts/generate_dependencies.R +++ b/scripts/generate_dependencies.R @@ -155,7 +155,7 @@ unversioned_deps <- uniq_deps[uniq_deps$version == "*",]$package versioned_dep_install_calls <- uniq_deps[uniq_deps$version != "*",] |> dplyr::mutate( inst_call = paste0( - "install_if_version(", shQuote(package), ", ", shQuote(version), ")")) |> + "ensure_version(", shQuote(package), ", ", shQuote(version), ")")) |> dplyr::pull(inst_call) |> sort() @@ -191,7 +191,7 @@ cat("#!/usr/bin/env Rscript", " getOption('repos'),", " sub(r'(\\d{4}-\\d{2}-\\d{2})', 'latest', getOption('repos'))", "))", - "install_if_version <- function(pkg, version) {", + "ensure_version <- function(pkg, version) {", " vers <- gsub('[^[:digit:].-]+', '', version)", " cmp <- get(gsub('[^<>=]+', '', version))", " ok <- requireNamespace(pkg, quietly = TRUE) &&",