diff --git a/DESCRIPTION b/DESCRIPTION index be616ee0..10af9589 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: hubValidations Title: Testing framework for hubverse hub validations -Version: 0.5.1 +Version: 0.6.0 Authors@R: c( person( given = "Anna", @@ -44,7 +44,6 @@ Imports: jsonvalidate, lubridate, magrittr, - octolog, purrr, rlang, stringr, @@ -63,13 +62,12 @@ Suggests: Remotes: hubverse-org/hubUtils, hubverse-org/hubData, - hubverse-org/hubAdmin, - assignUser/octolog + hubverse-org/hubAdmin Config/testthat/edition: 3 Config/Needs/website: pkgdown, hubverse-org/hubStyle Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 URL: https://github.com/hubverse-org/hubValidations, https://hubverse-org.github.io/hubValidations/ BugReports: https://github.com/hubverse-org/hubValidations/issues diff --git a/NEWS.md b/NEWS.md index 49cd7f1c..f0ae3a7e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,15 @@ +# hubValidations 0.6.0 + +* To make clearer that all checks resulting in `check_failure` are required to pass for files to be considered valid, `check_failure` class objects are elevated to errors (#111). Also, to make it easier for users to identify errors from visually scanning the printed output, the following custom bullets have been assigned. + - `✖` : `check_failure` class object. This indicates an error that does not impact the validation process. + - `ⓧ` : `check_error` class object. This also indicates early termination of the validation process. + - `☒` : `check_exec_error` class object. This indicates an error in the execution of a check function. +* `hub_validations` class object `combine()` method now ensures that check names are made unique across all `hub_validations` objects being combined. +* Additional improvements to `hub_validations` class object `print()` method. + - Check results for each file validated are now split and printed under file name header. + - The check name that can be used to access the check result from the `hub_validations` object is now included as the prefix to the check result message instead of the file name (#76). +* `octolog` dependency removed. This removes the annotation of validation results onto GitHub Action workflow logs (#113). + # hubValidations 0.5.1 * Remove dependency on development version of `arrow` package and bump required version to 17.0.0. diff --git a/R/capture_check_cnd.R b/R/capture_check_cnd.R index eaef191c..e96f9c31 100644 --- a/R/capture_check_cnd.R +++ b/R/capture_check_cnd.R @@ -11,7 +11,7 @@ #' the state when validation succeeds, the second element, when validation fails. #' @param error logical. In the case of validation failure, whether the function #' should return an object of class `` (`TRUE`) or -#' `` (`FALSE`, default). +#' `` (`FALSE`, default). #' @param details further details to be appended to the output message. #' @inheritParams rlang::error_cnd #' @@ -22,7 +22,7 @@ #' @return Depending on whether validation has succeeded and the value #' of the `error` argument, one of: #' - `` condition class object. -#' - `` condition class object. +#' - `` condition class object. #' - `` condition class object. #' #' Returned object also inherits from subclass ``. @@ -82,7 +82,7 @@ capture_check_cnd <- function(check, file_path, msg_subject, msg_attribute, use_cli_format = TRUE ) } else { - res <- rlang::warning_cnd( + res <- rlang::error_cnd( c("check_failure", "hub_check"), where = file_path, ..., diff --git a/R/check_tbl_col_types.R b/R/check_tbl_col_types.R index 188c5d54..ae341a76 100644 --- a/R/check_tbl_col_types.R +++ b/R/check_tbl_col_types.R @@ -7,7 +7,7 @@ #' @return #' Depending on whether validation has succeeded, one of: #' - `` condition class object. -#' - `` condition class object. +#' - `` condition class object. #' #' Returned object also inherits from subclass ``. #' @export diff --git a/R/check_tbl_unique_round_id.R b/R/check_tbl_unique_round_id.R index 49e1010d..5426fcfc 100644 --- a/R/check_tbl_unique_round_id.R +++ b/R/check_tbl_unique_round_id.R @@ -7,7 +7,7 @@ #' @return #' Depending on whether validation has succeeded, one of: #' - `` condition class object. -#' - `` condition class object. +#' - `` condition class object. #' #' If `round_id_from_variable: false` and no `round_id_col` name is provided, #' check is skipped and a `` condition class object is diff --git a/R/hub_validations_methods.R b/R/hub_validations_methods.R index 17203c0c..a8fb7681 100644 --- a/R/hub_validations_methods.R +++ b/R/hub_validations_methods.R @@ -8,26 +8,39 @@ print.hub_validations <- function(x, ...) { if (length(x) == 0L) { msg <- cli::format_inline("Empty {.cls hub_validations}") + cli::cli_inform(msg) } else { - msg <- stats::setNames( - paste( - fs::path_file(purrr::map_chr(x, "where")), - purrr::map_chr(x, "message"), - sep = ": " - ), - dplyr::case_when( - purrr::map_lgl(x, ~ rlang::inherits_any(.x, "check_success")) ~ "v", - purrr::map_lgl(x, ~ rlang::inherits_any(.x, "check_failure")) ~ "!", - purrr::map_lgl(x, ~ rlang::inherits_any(.x, "check_exec_warn")) ~ "!", - purrr::map_lgl(x, ~ rlang::inherits_any(.x, "check_error")) ~ "x", - purrr::map_lgl(x, ~ rlang::inherits_any(.x, "check_exec_error")) ~ "x", - purrr::map_lgl(x, ~ rlang::inherits_any(.x, "check_info")) ~ "i", - TRUE ~ "*" + print_file <- function(file_name, x) { + x <- x[get_filenames(x) == file_name] + msg <- stats::setNames( + paste( + apply_cli_span_class(names(x), class = "check_name"), + purrr::map_chr(x, "message"), + sep = ": " + ), + dplyr::case_when( + is_check_class(x, "check_success") ~ "v", + is_check_class(x, "check_failure") ~ "x", + is_check_class(x, "check_exec_warn") ~ "!", + is_check_class(x, "check_error") ~ "circle_cross", + is_check_class(x, "check_exec_error") ~ "lower_block_8", + is_check_class(x, "check_info") ~ "i", + TRUE ~ "*" + ) ) + + cli::cli_div(class = "hub_validations", theme = hub_validation_theme) + cli::cli_h2(file_name) + cli::cli_inform(msg) + cli::cli_end() + } + + purrr::walk( + .x = get_filenames(x, unique = TRUE), + .f = function(file_name, x) print_file(file_name, x), + x = x ) } - - octolog::octo_inform(msg) } @@ -47,8 +60,16 @@ combine.hub_validations <- function(...) { purrr::compact() %>% validate_internal_class(class = "hub_validations") - structure(c(...), - class = c("hub_validations", "list") + combined <- c(...) + if (is.null(names(combined))) { + combined_names <- NULL + } else { + combined_names <- make.unique(names(combined), sep = "_") + } + structure( + combined, + class = c("hub_validations", "list"), + names = combined_names ) } @@ -85,3 +106,61 @@ validate_internal_class <- function(x, class = c( print.pr_hub_validations <- function(x, ...) { purrr::map(x, print) } + + +# cli theme for hub_validations objects that add a circle cross to be applied +# to check_error objects +hub_validation_theme <- list( + ".bullets .bullet-circle_cross" = list( + "before" = function(x) { + paste0(cli::col_red(cli::symbol$circle_cross), " ") + }, + "text-exdent" = 2L + ), + ".bullets .bullet-checkbox_on" = list( + "before" = function(x) { + paste0(cli::col_red(cli::symbol$checkbox_on), " ") + }, + "text-exdent" = 2L + ), + "span.check_name" = list( + "before" = "[", + "after" = "]", + color = "grey" + ), + "h2" = list( + fmt = function(x) { + cli::col_br_cyan( + paste0( + cli::symbol$line, cli::symbol$line, + " ", cli::style_underline(x), " ", + cli::symbol$line, cli::symbol$line, + cli::symbol$line, cli::symbol$line + ) + ) + } + ) +) + +apply_cli_span_class <- function(x, class = "check_name") { + paste0("{.", class, " ", x, "}") +} + +is_check_class <- function(x, + class = c( + "check_success", "check_failure", + "check_exec_warn", "check_error", + "check_exec_error", "check_info" + )) { + class <- rlang::arg_match(class) + purrr::map_lgl(x, ~ rlang::inherits_any(.x, class)) +} + +get_filenames <- function(x, unique = FALSE) { + filenames <- fs::path_file(purrr::map_chr(x, "where")) + if (unique) { + unique(filenames) + } else { + filenames + } +} diff --git a/R/validate_pr.R b/R/validate_pr.R index 15b37c41..0c9f1915 100644 --- a/R/validate_pr.R +++ b/R/validate_pr.R @@ -285,7 +285,7 @@ check_pr_modf_del_files <- function(pr_df, file_type = c( purrr::compact() as_hub_validations(out) %>% - purrr::set_names(sprintf("%s_mod_%i", file_type, seq_along(out))) + purrr::set_names(sprintf("%s_mod", file_type)) } diff --git a/man/capture_check_cnd.Rd b/man/capture_check_cnd.Rd index 5b36e85b..ca1601f6 100644 --- a/man/capture_check_cnd.Rd +++ b/man/capture_check_cnd.Rd @@ -32,7 +32,7 @@ the state when validation succeeds, the second element, when validation fails.} \item{error}{logical. In the case of validation failure, whether the function should return an object of class \verb{} (\code{TRUE}) or -\verb{} (\code{FALSE}, default).} +\verb{} (\code{FALSE}, default).} \item{details}{further details to be appended to the output message.} @@ -44,7 +44,7 @@ Depending on whether validation has succeeded and the value of the \code{error} argument, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. \item \verb{} condition class object. } diff --git a/man/check_file_location.Rd b/man/check_file_location.Rd index f877e5ce..edcfa066 100644 --- a/man/check_file_location.Rd +++ b/man/check_file_location.Rd @@ -14,7 +14,7 @@ the hub's model-output directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_metadata_file_ext.Rd b/man/check_metadata_file_ext.Rd index 14a53fbc..953ec191 100644 --- a/man/check_metadata_file_ext.Rd +++ b/man/check_metadata_file_ext.Rd @@ -14,7 +14,7 @@ the hub's model-output directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_metadata_file_location.Rd b/man/check_metadata_file_location.Rd index 4c952f58..57dbff91 100644 --- a/man/check_metadata_file_location.Rd +++ b/man/check_metadata_file_location.Rd @@ -14,7 +14,7 @@ the hub's model-metadata directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_metadata_file_name.Rd b/man/check_metadata_file_name.Rd index 6618f75d..9c6e9718 100644 --- a/man/check_metadata_file_name.Rd +++ b/man/check_metadata_file_name.Rd @@ -25,7 +25,7 @@ files within the \code{hub-config} directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_metadata_matches_schema.Rd b/man/check_metadata_matches_schema.Rd index cb0e76aa..d7c2d56f 100644 --- a/man/check_metadata_matches_schema.Rd +++ b/man/check_metadata_matches_schema.Rd @@ -24,7 +24,7 @@ files within the \code{hub-config} directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_metadata_schema_exists.Rd b/man/check_metadata_schema_exists.Rd index 712b0f8f..b2f578ae 100644 --- a/man/check_metadata_schema_exists.Rd +++ b/man/check_metadata_schema_exists.Rd @@ -21,7 +21,7 @@ files within the \code{hub-config} directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_submission_time.Rd b/man/check_submission_time.Rd index 899b798f..b2a0dcb0 100644 --- a/man/check_submission_time.Rd +++ b/man/check_submission_time.Rd @@ -37,7 +37,7 @@ provided in the hub's config.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_col_types.Rd b/man/check_tbl_col_types.Rd index 56f9c4b3..83871514 100644 --- a/man/check_tbl_col_types.Rd +++ b/man/check_tbl_col_types.Rd @@ -46,7 +46,7 @@ behaviour so use with care.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_match_round_id.Rd b/man/check_tbl_match_round_id.Rd index 9fbff1a3..c29eb7a1 100644 --- a/man/check_tbl_match_round_id.Rd +++ b/man/check_tbl_match_round_id.Rd @@ -30,7 +30,7 @@ config file.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } If \code{round_id_from_variable: false} and no \code{round_id_col} name is provided, diff --git a/man/check_tbl_rows_unique.Rd b/man/check_tbl_rows_unique.Rd index 6d4a5759..7a45ea13 100644 --- a/man/check_tbl_rows_unique.Rd +++ b/man/check_tbl_rows_unique.Rd @@ -26,7 +26,7 @@ files within the \code{hub-config} directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_spl_n.Rd b/man/check_tbl_spl_n.Rd index 8c714607..f05df8b5 100644 --- a/man/check_tbl_spl_n.Rd +++ b/man/check_tbl_spl_n.Rd @@ -44,7 +44,7 @@ contain \code{NA}s.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_unique_round_id.Rd b/man/check_tbl_unique_round_id.Rd index 07b635bb..a1b26f72 100644 --- a/man/check_tbl_unique_round_id.Rd +++ b/man/check_tbl_unique_round_id.Rd @@ -30,7 +30,7 @@ config file.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } If \code{round_id_from_variable: false} and no \code{round_id_col} name is provided, diff --git a/man/check_tbl_value_col.Rd b/man/check_tbl_value_col.Rd index b443a871..13810f9d 100644 --- a/man/check_tbl_value_col.Rd +++ b/man/check_tbl_value_col.Rd @@ -38,7 +38,7 @@ contain \code{NA}s.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_value_col_ascending.Rd b/man/check_tbl_value_col_ascending.Rd index bf6f62df..4c5a7fa3 100644 --- a/man/check_tbl_value_col_ascending.Rd +++ b/man/check_tbl_value_col_ascending.Rd @@ -17,7 +17,7 @@ the hub's model-output directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_value_col_sum1.Rd b/man/check_tbl_value_col_sum1.Rd index 9e10d9b7..6347476f 100644 --- a/man/check_tbl_value_col_sum1.Rd +++ b/man/check_tbl_value_col_sum1.Rd @@ -16,7 +16,7 @@ the hub's model-output directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/check_tbl_values_required.Rd b/man/check_tbl_values_required.Rd index 83cd5aab..6304e18c 100644 --- a/man/check_tbl_values_required.Rd +++ b/man/check_tbl_values_required.Rd @@ -39,7 +39,7 @@ contain \code{NA}s.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/opt_check_metadata_team_max_model_n.Rd b/man/opt_check_metadata_team_max_model_n.Rd index 75b32e28..7e851b9e 100644 --- a/man/opt_check_metadata_team_max_model_n.Rd +++ b/man/opt_check_metadata_team_max_model_n.Rd @@ -27,7 +27,7 @@ files within the \code{hub-config} directory.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/opt_check_tbl_col_timediff.Rd b/man/opt_check_tbl_col_timediff.Rd index db960984..207c07dc 100644 --- a/man/opt_check_tbl_col_timediff.Rd +++ b/man/opt_check_tbl_col_timediff.Rd @@ -55,7 +55,7 @@ behaviour so use with care.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/opt_check_tbl_counts_lt_popn.Rd b/man/opt_check_tbl_counts_lt_popn.Rd index a381c681..1d46720d 100644 --- a/man/opt_check_tbl_counts_lt_popn.Rd +++ b/man/opt_check_tbl_counts_lt_popn.Rd @@ -48,7 +48,7 @@ Must be shared by both files.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/man/opt_check_tbl_horizon_timediff.Rd b/man/opt_check_tbl_horizon_timediff.Rd index 83c0a47e..c468da0c 100644 --- a/man/opt_check_tbl_horizon_timediff.Rd +++ b/man/opt_check_tbl_horizon_timediff.Rd @@ -61,7 +61,7 @@ behaviour so use with care.} Depending on whether validation has succeeded, one of: \itemize{ \item \verb{} condition class object. -\item \verb{} condition class object. +\item \verb{} condition class object. } Returned object also inherits from subclass \verb{}. diff --git a/tests/testthat/_snaps/capture_check_cnd.md b/tests/testthat/_snaps/capture_check_cnd.md index 4c7b10dc..2bc41a11 100644 --- a/tests/testthat/_snaps/capture_check_cnd.md +++ b/tests/testthat/_snaps/capture_check_cnd.md @@ -14,9 +14,9 @@ capture_check_cnd(check = FALSE, file_path = "test/file.csv", msg_subject = "{.var round_id}", msg_attribute = "valid.", error = FALSE, details = "Must be one of {.val {c('A', 'B')}}, not {.val C}") Output - - Warning: - `round_id` must be valid. Must be one of "A" and "B", not "C" + + Error: + ! `round_id` must be valid. Must be one of "A" and "B", not "C" --- @@ -46,9 +46,9 @@ msg_attribute = "consistent with expected round task IDs and std column names.", msg_verbs = c("are", "must always be")) Output - - Warning: - Column names must always be consistent with expected round task IDs and std column names. + + Error: + ! Column names must always be consistent with expected round task IDs and std column names. --- @@ -57,12 +57,14 @@ msg_attribute = "consistent with expected round task IDs and std column names.", msg_verbs = c("are", "must always be"))) Output - List of 4 + List of 6 $ message : chr "Column names must always be consistent with expected round task IDs and std column names. \n " + $ trace : NULL + $ parent : NULL $ where : chr "test/file.csv" $ call : chr "eval" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... # capture_check_cnd fails correctly diff --git a/tests/testthat/_snaps/check_file_location.md b/tests/testthat/_snaps/check_file_location.md index 4a8a5a9d..f8b9437b 100644 --- a/tests/testthat/_snaps/check_file_location.md +++ b/tests/testthat/_snaps/check_file_location.md @@ -12,7 +12,7 @@ Code check_file_location("team1-goodmodel/2022-10-08-team2-goodmodel.csv") Output - - Warning: - File directory name must match `model_id` metadata in file name. File should be submitted to directory "team2-goodmodel" not "team1-goodmodel" + + Error: + ! File directory name must match `model_id` metadata in file name. File should be submitted to directory "team2-goodmodel" not "team1-goodmodel" diff --git a/tests/testthat/_snaps/check_metadata_file_location.md b/tests/testthat/_snaps/check_metadata_file_location.md index c949b077..5ac68ff8 100644 --- a/tests/testthat/_snaps/check_metadata_file_location.md +++ b/tests/testthat/_snaps/check_metadata_file_location.md @@ -12,7 +12,7 @@ Code check_metadata_file_location("random_folder/hub-baseline.yml") Output - - Warning: - Metadata file directory name must match "model-metadata". Metadata files should be submitted to directory "model-metadata", not "model-metadata/random_folder". + + Error: + ! Metadata file directory name must match "model-metadata". Metadata files should be submitted to directory "model-metadata", not "model-metadata/random_folder". diff --git a/tests/testthat/_snaps/check_tbl_col_types.md b/tests/testthat/_snaps/check_tbl_col_types.md index dab1fa71..ed00e876 100644 --- a/tests/testthat/_snaps/check_tbl_col_types.md +++ b/tests/testthat/_snaps/check_tbl_col_types.md @@ -12,9 +12,9 @@ Code check_tbl_col_types(tbl, file_path, hub_path) Output - - Warning: - Column data types do not match hub schema. `origin_date` should be "character" not "Date", `horizon` should be "double" not "integer". + + Error: + ! Column data types do not match hub schema. `origin_date` should be "character" not "Date", `horizon` should be "double" not "integer". # Check '06' location value validated correctly in check_tbl_col_types diff --git a/tests/testthat/_snaps/check_tbl_match_round_id.md b/tests/testthat/_snaps/check_tbl_match_round_id.md index 5de6a994..cfbbbe27 100644 --- a/tests/testthat/_snaps/check_tbl_match_round_id.md +++ b/tests/testthat/_snaps/check_tbl_match_round_id.md @@ -45,9 +45,9 @@ check_tbl_match_round_id(tbl = tbl, file_path = file_path, hub_path = hub_path, round_id_col = "random_column") Output - - Warning: - `round_id_col` name must be valid. Must be one of "origin_date", "target", "horizon", and "location" not "random_column". + + Error: + ! `round_id_col` name must be valid. Must be one of "origin_date", "target", "horizon", and "location" not "random_column". --- @@ -55,10 +55,12 @@ str(check_tbl_match_round_id(tbl = tbl, file_path = file_path, hub_path = hub_path, round_id_col = "random_column")) Output - List of 4 + List of 6 $ message : chr "`round_id_col` name must be valid. \n Must be one of\n \"origin_date\", \""| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "hub-baseline/2022-10-01-hub-baseline.csv" $ call : chr "check_tbl_match_round_id" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... diff --git a/tests/testthat/_snaps/check_tbl_rows_unique.md b/tests/testthat/_snaps/check_tbl_rows_unique.md index 762c1666..a114f80c 100644 --- a/tests/testthat/_snaps/check_tbl_rows_unique.md +++ b/tests/testthat/_snaps/check_tbl_rows_unique.md @@ -12,7 +12,7 @@ Code check_tbl_rows_unique(rbind(tbl, tbl[c(5, 9), ]), file_path, hub_path) Output - - Warning: - All combinations of task ID column/`output_type`/`output_type_id` values must be unique. Rows containing duplicate combinations: 48 and 49 + + Error: + ! All combinations of task ID column/`output_type`/`output_type_id` values must be unique. Rows containing duplicate combinations: 48 and 49 diff --git a/tests/testthat/_snaps/check_tbl_spl_n.md b/tests/testthat/_snaps/check_tbl_spl_n.md index 7d646f3c..6debd3a6 100644 --- a/tests/testthat/_snaps/check_tbl_spl_n.md +++ b/tests/testthat/_snaps/check_tbl_spl_n.md @@ -12,9 +12,9 @@ Code check_tbl_spl_n(tbl_const_error, round_id, file_path, hub_path) Output - - Warning: - Number of samples per compound idx not consistent. Sample numbers supplied per compound idx vary between 100 and 99. See `errors` attribute for details. + + Error: + ! Number of samples per compound idx not consistent. Sample numbers supplied per compound idx vary between 100 and 99. See `errors` attribute for details. --- @@ -35,9 +35,9 @@ Code check_tbl_spl_n(tbl_min_error, round_id, file_path, hub_path) Output - - Warning: - Required samples per compound idx task not present. File contains less than the minimum required number of samples per task for compound idxs "1", "2", "3", "4", and "5". See `errors` attribute for details. + + Error: + ! Required samples per compound idx task not present. File contains less than the minimum required number of samples per task for compound idxs "1", "2", "3", "4", and "5". See `errors` attribute for details. --- @@ -160,8 +160,10 @@ Code str(check_tbl_spl_n(tbl_coarse, round_id, file_path, hub_path)) Output - List of 5 + List of 7 $ message : chr "Required samples per compound idx task not present. \n File contains less than the minimum required number of "| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "flu-base/2022-10-22-flu-base.csv" $ errors :List of 4 ..$ 1 :List of 5 @@ -214,16 +216,16 @@ .. .. ..$ target_end_date: chr [1:20] "2022-11-12" "2022-11-12" "2022-11-12" "2022-11-12" ... $ call : chr "check_tbl_spl_n" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code check_tbl_spl_n(tbl_coarse, round_id, file_path, hub_path, compound_taskid_set = compound_taskid_set) Output - - Warning: - Required samples per compound idx task not present. File contains less than the minimum required number of samples per task for compound idxs "1", "2", "3", and "4". See `errors` attribute for details. + + Error: + ! Required samples per compound idx task not present. File contains less than the minimum required number of samples per task for compound idxs "1", "2", "3", and "4". See `errors` attribute for details. --- @@ -240,9 +242,9 @@ check_tbl_spl_n(tbl_minus_1, round_id, file_path, hub_path, compound_taskid_set = compound_taskid_set) Output - - Warning: - Number of samples per compound idx not consistent. Sample numbers supplied per compound idx vary between 9 and 10. See `errors` attribute for details. + + Error: + ! Number of samples per compound idx not consistent. Sample numbers supplied per compound idx vary between 9 and 10. See `errors` attribute for details. # Ignoring derived_task_ids in check_tbl_spl_n works diff --git a/tests/testthat/_snaps/check_tbl_unique_round_id.md b/tests/testthat/_snaps/check_tbl_unique_round_id.md index b351d3c2..e43e9316 100644 --- a/tests/testthat/_snaps/check_tbl_unique_round_id.md +++ b/tests/testthat/_snaps/check_tbl_unique_round_id.md @@ -45,9 +45,9 @@ check_tbl_unique_round_id(tbl = multiple_rids, file_path = file_path, hub_path = hub_path, round_id_col = "random_column") Output - - Warning: - `round_id_col` name must be valid. Must be one of "origin_date", "target", "horizon", "location", and "age_group" not "random_column". + + Error: + ! `round_id_col` name must be valid. Must be one of "origin_date", "target", "horizon", "location", and "age_group" not "random_column". --- @@ -55,10 +55,12 @@ str(check_tbl_unique_round_id(tbl = multiple_rids, file_path = file_path, hub_path = hub_path, round_id_col = "random_column")) Output - List of 4 + List of 6 $ message : chr "`round_id_col` name must be valid. \n Must be one of\n \"origin_date\", \""| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" $ call : chr "check_tbl_unique_round_id" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... diff --git a/tests/testthat/_snaps/check_tbl_value_col.md b/tests/testthat/_snaps/check_tbl_value_col.md index 0aa81a7b..b57aa823 100644 --- a/tests/testthat/_snaps/check_tbl_value_col.md +++ b/tests/testthat/_snaps/check_tbl_value_col.md @@ -12,27 +12,27 @@ Code check_tbl_value_col(tbl, round_id, file_path, hub_path) Output - - Warning: - Values in column `value` are not all valid with respect to modeling task config. Values 196.83, 367.89, 244.85, 427.975, 310.9, 499.9, 394.8, 461.85, 629.7, 534.6, 599.75, 727.5, 669.7, 725.562456357284, 801.689162048122, 800.239938093011, 915.5, 949.59, ..., 2775.56600111008, and 2139.07 cannot be coerced to expected data type "integer" for output type "quantile". + + Error: + ! Values in column `value` are not all valid with respect to modeling task config. Values 196.83, 367.89, 244.85, 427.975, 310.9, 499.9, 394.8, 461.85, 629.7, 534.6, 599.75, 727.5, 669.7, 725.562456357284, 801.689162048122, 800.239938093011, 915.5, 949.59, ..., 2775.56600111008, and 2139.07 cannot be coerced to expected data type "integer" for output type "quantile". # check_tbl_value_col errors correctly Code check_tbl_value_col(tbl, round_id, file_path, hub_path) Output - - Warning: - Values in column `value` are not all valid with respect to modeling task config. Value "fail" cannot be coerced to expected data type "integer" for output type "quantile".Values in column `value` are not all valid with respect to modeling task config. Value -6 is smaller than allowed minimum value 0 for output type "quantile". + + Error: + ! Values in column `value` are not all valid with respect to modeling task config. Value "fail" cannot be coerced to expected data type "integer" for output type "quantile".Values in column `value` are not all valid with respect to modeling task config. Value -6 is smaller than allowed minimum value 0 for output type "quantile". --- Code check_tbl_value_col(tbl, round_id, file_path, hub_path) Output - - Warning: - Values in column `value` are not all valid with respect to modeling task config. Value "fail" cannot be coerced to expected data type "integer" for output type "quantile".Values in column `value` are not all valid with respect to modeling task config. Values "196.83", "244.85", "310.9", "499.9", "394.8", "461.85", "629.7", "534.6", "599.75", "727.5", "669.7", "725.562456357284", "801.689162048122", "800.239938093011", "915.5", "949.59", "938.67", "962.03", ..., "2775.56600111008", and "2139.07" cannot be coerced to expected data type "integer" for output type "quantile".Values in column `value` are not all valid with respect to modeling task config. Value -6 is smaller than allowed minimum value 0 for output type "quantile". + + Error: + ! Values in column `value` are not all valid with respect to modeling task config. Value "fail" cannot be coerced to expected data type "integer" for output type "quantile".Values in column `value` are not all valid with respect to modeling task config. Values "196.83", "244.85", "310.9", "499.9", "394.8", "461.85", "629.7", "534.6", "599.75", "727.5", "669.7", "725.562456357284", "801.689162048122", "800.239938093011", "915.5", "949.59", "938.67", "962.03", ..., "2775.56600111008", and "2139.07" cannot be coerced to expected data type "integer" for output type "quantile".Values in column `value` are not all valid with respect to modeling task config. Value -6 is smaller than allowed minimum value 0 for output type "quantile". # Ignoring derived_task_ids in check_tbl_value_col works diff --git a/tests/testthat/_snaps/check_tbl_value_col_ascending.md b/tests/testthat/_snaps/check_tbl_value_col_ascending.md index 0cd1ee09..6b03e724 100644 --- a/tests/testthat/_snaps/check_tbl_value_col_ascending.md +++ b/tests/testthat/_snaps/check_tbl_value_col_ascending.md @@ -30,8 +30,10 @@ Code str(check_tbl_value_col_ascending(tbl, file_path)) Output - List of 5 + List of 7 $ message : chr "Values in `value` column are not non-decreasing as output_type_ids increase for all unique task ID\n value/o"| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" $ error_tbl : tibble [1 x 5] (S3: tbl_df/tbl/data.frame) ..$ origin_date: Date[1:1], format: "2022-10-08" @@ -41,15 +43,17 @@ ..$ output_type: chr "quantile" $ call : chr "check_tbl_value_col_ascending" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(check_tbl_value_col_ascending(tbl_error, file_path)) Output - List of 5 + List of 7 $ message : chr "Values in `value` column are not non-decreasing as output_type_ids increase for all unique task ID\n value/o"| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" $ error_tbl : tibble [1 x 5] (S3: tbl_df/tbl/data.frame) ..$ forecast_date: Date[1:1], format: "2023-05-08" @@ -59,15 +63,17 @@ ..$ output_type : chr "quantile" $ call : chr "check_tbl_value_col_ascending" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(check_tbl_value_col_ascending(rbind(tbl, tbl_error), file_path)) Output - List of 5 + List of 7 $ message : chr "Values in `value` column are not non-decreasing as output_type_ids increase for all unique task ID\n value/o"| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" $ error_tbl : tibble [1 x 5] (S3: tbl_df/tbl/data.frame) ..$ forecast_date: Date[1:1], format: "2023-05-08" @@ -77,7 +83,7 @@ ..$ output_type : chr "quantile" $ call : chr "check_tbl_value_col_ascending" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... # check_tbl_value_col_ascending skips correctly diff --git a/tests/testthat/_snaps/check_tbl_value_col_sum1.md b/tests/testthat/_snaps/check_tbl_value_col_sum1.md index 9bb11458..0982dfc1 100644 --- a/tests/testthat/_snaps/check_tbl_value_col_sum1.md +++ b/tests/testthat/_snaps/check_tbl_value_col_sum1.md @@ -12,8 +12,10 @@ Code str(check_tbl_value_col_sum1(tbl, file_path)) Output - List of 5 + List of 7 $ message : chr "Values in `value` column do not sum to 1 for all unique task ID value combination of pmf\n output types. \n "| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "umass_ens/2023-05-08-umass_ens.csv" $ error_tbl : tibble [1 x 5] (S3: tbl_df/tbl/data.frame) ..$ forecast_date: Date[1:1], format: "2023-05-08" @@ -23,15 +25,17 @@ ..$ output_type : chr "pmf" $ call : chr "check_tbl_value_col_sum1" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(check_tbl_value_col_sum1(tbl, file_path)) Output - List of 5 + List of 7 $ message : chr "Values in `value` column do not sum to 1 for all unique task ID value combination of pmf\n output types. \n "| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "umass_ens/2023-05-08-umass_ens.csv" $ error_tbl : tibble [1 x 5] (S3: tbl_df/tbl/data.frame) ..$ forecast_date: Date[1:1], format: "2023-05-08" @@ -41,7 +45,7 @@ ..$ output_type : chr "pmf" $ call : chr "check_tbl_value_col_sum1" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- diff --git a/tests/testthat/_snaps/check_tbl_values_required.md b/tests/testthat/_snaps/check_tbl_values_required.md index a52aeb84..0fb1a59e 100644 --- a/tests/testthat/_snaps/check_tbl_values_required.md +++ b/tests/testthat/_snaps/check_tbl_values_required.md @@ -12,8 +12,10 @@ Code str(missing_req_block) Output - List of 5 + List of 7 $ message : chr "Required task ID/output type/output type ID combinations missing. \n See `missing` attribute for details." + $ trace : NULL + $ parent : NULL $ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" $ missing : tibble [23 x 6] (S3: tbl_df/tbl/data.frame) ..$ origin_date : Date[1:23], format: "2022-10-08" "2022-10-08" ... @@ -24,15 +26,17 @@ ..$ output_type_id: num [1:23] 0.01 0.025 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.4 ... $ call : chr "check_tbl_values_required" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(res_missing_otid) Output - List of 5 + List of 7 $ message : chr "Required task ID/output type/output type ID combinations missing. \n See `missing` attribute for details." + $ trace : NULL + $ parent : NULL $ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" $ missing : tibble [3 x 6] (S3: tbl_df/tbl/data.frame) ..$ origin_date : Date[1:3], format: "2022-10-08" "2022-10-08" ... @@ -43,7 +47,7 @@ ..$ output_type_id: num [1:3] 0.01 0.025 0.05 $ call : chr "check_tbl_values_required" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... # check_tbl_values_required works with 2 separate model tasks & completely missing cols @@ -59,8 +63,10 @@ Code str(missing_required) Output - List of 5 + List of 7 $ message : chr "Required task ID/output type/output type ID combinations missing. \n See `missing` attribute for details." + $ trace : NULL + $ parent : NULL $ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" $ missing : tibble [2 x 6] (S3: tbl_df/tbl/data.frame) ..$ forecast_date : Date[1:2], format: "2023-05-08" "2023-05-08" @@ -71,15 +77,17 @@ ..$ output_type_id: chr [1:2] "0.01" "0.025" $ call : chr "check_tbl_values_required" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(missing_opt_otid) Output - List of 5 + List of 7 $ message : chr "Required task ID/output type/output type ID combinations missing. \n See `missing` attribute for details." + $ trace : NULL + $ parent : NULL $ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" $ missing : tibble [2 x 6] (S3: tbl_df/tbl/data.frame) ..$ forecast_date : Date[1:2], format: "2023-05-08" "2023-05-08" @@ -90,15 +98,17 @@ ..$ output_type_id: chr [1:2] "0.01" "0.025" $ call : chr "check_tbl_values_required" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(missing_pmf) Output - List of 5 + List of 7 $ message : chr "Required task ID/output type/output type ID combinations missing. \n See `missing` attribute for details." + $ trace : NULL + $ parent : NULL $ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" $ missing : tibble [4 x 6] (S3: tbl_df/tbl/data.frame) ..$ forecast_date : Date[1:4], format: "2023-05-08" "2023-05-08" ... @@ -109,15 +119,17 @@ ..$ output_type_id: chr [1:4] "decrease" "stable" "increase" "large_increase" $ call : chr "check_tbl_values_required" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... --- Code str(missing_horizon) Output - List of 5 + List of 7 $ message : chr "Required task ID/output type/output type ID combinations missing. \n See `missing` attribute for details." + $ trace : NULL + $ parent : NULL $ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" $ missing : tibble [9 x 6] (S3: tbl_df/tbl/data.frame) ..$ forecast_date : Date[1:9], format: "2023-05-08" "2023-05-08" ... @@ -128,7 +140,7 @@ ..$ output_type_id: chr [1:9] "decrease" "stable" "increase" "large_increase" ... $ call : chr "check_tbl_values_required" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... # check_tbl_values_required works with v3 spec samples @@ -146,9 +158,9 @@ check_tbl_values_required(tbl = tbl, round_id = round_id, file_path = file_path, hub_path = hub_path) Output - - Warning: - Required task ID/output type/output type ID combinations missing. See `missing` attribute for details. + + Error: + ! Required task ID/output type/output type ID combinations missing. See `missing` attribute for details. --- diff --git a/tests/testthat/_snaps/check_valid_round_id_col.md b/tests/testthat/_snaps/check_valid_round_id_col.md index c30d5208..3e23f79c 100644 --- a/tests/testthat/_snaps/check_valid_round_id_col.md +++ b/tests/testthat/_snaps/check_valid_round_id_col.md @@ -48,7 +48,7 @@ check_valid_round_id_col(tbl = tbl, file_path = file_path, hub_path = hub_path, round_id_col = "random_column") Output - - Warning: - `round_id_col` name must be valid. Must be one of "origin_date", "target", "horizon", "location", and "age_group" not "random_column". + + Error: + ! `round_id_col` name must be valid. Must be one of "origin_date", "target", "horizon", "location", and "age_group" not "random_column". diff --git a/tests/testthat/_snaps/execute_custom_checks.md b/tests/testthat/_snaps/execute_custom_checks.md index d360a75c..affef472 100644 --- a/tests/testthat/_snaps/execute_custom_checks.md +++ b/tests/testthat/_snaps/execute_custom_checks.md @@ -19,12 +19,13 @@ str(test_custom_checks_caller(validations_cfg_path = testthat::test_path( "testdata", "config", "validations-error.yml"))) Output - List of 1 - $ horizon_timediff:List of 4 + Classes 'hub_validations', 'list' hidden list of 1 + $ horizon_timediff:List of 6 ..$ message : chr "Time differences between t0 var `forecast_date` and t1 var\n `target_end_date` do not all match expected"| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" ..$ call : NULL ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... - - attr(*, "class")= chr [1:2] "hub_validations" "list" + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... diff --git a/tests/testthat/_snaps/opt_check_metadata_team_max_model_n.md b/tests/testthat/_snaps/opt_check_metadata_team_max_model_n.md index 9406fa43..d82f9c66 100644 --- a/tests/testthat/_snaps/opt_check_metadata_team_max_model_n.md +++ b/tests/testthat/_snaps/opt_check_metadata_team_max_model_n.md @@ -13,7 +13,7 @@ opt_check_metadata_team_max_model_n(hub_path = hub_path, file_path = "hub-baseline.yml", n_max = 1L) Output - - Warning: - Maximum number of models per team (1) exceeded. Team "hub" has submitted valid metadata for 2 models: "baseline" and "ensemble". + + Error: + ! Maximum number of models per team (1) exceeded. Team "hub" has submitted valid metadata for 2 models: "baseline" and "ensemble". diff --git a/tests/testthat/_snaps/opt_check_tbl_col_timediff.md b/tests/testthat/_snaps/opt_check_tbl_col_timediff.md index 774d4d3a..6d633751 100644 --- a/tests/testthat/_snaps/opt_check_tbl_col_timediff.md +++ b/tests/testthat/_snaps/opt_check_tbl_col_timediff.md @@ -24,9 +24,9 @@ opt_check_tbl_col_timediff(tbl, file_path, hub_path, t0_colname = "forecast_date", t1_colname = "target_end_date", timediff = lubridate::weeks(2)) Output - - Warning: - Time differences between t0 var `forecast_date` and t1 var `target_end_date` do not all match expected period of 14d 0H 0M 0S. t1 var value 2023-05-15 invalid. + + Error: + ! Time differences between t0 var `forecast_date` and t1 var `target_end_date` do not all match expected period of 14d 0H 0M 0S. t1 var value 2023-05-15 invalid. # opt_check_tbl_col_timediff fails correctly diff --git a/tests/testthat/_snaps/opt_check_tbl_counts_lt_popn.md b/tests/testthat/_snaps/opt_check_tbl_counts_lt_popn.md index b48f6cbb..4961ae9e 100644 --- a/tests/testthat/_snaps/opt_check_tbl_counts_lt_popn.md +++ b/tests/testthat/_snaps/opt_check_tbl_counts_lt_popn.md @@ -12,9 +12,9 @@ Code opt_check_tbl_counts_lt_popn(tbl, file_path, hub_path, targets = targets) Output - - Warning: - Target counts must be less than location population size. Affected rows: 1 and 2. + + Error: + ! Target counts must be less than location population size. Affected rows: 1 and 2. # opt_check_tbl_counts_lt_popn fails correctly diff --git a/tests/testthat/_snaps/opt_check_tbl_horizon_timediff.md b/tests/testthat/_snaps/opt_check_tbl_horizon_timediff.md index d054869e..26b082b6 100644 --- a/tests/testthat/_snaps/opt_check_tbl_horizon_timediff.md +++ b/tests/testthat/_snaps/opt_check_tbl_horizon_timediff.md @@ -24,9 +24,9 @@ opt_check_tbl_horizon_timediff(tbl, file_path, hub_path, t0_colname = "forecast_date", t1_colname = "target_end_date") Output - - Warning: - Time differences between t0 var `forecast_date` and t1 var `target_end_date` do not all match expected period of 7d 0H 0M 0S * `horizon`. t1 var value "2023-05-22 (horizon = 1)" are invalid. + + Error: + ! Time differences between t0 var `forecast_date` and t1 var `target_end_date` do not all match expected period of 7d 0H 0M 0S * `horizon`. t1 var value "2023-05-22 (horizon = 1)" are invalid. --- @@ -34,9 +34,9 @@ opt_check_tbl_horizon_timediff(tbl, file_path, hub_path, t0_colname = "forecast_date", t1_colname = "target_end_date", timediff = lubridate::weeks(2)) Output - - Warning: - Time differences between t0 var `forecast_date` and t1 var `target_end_date` do not all match expected period of 14d 0H 0M 0S * `horizon`. t1 var values "2023-05-15 (horizon = 1)" and "2023-05-22 (horizon = 2)" are invalid. + + Error: + ! Time differences between t0 var `forecast_date` and t1 var `target_end_date` do not all match expected period of 14d 0H 0M 0S * `horizon`. t1 var values "2023-05-15 (horizon = 1)" and "2023-05-22 (horizon = 2)" are invalid. # opt_check_tbl_horizon_timediff fails correctly diff --git a/tests/testthat/_snaps/validate_model_data.md b/tests/testthat/_snaps/validate_model_data.md index 891a011d..fb695db4 100644 --- a/tests/testthat/_snaps/validate_model_data.md +++ b/tests/testthat/_snaps/validate_model_data.md @@ -107,26 +107,29 @@ Code str(validate_model_data(hub_path, file_path, round_id_col = "random_col")) Output - List of 3 + Classes 'hub_validations', 'list' hidden list of 3 $ file_read :List of 4 ..$ message : chr "File could be read successfully. \n " ..$ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" ..$ call : chr "check_file_read" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ valid_round_id_col:List of 4 + $ valid_round_id_col:List of 6 ..$ message : chr "`round_id_col` name must be valid. \n Must be one of\n \"origin_date\", \""| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id_col" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... - $ unique_round_id :List of 4 + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... + $ unique_round_id :List of 6 ..$ message : chr "`round_id_col` name must be valid. \n Must be one of\n \"origin_date\", \""| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" ..$ call : chr "check_tbl_unique_round_id" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_warning" "warning" ... - - attr(*, "class")= chr [1:2] "hub_validations" "list" + ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... --- @@ -164,12 +167,14 @@ ..$ call : chr "check_tbl_colnames" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ col_types :List of 4 + $ col_types :List of 6 ..$ message : chr "Column data types do not match hub schema. \n `output_type_id` should be \"character\" not \"double\"." + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" ..$ call : chr "check_tbl_col_types" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... $ valid_vals :List of 5 ..$ message : chr "`tbl` contains valid values/value combinations. \n " ..$ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" @@ -196,12 +201,14 @@ ..$ call : chr "check_tbl_values_required" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ value_col_valid :List of 4 + $ value_col_valid :List of 6 ..$ message : chr "Values in column `value` are not all valid with respect to modeling task config. \n Values 196.83, 367.89, 244"| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" ..$ call : chr "check_tbl_value_col" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... $ value_col_non_desc:List of 5 ..$ message : chr "Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID\n value/outpu"| __truncated__ ..$ where : chr "hub-ensemble/2023-05-08-hub-ensemble.parquet" @@ -242,100 +249,84 @@ Code validate_model_data(hub_path, file_path) Message - v 2022-10-08-team1-goodmodel.csv: File could be read successfully. - v 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid. - v 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value. - v 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name. - v 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names. - v 2022-10-08-team1-goodmodel.csv: Column data types match hub schema. - v 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations. - v 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique. - v 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present. - v 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config. - v 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. - i 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. - ---- - - Code - validate_model_data(hub_path, file_path) - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::v 2022-10-08-team1-goodmodel.csv: File could be read successfully.%0Av 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid.%0Av 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value.%0Av 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name.%0Av 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names.%0Av 2022-10-08-team1-goodmodel.csv: Column data types match hub schema.%0Av 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations.%0Av 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique.%0Av 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present.%0Av 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config.%0Av 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types.%0Ai 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. + + -- 2022-10-08-team1-goodmodel.csv ---- + + v [file_read]: File could be read successfully. + v [valid_round_id_col]: `round_id_col` name is valid. + v [unique_round_id]: `round_id` column "origin_date" contains a single, unique round ID value. + v [match_round_id]: All `round_id_col` "origin_date" values match submission `round_id` from file name. + v [colnames]: Column names are consistent with expected round task IDs and std column names. + v [col_types]: Column data types match hub schema. + v [valid_vals]: `tbl` contains valid values/value combinations. + v [rows_unique]: All combinations of task ID column/`output_type`/`output_type_id` values are unique. + v [req_vals]: Required task ID/output type/output type ID combinations all present. + v [value_col_valid]: Values in column `value` all valid with respect to modeling task config. + v [value_col_non_desc]: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. + i [value_col_sum1]: No pmf output types to check for sum of 1. Check skipped. # validate_model_data print method work [ansi] Code validate_model_data(hub_path, file_path) Message - v 2022-10-08-team1-goodmodel.csv: File could be read successfully. - v 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid. - v 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value. - v 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name. - v 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names. - v 2022-10-08-team1-goodmodel.csv: Column data types match hub schema. - v 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations. - v 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique. - v 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present. - v 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config. - v 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. - i 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. - ---- - - Code - validate_model_data(hub_path, file_path) - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::v 2022-10-08-team1-goodmodel.csv: File could be read successfully.%0Av 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid.%0Av 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value.%0Av 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name.%0Av 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names.%0Av 2022-10-08-team1-goodmodel.csv: Column data types match hub schema.%0Av 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations.%0Av 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique.%0Av 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present.%0Av 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config.%0Av 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types.%0Ai 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. + + -- 2022-10-08-team1-goodmodel.csv ---- + + v [file_read]: File could be read successfully. + v [valid_round_id_col]: `round_id_col` name is valid. + v [unique_round_id]: `round_id` column "origin_date" contains a single, unique round ID value. + v [match_round_id]: All `round_id_col` "origin_date" values match submission `round_id` from file name. + v [colnames]: Column names are consistent with expected round task IDs and std column names. + v [col_types]: Column data types match hub schema. + v [valid_vals]: `tbl` contains valid values/value combinations. + v [rows_unique]: All combinations of task ID column/`output_type`/`output_type_id` values are unique. + v [req_vals]: Required task ID/output type/output type ID combinations all present. + v [value_col_valid]: Values in column `value` all valid with respect to modeling task config. + v [value_col_non_desc]: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. + i [value_col_sum1]: No pmf output types to check for sum of 1. Check skipped. # validate_model_data print method work [unicode] Code validate_model_data(hub_path, file_path) Message - ✔ 2022-10-08-team1-goodmodel.csv: File could be read successfully. - ✔ 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid. - ✔ 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value. - ✔ 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name. - ✔ 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names. - ✔ 2022-10-08-team1-goodmodel.csv: Column data types match hub schema. - ✔ 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations. - ✔ 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique. - ✔ 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present. - ✔ 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config. - ✔ 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. - ℹ 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. - ---- - - Code - validate_model_data(hub_path, file_path) - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✔ 2022-10-08-team1-goodmodel.csv: File could be read successfully.%0A✔ 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid.%0A✔ 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value.%0A✔ 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name.%0A✔ 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names.%0A✔ 2022-10-08-team1-goodmodel.csv: Column data types match hub schema.%0A✔ 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations.%0A✔ 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique.%0A✔ 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present.%0A✔ 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config.%0A✔ 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types.%0Aℹ 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. + + ── 2022-10-08-team1-goodmodel.csv ──── + + ✔ [file_read]: File could be read successfully. + ✔ [valid_round_id_col]: `round_id_col` name is valid. + ✔ [unique_round_id]: `round_id` column "origin_date" contains a single, unique round ID value. + ✔ [match_round_id]: All `round_id_col` "origin_date" values match submission `round_id` from file name. + ✔ [colnames]: Column names are consistent with expected round task IDs and std column names. + ✔ [col_types]: Column data types match hub schema. + ✔ [valid_vals]: `tbl` contains valid values/value combinations. + ✔ [rows_unique]: All combinations of task ID column/`output_type`/`output_type_id` values are unique. + ✔ [req_vals]: Required task ID/output type/output type ID combinations all present. + ✔ [value_col_valid]: Values in column `value` all valid with respect to modeling task config. + ✔ [value_col_non_desc]: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. + ℹ [value_col_sum1]: No pmf output types to check for sum of 1. Check skipped. # validate_model_data print method work [fancy] Code validate_model_data(hub_path, file_path) Message - ✔ 2022-10-08-team1-goodmodel.csv: File could be read successfully. - ✔ 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid. - ✔ 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value. - ✔ 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name. - ✔ 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names. - ✔ 2022-10-08-team1-goodmodel.csv: Column data types match hub schema. - ✔ 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations. - ✔ 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique. - ✔ 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present. - ✔ 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config. - ✔ 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. - ℹ 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. - ---- - - Code - validate_model_data(hub_path, file_path) - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✔ 2022-10-08-team1-goodmodel.csv: File could be read successfully.%0A✔ 2022-10-08-team1-goodmodel.csv: `round_id_col` name is valid.%0A✔ 2022-10-08-team1-goodmodel.csv: `round_id` column "origin_date" contains a single, unique round ID value.%0A✔ 2022-10-08-team1-goodmodel.csv: All `round_id_col` "origin_date" values match submission `round_id` from file name.%0A✔ 2022-10-08-team1-goodmodel.csv: Column names are consistent with expected round task IDs and std column names.%0A✔ 2022-10-08-team1-goodmodel.csv: Column data types match hub schema.%0A✔ 2022-10-08-team1-goodmodel.csv: `tbl` contains valid values/value combinations.%0A✔ 2022-10-08-team1-goodmodel.csv: All combinations of task ID column/`output_type`/`output_type_id` values are unique.%0A✔ 2022-10-08-team1-goodmodel.csv: Required task ID/output type/output type ID combinations all present.%0A✔ 2022-10-08-team1-goodmodel.csv: Values in column `value` all valid with respect to modeling task config.%0A✔ 2022-10-08-team1-goodmodel.csv: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types.%0Aℹ 2022-10-08-team1-goodmodel.csv: No pmf output types to check for sum of 1. Check skipped. + + ── 2022-10-08-team1-goodmodel.csv ──── + + ✔ [file_read]: File could be read successfully. + ✔ [valid_round_id_col]: `round_id_col` name is valid. + ✔ [unique_round_id]: `round_id` column "origin_date" contains a single, unique round ID value. + ✔ [match_round_id]: All `round_id_col` "origin_date" values match submission `round_id` from file name. + ✔ [colnames]: Column names are consistent with expected round task IDs and std column names. + ✔ [col_types]: Column data types match hub schema. + ✔ [valid_vals]: `tbl` contains valid values/value combinations. + ✔ [rows_unique]: All combinations of task ID column/`output_type`/`output_type_id` values are unique. + ✔ [req_vals]: Required task ID/output type/output type ID combinations all present. + ✔ [value_col_valid]: Values in column `value` all valid with respect to modeling task config. + ✔ [value_col_non_desc]: Values in `value` column are non-decreasing as output_type_ids increase for all unique task ID value/output type combinations of quantile or cdf output types. + ℹ [value_col_sum1]: No pmf output types to check for sum of 1. Check skipped. # validate_model_data errors correctly diff --git a/tests/testthat/_snaps/validate_model_file.md b/tests/testthat/_snaps/validate_model_file.md index 3a32ae4c..a0bab3cc 100644 --- a/tests/testthat/_snaps/validate_model_file.md +++ b/tests/testthat/_snaps/validate_model_file.md @@ -62,206 +62,158 @@ Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") Message - v 2022-10-08-team1-goodmodel.csv: File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'. - v 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid. - v 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name. - v 2022-10-08-team1-goodmodel.csv: `round_id` is valid. - v 2022-10-08-team1-goodmodel.csv: File is accepted hub format. - v 2022-10-08-team1-goodmodel.csv: Metadata file exists at path 'model-metadata/team1-goodmodel.yaml'. + + -- 2022-10-08-team1-goodmodel.csv ---- + + v [file_exists]: File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'. + v [file_name]: File name "2022-10-08-team1-goodmodel.csv" is valid. + v [file_location]: File directory name matches `model_id` metadata in file name. + v [round_id_valid]: `round_id` is valid. + v [file_format]: File is accepted hub format. + v [metadata_exists]: Metadata file exists at path 'model-metadata/team1-goodmodel.yaml'. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") Message - x 2022-10-15-team1-goodmodel.csv: File does not exist at path 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv'. + + -- 2022-10-15-team1-goodmodel.csv ---- + + (x) [file_exists]: File does not exist at path 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv'. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") Message - v 2022-10-15-hub-baseline.csv: File exists at path 'model-output/team1-goodmodel/2022-10-15-hub-baseline.csv'. - v 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid. - ! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" - v 2022-10-15-hub-baseline.csv: `round_id` is valid. - v 2022-10-15-hub-baseline.csv: File is accepted hub format. - v 2022-10-15-hub-baseline.csv: Metadata file exists at path 'model-metadata/hub-baseline.yml'. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::v 2022-10-08-team1-goodmodel.csv: File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'.%0Av 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid.%0Av 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name.%0Av 2022-10-08-team1-goodmodel.csv: `round_id` is valid.%0Av 2022-10-08-team1-goodmodel.csv: File is accepted hub format.%0Av 2022-10-08-team1-goodmodel.csv: Metadata file exists at path 'model-metadata/team1-goodmodel.yaml'. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::x 2022-10-15-team1-goodmodel.csv: File does not exist at path 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv'. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::v 2022-10-15-hub-baseline.csv: File exists at path 'model-output/team1-goodmodel/2022-10-15-hub-baseline.csv'.%0Av 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid.%0A! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel"%0Av 2022-10-15-hub-baseline.csv: `round_id` is valid.%0Av 2022-10-15-hub-baseline.csv: File is accepted hub format.%0Av 2022-10-15-hub-baseline.csv: Metadata file exists at path 'model-metadata/hub-baseline.yml'. + + -- 2022-10-15-hub-baseline.csv ---- + + v [file_exists]: File exists at path 'model-output/team1-goodmodel/2022-10-15-hub-baseline.csv'. + v [file_name]: File name "2022-10-15-hub-baseline.csv" is valid. + x [file_location]: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" + v [round_id_valid]: `round_id` is valid. + v [file_format]: File is accepted hub format. + v [metadata_exists]: Metadata file exists at path 'model-metadata/hub-baseline.yml'. # validate_model_file print method work [ansi] Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") Message - v 2022-10-08-team1-goodmodel.csv: File exists at path model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv. - v 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid. - v 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name. - v 2022-10-08-team1-goodmodel.csv: `round_id` is valid. - v 2022-10-08-team1-goodmodel.csv: File is accepted hub format. - v 2022-10-08-team1-goodmodel.csv: Metadata file exists at path model-metadata/team1-goodmodel.yaml. + + -- 2022-10-08-team1-goodmodel.csv ---- + + v [file_exists]: File exists at path model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv. + v [file_name]: File name "2022-10-08-team1-goodmodel.csv" is valid. + v [file_location]: File directory name matches `model_id` metadata in file name. + v [round_id_valid]: `round_id` is valid. + v [file_format]: File is accepted hub format. + v [metadata_exists]: Metadata file exists at path model-metadata/team1-goodmodel.yaml. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") Message - x 2022-10-15-team1-goodmodel.csv: File does not exist at path model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv. + + -- 2022-10-15-team1-goodmodel.csv ---- + + (x) [file_exists]: File does not exist at path model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") Message - v 2022-10-15-hub-baseline.csv: File exists at path model-output/team1-goodmodel/2022-10-15-hub-baseline.csv. - v 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid. - ! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" - v 2022-10-15-hub-baseline.csv: `round_id` is valid. - v 2022-10-15-hub-baseline.csv: File is accepted hub format. - v 2022-10-15-hub-baseline.csv: Metadata file exists at path model-metadata/hub-baseline.yml. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::v 2022-10-08-team1-goodmodel.csv: File exists at path model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv.%0Av 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid.%0Av 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name.%0Av 2022-10-08-team1-goodmodel.csv: `round_id` is valid.%0Av 2022-10-08-team1-goodmodel.csv: File is accepted hub format.%0Av 2022-10-08-team1-goodmodel.csv: Metadata file exists at path model-metadata/team1-goodmodel.yaml. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::x 2022-10-15-team1-goodmodel.csv: File does not exist at path model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::v 2022-10-15-hub-baseline.csv: File exists at path model-output/team1-goodmodel/2022-10-15-hub-baseline.csv.%0Av 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid.%0A! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel"%0Av 2022-10-15-hub-baseline.csv: `round_id` is valid.%0Av 2022-10-15-hub-baseline.csv: File is accepted hub format.%0Av 2022-10-15-hub-baseline.csv: Metadata file exists at path model-metadata/hub-baseline.yml. + + -- 2022-10-15-hub-baseline.csv ---- + + v [file_exists]: File exists at path model-output/team1-goodmodel/2022-10-15-hub-baseline.csv. + v [file_name]: File name "2022-10-15-hub-baseline.csv" is valid. + x [file_location]: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" + v [round_id_valid]: `round_id` is valid. + v [file_format]: File is accepted hub format. + v [metadata_exists]: Metadata file exists at path model-metadata/hub-baseline.yml. # validate_model_file print method work [unicode] Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") Message - ✔ 2022-10-08-team1-goodmodel.csv: File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'. - ✔ 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid. - ✔ 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name. - ✔ 2022-10-08-team1-goodmodel.csv: `round_id` is valid. - ✔ 2022-10-08-team1-goodmodel.csv: File is accepted hub format. - ✔ 2022-10-08-team1-goodmodel.csv: Metadata file exists at path 'model-metadata/team1-goodmodel.yaml'. + + ── 2022-10-08-team1-goodmodel.csv ──── + + ✔ [file_exists]: File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'. + ✔ [file_name]: File name "2022-10-08-team1-goodmodel.csv" is valid. + ✔ [file_location]: File directory name matches `model_id` metadata in file name. + ✔ [round_id_valid]: `round_id` is valid. + ✔ [file_format]: File is accepted hub format. + ✔ [metadata_exists]: Metadata file exists at path 'model-metadata/team1-goodmodel.yaml'. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") Message - ✖ 2022-10-15-team1-goodmodel.csv: File does not exist at path 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv'. + + ── 2022-10-15-team1-goodmodel.csv ──── + + ⓧ [file_exists]: File does not exist at path 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv'. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") Message - ✔ 2022-10-15-hub-baseline.csv: File exists at path 'model-output/team1-goodmodel/2022-10-15-hub-baseline.csv'. - ✔ 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid. - ! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" - ✔ 2022-10-15-hub-baseline.csv: `round_id` is valid. - ✔ 2022-10-15-hub-baseline.csv: File is accepted hub format. - ✔ 2022-10-15-hub-baseline.csv: Metadata file exists at path 'model-metadata/hub-baseline.yml'. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✔ 2022-10-08-team1-goodmodel.csv: File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'.%0A✔ 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid.%0A✔ 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name.%0A✔ 2022-10-08-team1-goodmodel.csv: `round_id` is valid.%0A✔ 2022-10-08-team1-goodmodel.csv: File is accepted hub format.%0A✔ 2022-10-08-team1-goodmodel.csv: Metadata file exists at path 'model-metadata/team1-goodmodel.yaml'. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✖ 2022-10-15-team1-goodmodel.csv: File does not exist at path 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv'. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✔ 2022-10-15-hub-baseline.csv: File exists at path 'model-output/team1-goodmodel/2022-10-15-hub-baseline.csv'.%0A✔ 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid.%0A! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel"%0A✔ 2022-10-15-hub-baseline.csv: `round_id` is valid.%0A✔ 2022-10-15-hub-baseline.csv: File is accepted hub format.%0A✔ 2022-10-15-hub-baseline.csv: Metadata file exists at path 'model-metadata/hub-baseline.yml'. + + ── 2022-10-15-hub-baseline.csv ──── + + ✔ [file_exists]: File exists at path 'model-output/team1-goodmodel/2022-10-15-hub-baseline.csv'. + ✔ [file_name]: File name "2022-10-15-hub-baseline.csv" is valid. + ✖ [file_location]: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" + ✔ [round_id_valid]: `round_id` is valid. + ✔ [file_format]: File is accepted hub format. + ✔ [metadata_exists]: Metadata file exists at path 'model-metadata/hub-baseline.yml'. # validate_model_file print method work [fancy] Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") Message - ✔ 2022-10-08-team1-goodmodel.csv: File exists at path model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv. - ✔ 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid. - ✔ 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name. - ✔ 2022-10-08-team1-goodmodel.csv: `round_id` is valid. - ✔ 2022-10-08-team1-goodmodel.csv: File is accepted hub format. - ✔ 2022-10-08-team1-goodmodel.csv: Metadata file exists at path model-metadata/team1-goodmodel.yaml. + + ── 2022-10-08-team1-goodmodel.csv ──── + + ✔ [file_exists]: File exists at path model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv. + ✔ [file_name]: File name "2022-10-08-team1-goodmodel.csv" is valid. + ✔ [file_location]: File directory name matches `model_id` metadata in file name. + ✔ [round_id_valid]: `round_id` is valid. + ✔ [file_format]: File is accepted hub format. + ✔ [metadata_exists]: Metadata file exists at path model-metadata/team1-goodmodel.yaml. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") Message - ✖ 2022-10-15-team1-goodmodel.csv: File does not exist at path model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv. + + ── 2022-10-15-team1-goodmodel.csv ──── + + ⓧ [file_exists]: File does not exist at path model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv. --- Code validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") Message - ✔ 2022-10-15-hub-baseline.csv: File exists at path model-output/team1-goodmodel/2022-10-15-hub-baseline.csv. - ✔ 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid. - ! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" - ✔ 2022-10-15-hub-baseline.csv: `round_id` is valid. - ✔ 2022-10-15-hub-baseline.csv: File is accepted hub format. - ✔ 2022-10-15-hub-baseline.csv: Metadata file exists at path model-metadata/hub-baseline.yml. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✔ 2022-10-08-team1-goodmodel.csv: File exists at path model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv.%0A✔ 2022-10-08-team1-goodmodel.csv: File name "2022-10-08-team1-goodmodel.csv" is valid.%0A✔ 2022-10-08-team1-goodmodel.csv: File directory name matches `model_id` metadata in file name.%0A✔ 2022-10-08-team1-goodmodel.csv: `round_id` is valid.%0A✔ 2022-10-08-team1-goodmodel.csv: File is accepted hub format.%0A✔ 2022-10-08-team1-goodmodel.csv: Metadata file exists at path model-metadata/team1-goodmodel.yaml. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✖ 2022-10-15-team1-goodmodel.csv: File does not exist at path model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv. - ---- - - Code - validate_model_file(hub_path, file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv") - Output - ::notice file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3::✔ 2022-10-15-hub-baseline.csv: File exists at path model-output/team1-goodmodel/2022-10-15-hub-baseline.csv.%0A✔ 2022-10-15-hub-baseline.csv: File name "2022-10-15-hub-baseline.csv" is valid.%0A! 2022-10-15-hub-baseline.csv: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel"%0A✔ 2022-10-15-hub-baseline.csv: `round_id` is valid.%0A✔ 2022-10-15-hub-baseline.csv: File is accepted hub format.%0A✔ 2022-10-15-hub-baseline.csv: Metadata file exists at path model-metadata/hub-baseline.yml. + + ── 2022-10-15-hub-baseline.csv ──── + + ✔ [file_exists]: File exists at path model-output/team1-goodmodel/2022-10-15-hub-baseline.csv. + ✔ [file_name]: File name "2022-10-15-hub-baseline.csv" is valid. + ✖ [file_location]: File directory name must match `model_id` metadata in file name. File should be submitted to directory "hub-baseline" not "team1-goodmodel" + ✔ [round_id_valid]: `round_id` is valid. + ✔ [file_format]: File is accepted hub format. + ✔ [metadata_exists]: Metadata file exists at path model-metadata/hub-baseline.yml. diff --git a/tests/testthat/_snaps/validate_pr.md b/tests/testthat/_snaps/validate_pr.md index f4035516..9ae58713 100644 --- a/tests/testthat/_snaps/validate_pr.md +++ b/tests/testthat/_snaps/validate_pr.md @@ -247,7 +247,7 @@ ..$ call : chr "check_config_hub_valid" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ model_output_mod_1 :List of 6 + $ model_output_mod :List of 6 ..$ message : chr "Previously submitted model output files must not be modified. \n 'model-output/hub-baseline/2022-10-08-hub-base"| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -255,7 +255,7 @@ ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... - $ model_output_mod_2 :List of 6 + $ model_output_mod_1 :List of 6 ..$ message : chr "Previously submitted model output files must not be removed. \n 'model-output/team1-goodmodel/2022-10-15-team1-"| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -263,7 +263,7 @@ ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... - $ model_metadata_mod_1 :List of 6 + $ model_metadata_mod :List of 6 ..$ message : chr "Previously submitted model metadata files must not be removed. \n 'model-metadata/team1-goodmodel.yaml' removed." ..$ trace : NULL ..$ parent : NULL @@ -412,37 +412,37 @@ ..$ call : chr "check_tbl_spl_n" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ file_exists :List of 4 + $ file_exists_1 :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-22-team1-goodmodel.csv'. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_exists" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_name :List of 4 + $ file_name_1 :List of 4 ..$ message : chr "File name \"2022-10-22-team1-goodmodel.csv\" is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location_1 :List of 4 ..$ message : chr "File directory name matches `model_id`\n metadata in file name. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_location" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ round_id_valid :List of 4 + $ round_id_valid_1 :List of 4 ..$ message : chr "`round_id` is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_format :List of 4 + $ file_format_1 :List of 4 ..$ message : chr "File is accepted hub format. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_format" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ metadata_exists :List of 6 + $ metadata_exists_1 :List of 6 ..$ message : chr "Metadata file does not exist at path 'model-metadata/team1-goodmodel.yml' or\n "| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -463,24 +463,30 @@ ..$ call : chr "check_config_hub_valid" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ model_output_mod_1 :List of 4 + $ model_output_mod :List of 6 ..$ message : chr "Previously submitted model output files must not be modified. \n 'model-output/hub-baseline/2022-10-08-hub-base"| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : 'fs_path' chr "hub-baseline/2022-10-08-hub-baseline.csv" ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... - $ model_output_mod_2 :List of 4 + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... + $ model_output_mod_1 :List of 6 ..$ message : chr "Previously submitted model output files must not be removed. \n 'model-output/team1-goodmodel/2022-10-15-team1-"| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-15-team1-goodmodel.csv" ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... - $ model_metadata_mod_1 :List of 4 + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... + $ model_metadata_mod :List of 6 ..$ message : chr "Previously submitted model metadata files must not be removed. \n 'model-metadata/team1-goodmodel.yaml' removed." + ..$ trace : NULL + ..$ parent : NULL ..$ where : 'fs_path' chr "team1-goodmodel.yaml" ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... $ file_exists :List of 4 ..$ message : chr "File exists at path 'model-output/hub-baseline/2022-10-08-hub-baseline.csv'. \n " ..$ where : 'fs_path' chr "hub-baseline/2022-10-08-hub-baseline.csv" @@ -622,37 +628,37 @@ ..$ call : chr "check_tbl_spl_n" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ file_exists :List of 4 + $ file_exists_1 :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-22-team1-goodmodel.csv'. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_exists" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_name :List of 4 + $ file_name_1 :List of 4 ..$ message : chr "File name \"2022-10-22-team1-goodmodel.csv\" is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location_1 :List of 4 ..$ message : chr "File directory name matches `model_id`\n metadata in file name. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_location" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ round_id_valid :List of 4 + $ round_id_valid_1 :List of 4 ..$ message : chr "`round_id` is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_format :List of 4 + $ file_format_1 :List of 4 ..$ message : chr "File is accepted hub format. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_format" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ metadata_exists :List of 6 + $ metadata_exists_1 :List of 6 ..$ message : chr "Metadata file does not exist at path 'model-metadata/team1-goodmodel.yml' or\n "| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -673,19 +679,19 @@ ..$ call : chr "check_config_hub_valid" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ model_output_mod_1 :List of 4 + $ model_output_mod :List of 4 ..$ message : chr "Previously submitted model output file\n 'model-output/hub-baseline/2022-10-08-hub-baseline.csv' modified." ..$ where : 'fs_path' chr "hub-baseline/2022-10-08-hub-baseline.csv" ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ model_output_mod_2 :List of 4 + $ model_output_mod_1 :List of 4 ..$ message : chr "Previously submitted model output file\n 'model-output/team1-goodmodel/2022-10-15-team1-goodmodel.csv' removed." ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-15-team1-goodmodel.csv" ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ model_metadata_mod_1 :List of 4 + $ model_metadata_mod :List of 4 ..$ message : chr "Previously submitted model metadata file\n 'model-metadata/team1-goodmodel.yaml' removed." ..$ where : 'fs_path' chr "team1-goodmodel.yaml" ..$ call : chr "check_pr_modf_del_file" @@ -832,37 +838,37 @@ ..$ call : chr "check_tbl_spl_n" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ file_exists :List of 4 + $ file_exists_1 :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-22-team1-goodmodel.csv'. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_exists" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_name :List of 4 + $ file_name_1 :List of 4 ..$ message : chr "File name \"2022-10-22-team1-goodmodel.csv\" is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location_1 :List of 4 ..$ message : chr "File directory name matches `model_id`\n metadata in file name. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_location" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ round_id_valid :List of 4 + $ round_id_valid_1 :List of 4 ..$ message : chr "`round_id` is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_format :List of 4 + $ file_format_1 :List of 4 ..$ message : chr "File is accepted hub format. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_format" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ metadata_exists :List of 6 + $ metadata_exists_1 :List of 6 ..$ message : chr "Metadata file does not exist at path 'model-metadata/team1-goodmodel.yml' or\n "| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -1024,37 +1030,37 @@ ..$ call : chr "check_tbl_spl_n" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ file_exists :List of 4 + $ file_exists_1 :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-22-team1-goodmodel.csv'. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_exists" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_name :List of 4 + $ file_name_1 :List of 4 ..$ message : chr "File name \"2022-10-22-team1-goodmodel.csv\" is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location_1 :List of 4 ..$ message : chr "File directory name matches `model_id`\n metadata in file name. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_location" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ round_id_valid :List of 4 + $ round_id_valid_1 :List of 4 ..$ message : chr "`round_id` is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_format :List of 4 + $ file_format_1 :List of 4 ..$ message : chr "File is accepted hub format. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_format" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ metadata_exists :List of 6 + $ metadata_exists_1 :List of 6 ..$ message : chr "Metadata file does not exist at path 'model-metadata/team1-goodmodel.yml' or\n "| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -1075,7 +1081,7 @@ ..$ call : chr "check_config_hub_valid" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ model_output_mod_1 :List of 6 + $ model_output_mod :List of 6 ..$ message : chr "Previously submitted model output files must not be removed. \n 'model-output/team1-goodmodel/2022-10-15-team1-"| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -1083,7 +1089,7 @@ ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... - $ model_metadata_mod_1 :List of 6 + $ model_metadata_mod :List of 6 ..$ message : chr "Previously submitted model metadata files must not be removed. \n 'model-metadata/team1-goodmodel.yaml' removed." ..$ trace : NULL ..$ parent : NULL @@ -1232,37 +1238,37 @@ ..$ call : chr "check_tbl_spl_n" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_info" "hub_check" "rlang_message" "message" ... - $ file_exists :List of 4 + $ file_exists_1 :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-22-team1-goodmodel.csv'. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_exists" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_name :List of 4 + $ file_name_1 :List of 4 ..$ message : chr "File name \"2022-10-22-team1-goodmodel.csv\" is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location_1 :List of 4 ..$ message : chr "File directory name matches `model_id`\n metadata in file name. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_location" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ round_id_valid :List of 4 + $ round_id_valid_1 :List of 4 ..$ message : chr "`round_id` is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_format :List of 4 + $ file_format_1 :List of 4 ..$ message : chr "File is accepted hub format. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_format" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ metadata_exists :List of 6 + $ metadata_exists_1 :List of 6 ..$ message : chr "Metadata file does not exist at path 'model-metadata/team1-goodmodel.yml' or\n "| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -1277,13 +1283,13 @@ str(mod_checks_exec_error[1:5]) Output List of 5 - $ valid_config :List of 4 + $ valid_config :List of 4 ..$ message : chr "All hub config files are valid. \n " ..$ where : chr "mod_exec_error_hub" ..$ call : chr "check_config_hub_valid" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ model_output_mod_1:List of 6 + $ model_output_mod:List of 6 ..$ message : chr "Could not check submission window for file \"team1-goodmodel/2022-10-team1-goodmodel.csv\". EXEC ERROR: Error i"| __truncated__ ..$ trace : NULL ..$ parent : NULL @@ -1291,19 +1297,19 @@ ..$ call : chr "check_pr_modf_del_file" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_exec_error" "hub_check" "rlang_error" "error" ... - $ file_exists :List of 4 + $ file_exists :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-22-team1-goodmodel.csv'. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_exists" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_name :List of 4 + $ file_name :List of 4 ..$ message : chr "File name \"2022-10-22-team1-goodmodel.csv\" is valid. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location :List of 4 ..$ message : chr "File directory name matches `model_id`\n metadata in file name. \n " ..$ where : 'fs_path' chr "team1-goodmodel/2022-10-22-team1-goodmodel.csv" ..$ call : chr "check_file_location" diff --git a/tests/testthat/_snaps/validate_submission.md b/tests/testthat/_snaps/validate_submission.md index bbd82308..ca6c6e29 100644 --- a/tests/testthat/_snaps/validate_submission.md +++ b/tests/testthat/_snaps/validate_submission.md @@ -159,12 +159,14 @@ ..$ call : chr "check_file_name" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ file_location :List of 4 + $ file_location :List of 6 ..$ message : chr "File directory name must match `model_id`\n metadata in file name. \n"| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "team1-goodmodel/2022-10-15-hub-baseline.csv" ..$ call : chr "check_file_location" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... $ round_id_valid :List of 4 ..$ message : chr "`round_id` is valid. \n " ..$ where : chr "team1-goodmodel/2022-10-15-hub-baseline.csv" @@ -217,7 +219,7 @@ round_id_col = "random_col", skip_submit_window_check = TRUE, skip_check_config = TRUE)) Output - List of 9 + Classes 'hub_validations', 'list' hidden list of 9 $ file_exists :List of 4 ..$ message : chr "File exists at path 'model-output/team1-goodmodel/2022-10-08-team1-goodmodel.csv'. \n " ..$ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" @@ -260,19 +262,22 @@ ..$ call : chr "check_file_read" ..$ use_cli_format: logi TRUE ..- attr(*, "class")= chr [1:5] "check_success" "hub_check" "rlang_message" "message" ... - $ valid_round_id_col:List of 4 + $ valid_round_id_col:List of 6 ..$ message : chr "`round_id_col` name must be valid. \n Must be one of\n \"origin_date\", \""| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" ..$ call : chr "check_valid_round_id_col" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... - $ unique_round_id :List of 4 + ..- attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... + $ unique_round_id :List of 6 ..$ message : chr "`round_id_col` name must be valid. \n Must be one of\n \"origin_date\", \""| __truncated__ + ..$ trace : NULL + ..$ parent : NULL ..$ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" ..$ call : chr "check_tbl_unique_round_id" ..$ use_cli_format: logi TRUE - ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_warning" "warning" ... - - attr(*, "class")= chr [1:2] "hub_validations" "list" + ..- attr(*, "class")= chr [1:5] "check_error" "hub_check" "rlang_error" "error" ... --- @@ -425,12 +430,14 @@ str(validate_submission(hub_path, file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv")[[ "submission_time"]]) Output - List of 4 + List of 6 $ message : chr "Submission time must be within accepted submission window for round. \n Current time \"2023-10-08 18:01:00 UTC\"| __truncated__ + $ trace : NULL + $ parent : NULL $ where : chr "team1-goodmodel/2022-10-08-team1-goodmodel.csv" $ call : chr "check_submission_time" $ use_cli_format: logi TRUE - - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_warning" "warning" ... + - attr(*, "class")= chr [1:5] "check_failure" "hub_check" "rlang_error" "error" ... # validate_submission csv file read in and validated according to schema. @@ -874,9 +881,9 @@ validate_submission(hub_path = test_path("testdata/hub-it"), file_path = "Tm-Md/2023-11-11-Tm-Md.parquet", skip_submit_window_check = TRUE)[["col_types"]] Output - - Warning: - Column data types do not match hub schema. `output_type_id` should be "character" not "double". + + Error: + ! Column data types do not match hub schema. `output_type_id` should be "character" not "double". --- @@ -907,9 +914,9 @@ skip_submit_window_check = TRUE, output_type_id_datatype = "character")[[ "col_types"]] Output - - Warning: - Column data types do not match hub schema. `output_type_id` should be "character" not "double". + + Error: + ! Column data types do not match hub schema. `output_type_id` should be "character" not "double". --- @@ -928,9 +935,9 @@ skip_submit_window_check = TRUE, output_type_id_datatype = "double")[[ "col_types"]] Output - - Warning: - Column data types do not match hub schema. `output_type_id` should be "double" not "character". + + Error: + ! Column data types do not match hub schema. `output_type_id` should be "double" not "character". --- diff --git a/tests/testthat/test-validate_model_data.R b/tests/testthat/test-validate_model_data.R index 457b28e4..92f899c8 100644 --- a/tests/testthat/test-validate_model_data.R +++ b/tests/testthat/test-validate_model_data.R @@ -54,15 +54,8 @@ test_that("validate_model_data with config function works", { cli::test_that_cli("validate_model_data print method work", { - mockery::stub( - octolog:::signal_github_condition, - "get_location_string", - "file=test-validate_model_data.R,line=57,endLine=61,col=3,endCol=3", - 2 - ) hub_path <- system.file("testhubs/simple", package = "hubValidations") file_path <- "team1-goodmodel/2022-10-08-team1-goodmodel.csv" - withr::local_envvar(GITHUB_ACTIONS = "false") # File that passes validation expect_snapshot( @@ -71,13 +64,6 @@ cli::test_that_cli("validate_model_data print method work", { # File with validation error validate_model_data(hub_path, file_path, round_id_col = "random_col") - withr::local_envvar(GITHUB_ACTIONS = "true") - # File that passes validation - expect_snapshot( - validate_model_data(hub_path, file_path) - ) - # File with validation error - validate_model_data(hub_path, file_path, round_id_col = "random_col") }) diff --git a/tests/testthat/test-validate_model_file.R b/tests/testthat/test-validate_model_file.R index 9d5bd32d..b77fae5a 100644 --- a/tests/testthat/test-validate_model_file.R +++ b/tests/testthat/test-validate_model_file.R @@ -33,15 +33,8 @@ test_that("validate_model_file works", { }) cli::test_that_cli("validate_model_file print method work", { - mockery::stub( - octolog:::signal_github_condition, - "get_location_string", - "file=test-validate_model_file.R,line=57,endLine=61,col=3,endCol=3", - 2 - ) hub_path <- system.file("testhubs/simple", package = "hubValidations") - withr::local_envvar(GITHUB_ACTIONS = "false") # File that passes validation expect_snapshot( validate_model_file( @@ -60,27 +53,4 @@ cli::test_that_cli("validate_model_file print method work", { file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv" ) ) - - withr::local_envvar(GITHUB_ACTIONS = "true") - expect_snapshot( - validate_model_file( - hub_path, - file_path = "team1-goodmodel/2022-10-08-team1-goodmodel.csv" - ) - ) - # File with validation error - expect_snapshot( - validate_model_file(hub_path, - file_path = "team1-goodmodel/2022-10-15-team1-goodmodel.csv" - ) - ) - expect_snapshot( - validate_model_file(hub_path, - file_path = "team1-goodmodel/2022-10-15-hub-baseline.csv" - ) - ) - # BELOW: Octolog example of testing GA action error behaviour. - # Placemark for future testing - # expect_snapshot(octo_abort("An error"), error = TRUE) - # expect_snapshot(octo_abort("An error", .fail_fast = FALSE)) }) diff --git a/vignettes/articles/hub-validations-class.Rmd b/vignettes/articles/hub-validations-class.Rmd index 1d38cbdc..8641aebb 100644 --- a/vignettes/articles/hub-validations-class.Rmd +++ b/vignettes/articles/hub-validations-class.Rmd @@ -40,22 +40,28 @@ The super class returned in each element depends on the status of the check: - If a check is skipped, a `` condition class object is returned. -- Checks vary with respect to whether they return an `` or `` condition class object if the check fails. -Ultimately, both will cause overall validation to fail and the two classes are used primarily to communicate the severity of a failing check. +- Checks vary with respect to whether they return an `` or `` condition class object if the check fails. + - **``** class objects indicate a check that failed but does not affect downstream checks so validation was able to proceed. + - **`` ** class objects indicate early termination of the validation process because of failure of a check downstream checks depend on. + +Ultimately, both will cause overall validation to fail. The `` class exists to alert you to the fact that _there may be more errors not yet reported_ due to early termination of the check process. ## `hub_validations` print method -`hub_validations` objects have their own print method which displays the result, the file name and message of each check: +`hub_validations` objects have their own print method which displays the result, the check name and message of each check: - `✔` indicates a check was successful (a `` condition class object was returned) -- `✖` indicates a high severity check failed (a `` condition class object was returned) -- `!` indicates a lower severity check failed (a `` condition class object was returned) +- `✖` indicates a check failed but, because it does not affect downstream checks, validation was able to proceed (a `` condition class object was returned) +- `ⓧ` indicates a check that downstream checks depend on failed, causing early return of the validation process (a `` condition class object was returned) +- `☒` indicates an execution error occured and the check was not able to complete (a `` condition class object was returned). Will cause early return if expected check failure output was a ``. - `ℹ` indicates a check was skipped (a `` condition class object was returned) ```{r} v ``` +Note that the submission window check is always performed and reported last. + ## Structure of a `` object Let's look more closely at the structure of the first few elements of the `hub_validations` object retuned by `validate_submission()` diff --git a/vignettes/articles/validate-pr.Rmd b/vignettes/articles/validate-pr.Rmd index 9571c1af..608b42be 100644 --- a/vignettes/articles/validate-pr.Rmd +++ b/vignettes/articles/validate-pr.Rmd @@ -84,7 +84,7 @@ Ignoring derived task IDs means that the validity of derived task ID value combi Most hubs require that model output files for a given round are submitted within a submission window [defined in the `"submission_due"` property of the `tasks.json` hub config file](https://hubverse.io/en/latest/quickstart-hub-admin/tasks-config.html#setting-up-submissions-due). -`validate_pr()` includes submission window checks for model output files and returns a `` condition class object if a file is submitted outside the accepted submission window. +`validate_pr()` includes submission window checks for model output files and returns a `` condition class object if a file is submitted outside the accepted submission window. To disable submission window checks, argument `skip_submit_window_check` can be set to `TRUE`.