HopkinsIDD · saraloo · Apr 26, 2024 · Apr 15, 2024 · Apr 15, 2024 · Apr 15, 2024
diff --git a/batch/inference_job_launcher.py b/batch/inference_job_launcher.py
@@ -431,7 +431,9 @@ def autodetect_params(config, data_path, *, num_jobs=None, sims_per_job=None, nu
             print(f"Setting number of blocks to {num_blocks} [via num_blocks (-k) argument]")
             print(f"Setting sims per job to {sims_per_job} [via {iterations_per_slot} iterations_per_slot in config]")
         else:
-            geodata_fname = pathlib.Path(data_path, config["data_path"]) / config["subpop_setup"]["geodata"]
+            if config["data_path"]:
+                raise ValueError("The config has a data_path section. This is no longer supported.")
+            geodata_fname = pathlib.Path(data_path) / config["subpop_setup"]["geodata"]
             with open(geodata_fname) as geodata_fp:
                 num_subpops = sum(1 for line in geodata_fp)
 

diff --git a/datasetup/build_US_setup.R b/datasetup/build_US_setup.R
@@ -51,13 +51,10 @@ if (length(config) == 0) {
   stop("no configuration found -- please set CONFIG_PATH environment variable or use the -c command flag")
 }
 
-outdir <- config$data_path
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
 # Aggregation to state level if in config
 state_level <- ifelse(!is.null(config$subpop_setup$state_level) && config$subpop_setup$state_level, TRUE, FALSE)
 
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 # commute_data <- arrow::read_parquet(file.path(opt$p,"datasetup", "usdata","united-states-commutes","commute_data.gz.parquet"))
 # census_data <- arrow::read_parquet(file.path(opt$p,"datasetup", "usdata","united-states-commutes","census_tracts_2010.gz.parquet"))
 
@@ -156,22 +153,25 @@ if (length(config$subpop_setup$geodata) > 0) {
 # manually remove PR
 census_data <- census_data %>% filter(USPS != "PR")
 
-write.csv(file = file.path(outdir, geodata_file), census_data, row.names=FALSE)
-print(paste("Wrote geodata file:", file.path(outdir, geodata_file)))
+
+if(!dir.exists(dirname(config$subpop_setup$geodata))){
+  dir.create(dirname(config$subpop_setup$geodata))}
+write.csv(file = file.path(geodata_file), census_data, row.names=FALSE)
+print(paste("Wrote geodata file:", file.path(geodata_file)))
 
 
 
 
 # MOBILITY DATA (COMMUTER DATA) ------------------------------------------------------------
 
 
-if(state_level & !file.exists(paste0(config$data_path, "/", config$subpop_setup$mobility))){
+if(state_level & !file.exists(paste0(config$subpop_setup$mobility))){
 
   warning(paste("State-level mobility files must be created manually because `build_US_setup.R` does not generate a state-level mobility file automatically. No valid mobility file named", paste0(config$data_path, "/", config$subpop_setup$mobility), "(specified in the config) currently exists. Please check again."))
 
-} else if(state_level & file.exists(paste0(config$data_path, "/", config$subpop_setup$mobility))){
+} else if(state_level & file.exists(paste0(config$subpop_setup$mobility))){
 
-  warning(paste("Using existing state-level mobility file named", paste0(config$data_path, "/", config$subpop_setup$mobility)))
+  warning(paste("Using existing state-level mobility file named", paste0(config$subpop_setup$mobility)))
 
 } else{
 
@@ -209,21 +209,21 @@ if(state_level & !file.exists(paste0(config$data_path, "/", config$subpop_setup$
       print(census_data$subpop)
       stop("There was a problem generating the mobility matrix")
     }
-    write.table(file = file.path(outdir, mobility_file), as.matrix(rc[,-1]), row.names=FALSE, col.names = FALSE, sep = " ")
+    write.table(file = file.path(mobility_file), as.matrix(rc[,-1]), row.names=FALSE, col.names = FALSE, sep = " ")
 
     } else if(endsWith(mobility_file, '.csv')) {
 
       rc <- commute_data
       names(rc) <- c("ori","dest","amount")
 
       rc <- rc[rc$ori != rc$dest,]
-      write.csv(file = file.path(outdir, mobility_file), rc, row.names=FALSE)
+      write.csv(file = file.path(mobility_file), rc, row.names=FALSE)
 
     } else {
       stop("Only .txt and .csv extensions supported for mobility matrix. Please check config's subpop_setup::mobility.")
     }
 
-    print(paste("Wrote mobility file:", file.path(outdir, mobility_file)))
+    print(paste("Wrote mobility file:", file.path(mobility_file)))
 }
 
 

diff --git a/datasetup/build_covid_data.R b/datasetup/build_covid_data.R
@@ -30,17 +30,14 @@ if (exists("config$inference$gt_source")) {
     opt$gt_data_source <- config$inference$gt_source
 }
 
-outdir <- config$data_path
 # filterUSPS <- config$subpop_setup$modeled_states
 filterUSPS <- c("WY","VT","DC","AK","ND","SD","DE","MT","RI","ME","NH","HI","ID","WV","NE","NM",
                 "KS","NV","MS","AR","UT","IA","CT","OK","OR","KY","LA","AL","SC","MN","CO","WI",
                 "MD","MO","IN","TN","MA","AZ","WA","VA","NJ","MI","NC","GA","OH","IL","PA","NY","FL","TX","CA")
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
 # Aggregation to state level if in config
 state_level <- ifelse(!is.null(config$subpop_setup$state_level) && config$subpop_setup$state_level, TRUE, FALSE)
 
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
 # source data functions
 source(file.path(opt$path, "datasetup/data_setup_source.R"))
@@ -221,7 +218,7 @@ if (any(grepl("fluview", opt$gt_data_source))){
 
     max(fluview_data$Update)
 
-    census_data <- read_csv(file = file.path(config$data_path, config$subpop_setup$geodata))
+    census_data <- read_csv(file = file.path(config$subpop_setup$geodata))
     fluview_data <- fluview_data %>%
         dplyr::inner_join(census_data %>% dplyr::select(source = USPS, FIPS = subpop)) %>%
         dplyr::select(Update, source, FIPS, incidD)
@@ -286,7 +283,7 @@ if (any(grepl("fluview", opt$gt_data_source))){
 #
 #     max(fluview_data$Update)
 #
-#     census_data <- read_csv(file = file.path(config$data_path, config$subpop_setup$geodata))
+#     census_data <- read_csv(file = file.path(config$subpop_setup$geodata))
 #     fluview_data <- fluview_data %>%
 #         left_join(census_data %>% dplyr::select(source = USPS, FIPS = subpop)) %>%
 #         dplyr::select(Update, source, FIPS, incidD)
@@ -392,7 +389,8 @@ us_data <- us_data %>%
     # mutate(across(starts_with("incid"), ~ replace_na(.x, 0))) %>%
     mutate(across(starts_with("incid"), ~ as.numeric(.x)))
 
-
+if(!dir.exists(dirname(config$inference$gt_data_path))){
+  dir.create(dirname(config$inference$gt_data_path))}
 # Save
 write_csv(us_data, config$inference$gt_data_path)
 

diff --git a/datasetup/build_flu_data.R b/datasetup/build_flu_data.R
@@ -31,14 +31,11 @@ if (length(config) == 0) {
     stop("no configuration found -- please set CONFIG_PATH environment variable or use the -c command flag")
 }
 
-outdir <- config$data_path
 filterUSPS <- config$subpop_setup$modeled_states
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
 # Aggregation to state level if in config
 state_level <- ifelse(!is.null(config$subpop_setup$state_level) && config$subpop_setup$state_level, TRUE, FALSE)
 
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
 
 
@@ -59,7 +56,7 @@ source("https://raw.githubusercontent.com/cdcepi/Flusight-forecast-data/master/d
 
 # Pull daily hospitalizations for model run
 us_data <- load_flu_hosp_data(temporal_resolution = 'daily', na.rm = TRUE)
-locs <- read_csv(file.path(config$data_path, config$subpop_setup$geodata))
+locs <- read_csv(file.path(config$subpop_setup$geodata))
 
 # fix string pad issue on left side
 us_data <- us_data %>%
@@ -83,6 +80,8 @@ us_data <- us_data %>%
     filter(date >= lubridate::as_date(config$start_date) & date <= lubridate::as_date(end_date_)) %>%
     filter(!is.na(source))
 
+if(!dir.exists(dirname(config$inference$gt_data_path))){
+  dir.create(dirname(config$inference$gt_data_path))}
 write_csv(us_data, config$inference$gt_data_path)
 
 
@@ -100,7 +99,7 @@ adjust_for_variant <- !is.null(variant_props_file)
 # if (adjust_for_variant){
 #
 #     # Variant Data (need to automate this data pull still)
-#     #variant_data <- read_csv(file.path(config$data_path, "variant/WHO_NREVSS_Clinical_Labs.csv"), skip = 1)
+#     #variant_data <- read_csv(file.path("variant/WHO_NREVSS_Clinical_Labs.csv"), skip = 1)
 #     variant_data <- cdcfluview::who_nrevss(region="state", years = 2022)$clinical_labs
 #
 #     # location data

diff --git a/datasetup/build_nonUS_setup.R b/datasetup/build_nonUS_setup.R
@@ -41,16 +41,14 @@ if (length(config) == 0) {
   stop("no configuration found -- please set CONFIG_PATH environment variable or use the -c command flag")
 }
 
-outdir <- config$data_path
 filterADMIN0 <- config$subpop_setup$modeled_states
 
-dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
 # Read in needed data
-commute_data <- readr::read_csv(file.path(config$data_path, "geodata", opt$mobility)) %>%
+commute_data <- readr::read_csv(file.path("geodata", opt$mobility)) %>%
   mutate(OGEOID = as.character(OGEOID),
          DGEOID = as.character(DGEOID))
-census_data <- readr::read_csv(file.path(config$data_path, "geodata", opt$population)) %>%
+census_data <- readr::read_csv(file.path("geodata", opt$population)) %>%
   mutate(GEOID = as.character(GEOID))
 
 # Filter if needed
@@ -97,22 +95,24 @@ if(opt$w){
   if(!isTRUE(all(rc$OGEOID == census_data$GEOID))){
     stop("There was a problem generating the mobility matrix")
   }
-  write.table(file = file.path(outdir,'mobility.txt'), as.matrix(rc[,-1]), row.names=FALSE, col.names = FALSE, sep = " ")
+  write.table(file = file.path('mobility.txt'), as.matrix(rc[,-1]), row.names=FALSE, col.names = FALSE, sep = " ")
 } else {
   names(rc) <- c("ori","dest","amount")
   rc <- rc[rc$ori != rc$dest,]
-  write.csv(file = file.path(outdir,'mobility.csv'), rc, row.names=FALSE)
+  write.csv(file = file.path('mobility.csv'), rc, row.names=FALSE)
 }
 
 # Save population geodata
+if(!dir.exists(dirname(config$subpop_setup$geodata))){
+  dir.create(dirname(config$subpop_setup$geodata))}
 names(census_data) <- c("subpop","admin2","admin0","pop")
-write.csv(file = file.path(outdir,'geodata.csv'), census_data,row.names=FALSE)
+write.csv(file = file.path('geodata.csv'), census_data,row.names=FALSE)
 
 print("Census Data Check (up to 6 rows)")
 print(head(census_data))
 print("Commute Data Check (up to 6 rows)")
 print(head(commute_data))
 
-print(paste0("mobility.csv/.txt and geodata.csv saved to: ", outdir))
+#print(paste0("mobility.csv/.txt and geodata.csv saved to: ", outdir))
 
 
diff --git a/flepimop/R_packages/flepicommon/R/config_test_new.R b/flepimop/R_packages/flepicommon/R/config_test_new.R
@@ -122,7 +122,7 @@ validation_list$subpop_setup$geodata <- function(value, full_config,config_name)
     print("No geodata path mentioned in the configuration file")
     return(FALSE)
   }else{
-    path=paste(full_config$data_path,'/',value,sep='')
+    path=paste(full_config$subpop_setup$geodata,'/',value,sep='')
     if (!file.exists(path)) {
       print(paste("The mentioned geodata file :", value, "could not be found."))
       return(FALSE)
@@ -136,7 +136,7 @@ validation_list$subpop_setup$mobility <- function(value, full_config,config_name
     print("No mobility path mentioned in the configuration file")
     return(FALSE)
   }else{
-    path=paste(full_config$data_path,'/',value,sep='')
+    path=paste(full_config$subpop_setup$mobility,'/',value,sep='')
     if (!file.exists(path)) {
       print(paste("The mentioned mobility file :", value, "could not be found."))
       return(FALSE)

diff --git a/flepimop/gempyor_pkg/docs/integration_benchmark.ipynb b/flepimop/gempyor_pkg/docs/integration_benchmark.ipynb
@@ -122,7 +122,7 @@
     "config.set_file(config_path)\n",
     "\n",
     "spatial_config = config[\"subpop_setup\"]\n",
-    "spatial_base_path = pathlib.Path(\"../../COVID19_USA/\" + config[\"data_path\"].get())\n",
+    "spatial_base_path = pathlib.Path(\"../../COVID19_USA/\")\n",
     "seir_modifiers_scenario = seir_modifiers_scenario\n",
     "outcome_modifiers_scenario = outcome_modifiers_scenario\n",
     "stoch_traj_flag = stoch_traj_flag  # Truthy: stochastic simulation, Falsy: determnistic mean of the binomial draws\n",

diff --git a/flepimop/gempyor_pkg/src/gempyor/model_info.py b/flepimop/gempyor_pkg/src/gempyor/model_info.py
@@ -77,8 +77,9 @@ def __init__(
 
         # 3. What about subpopulations
         spatial_config = config["subpop_setup"]
-        spatial_base_path = config["data_path"].get()
-        spatial_base_path = pathlib.Path(spatial_path_prefix + spatial_base_path)
+        if config["data_path"].exists():
+            raise ValueError("The config has a data_path section. This is no longer supported.")
+        spatial_base_path = pathlib.Path(spatial_path_prefix)
 
         self.subpop_struct = subpopulation_structure.SubpopulationStructure(
             setup_name=config["setup_name"].get(),

diff --git a/flepimop/gempyor_pkg/tests/interface/data/config.yml b/flepimop/gempyor_pkg/tests/interface/data/config.yml
@@ -2,13 +2,12 @@ name: minimal for interface
 setup_name: minimal4interface
 start_date: 2020-01-31
 end_date: 2020-05-31
-data_path: data
 nslots: 1
 
 
 spatial_setup:
-  geodata: geodata.csv
-  mobility: mobility.csv
+  geodata: data/geodata.csv
+  mobility: data/mobility.csv
   popnodes_key: population
   subpop_names_key: subpop
 

diff --git a/flepimop/gempyor_pkg/tests/interface/data/config_min_test.yml b/flepimop/gempyor_pkg/tests/interface/data/config_min_test.yml
@@ -2,13 +2,12 @@ name: minimal for interface
 setup_name: minimal4interface
 start_date: 2020-01-31
 end_date: 2020-05-31
-data_path: data
 nslots: 1
 
 
 spatial_setup:
-  geodata: geodata.csv
-  mobility: mobility.csv
+  geodata: data/geodata.csv
+  mobility: data/mobility.csv
   popnodes: population
   nodenames: geoid
 

diff --git a/flepimop/gempyor_pkg/tests/interface/data/config_minimal.yaml b/flepimop/gempyor_pkg/tests/interface/data/config_minimal.yaml
@@ -2,13 +2,12 @@ name: minimal
 setup_name: minimal
 start_date: 2020-01-31
 end_date: 2020-05-31
-data_path: data
 nslots: 15
 
 
 spatial_setup:
-  geodata: geodata.csv
-  mobility: mobility.txt
+  geodata: data/geodata.csv
+  mobility: data/mobility.txt
   popnodes: population
   nodenames: geoid
 

diff --git a/flepimop/gempyor_pkg/tests/interface/data/config_test.yml b/flepimop/gempyor_pkg/tests/interface/data/config_test.yml
@@ -2,13 +2,12 @@ name: minimal_test
 setup_name: minimal_test_setup
 start_date: 2020-01-31
 end_date: 2020-05-31
-data_path: data
 nslots: 5
 
 
 subpop_setup:
-  geodata: geodata.csv
-  mobility: mobility.csv
+  geodata: data/geodata.csv
+  mobility: data/mobility.csv
 
 
 seeding:

diff --git a/flepimop/gempyor_pkg/tests/npi/config_npi.yml b/flepimop/gempyor_pkg/tests/npi/config_npi.yml
@@ -1,7 +1,6 @@
 name: USA
 setup_name: FCH
 smh_round: R12
-data_path: data
 start_date: 2020-01-01
 end_date: 2022-09-03
 end_date_groundtruth: 2022-03-05
@@ -14,8 +13,8 @@ compartments:
   age_strata: ["age0to17", "age18to64", "age65to100"]
 
 subpop_setup:
-  geodata: geodata_2019_statelevel.csv
-  mobility: mobility_2011-2015_statelevel.csv
+  geodata: data/geodata_2019_statelevel.csv
+  mobility: data/mobility_2011-2015_statelevel.csv
 
 
 seeding:

diff --git a/flepimop/gempyor_pkg/tests/npi/config_test_spatial_group_npi.yml b/flepimop/gempyor_pkg/tests/npi/config_test_spatial_group_npi.yml
@@ -1,7 +1,6 @@
 name: USA
 setup_name: FCH
 smh_round: R12
-data_path: data
 start_date: 2020-01-01
 end_date: 2022-09-03
 end_date_groundtruth: 2022-03-05
@@ -14,8 +13,8 @@ compartments:
   age_strata: ["age0to17", "age18to64", "age65to100"]
 
 subpop_setup:
-  geodata: geodata_2019_statelevel.csv
-  mobility: mobility_2011-2015_statelevel.csv
+  geodata: data/geodata_2019_statelevel.csv
+  mobility: data/mobility_2011-2015_statelevel.csv
   include_in_report: include_in_report
   state_level: TRUE
 

diff --git a/flepimop/gempyor_pkg/tests/npi/data/config_minimal.yaml b/flepimop/gempyor_pkg/tests/npi/data/config_minimal.yaml
@@ -2,13 +2,12 @@ name: minimal
 setup_name: minimal
 start_date: 2020-01-31
 end_date: 2020-05-31
-data_path: data
 nslots: 15
 
 
 spatial_setup:
-  geodata: geodata.csv
-  mobility: mobility.txt
+  geodata: data/geodata.csv
+  mobility: data/mobility.txt
   popnodes: population
   nodenames: geoid
 

diff --git a/flepimop/gempyor_pkg/tests/npi/data/config_test.yml b/flepimop/gempyor_pkg/tests/npi/data/config_test.yml
@@ -2,13 +2,12 @@ name: minimal_test
 setup_name: minimal_test_setup
 start_date: 2020-01-31
 end_date: 2020-05-31
-data_path: data
 nslots: 5
 
 
 subpop_setup:
-  geodata: geodata.csv
-  mobility: mobility.csv
+  geodata: data/geodata.csv
+  mobility: data/mobility.csv
 
 
 seeding: