From c47f5133385a6271aff8d03d78176439a4caea01 Mon Sep 17 00:00:00 2001 From: Joseph Lemaitre Date: Mon, 16 Oct 2023 14:11:29 +0200 Subject: [PATCH 1/5] gempyor to provide run_id and not assume trailing --- flepimop/gempyor_pkg/src/gempyor/file_paths.py | 2 +- flepimop/gempyor_pkg/src/gempyor/interface.py | 4 ++-- flepimop/gempyor_pkg/src/gempyor/seeding_ic.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/file_paths.py b/flepimop/gempyor_pkg/src/gempyor/file_paths.py index 2a5ad4e5c..ce4414351 100644 --- a/flepimop/gempyor_pkg/src/gempyor/file_paths.py +++ b/flepimop/gempyor_pkg/src/gempyor/file_paths.py @@ -13,7 +13,7 @@ def create_file_name_without_extension(run_id, prefix, index, ftype, inference_f if create_directory: os.makedirs(create_dir_name(run_id, prefix, ftype, inference_filepath_suffix, inference_filename_prefix), exist_ok=True) filename = pathlib.Path("model_output", prefix, ftype, inference_filepath_suffix, - f"{inference_filename_prefix}.{index:>09}.{run_id}.{ftype}") + f"{inference_filename_prefix}{index:>09}.{run_id}.{ftype}") print(f" making {filename}") # old: "model_output/%s/%s%09d.%s.%s" % (ftype, prefix, index, run_id, ftype) diff --git a/flepimop/gempyor_pkg/src/gempyor/interface.py b/flepimop/gempyor_pkg/src/gempyor/interface.py index 782b9d5a2..6a5d8852b 100644 --- a/flepimop/gempyor_pkg/src/gempyor/interface.py +++ b/flepimop/gempyor_pkg/src/gempyor/interface.py @@ -43,7 +43,7 @@ def __init__( self, config_path, run_id="test_run_id", - prefix="test_prefix", + prefix=None, first_sim_index=1, seir_modifiers_scenario=None, outcome_modifiers_scenario=None, @@ -97,7 +97,7 @@ def __init__( print( f""" gempyor >> Running ***{'STOCHASTIC' if stoch_traj_flag else 'DETERMINISTIC'}*** simulation;\n""" f""" gempyor >> ModelInfo {self.modinf.setup_name}; index: {self.modinf.first_sim_index}; run_id: {in_run_id},\n""" - f""" gempyor >> prefix: {in_prefix};""" # ti: {s.ti}; tf: {s.tf}; + f""" gempyor >> prefix: {self.modinf.in_prefix};""" # ti: {s.ti}; tf: {s.tf}; ) self.already_built = False # whether we have already build the costly objects that need just one build diff --git a/flepimop/gempyor_pkg/src/gempyor/seeding_ic.py b/flepimop/gempyor_pkg/src/gempyor/seeding_ic.py index 18020b755..64f0b93b8 100644 --- a/flepimop/gempyor_pkg/src/gempyor/seeding_ic.py +++ b/flepimop/gempyor_pkg/src/gempyor/seeding_ic.py @@ -267,7 +267,7 @@ def draw_seeding(self, sim_id: int, setup) -> nb.typed.Dict: elif method == "FolderDraw": seeding = pd.read_csv( setup.get_input_filename( - ftype=setup.seeding_config["seeding_file_type"], + ftype=setup.seeding_config["seeding_file_type"].get(), sim_id=sim_id, extension_override="csv", ), From f1023ddc77b2169e276737674e7201db2edc79ec Mon Sep 17 00:00:00 2001 From: Joseph Lemaitre Date: Mon, 16 Oct 2023 16:57:20 +0200 Subject: [PATCH 2/5] File management > upgraded + explicit arguments --- .../R_packages/flepicommon/R/file_paths.R | 6 +- .../inference/R/inference_slot_runner_funcs.R | 238 +++++++++--------- .../testthat/test-perform_MCMC_step_copies.R | 76 +++--- .../gempyor_pkg/src/gempyor/file_paths.py | 2 - flepimop/main_scripts/inference_slot.R | 180 +++++++------ 5 files changed, 265 insertions(+), 237 deletions(-) diff --git a/flepimop/R_packages/flepicommon/R/file_paths.R b/flepimop/R_packages/flepicommon/R/file_paths.R index 701c9b1bd..806f3fa85 100644 --- a/flepimop/R_packages/flepicommon/R/file_paths.R +++ b/flepimop/R_packages/flepicommon/R/file_paths.R @@ -49,9 +49,9 @@ create_prefix <- function(..., prefix='',sep='-',trailing_separator=""){ ## Function for creating file names from their components ##' @export -create_file_name <- function(run_id, prefix, suffix, index, type, extension='parquet', create_directory = TRUE){ - rc <- sprintf("model_output/%s/%s/%s/%09d.%s.%s.%s", prefix, type, suffix, index, run_id, type, extension) - if(create_directory){ +create_file_name <- function(run_id, prefix, filepath_suffix, filename_prefix, index, type, extension='parquet', create_directory = TRUE){ + rc <- sprintf("model_output/%s/%s/%s/%s/%s%09d.%s.%s.%s", prefix,run_id, type, filepath_suffix, filename_prefix, index, run_id, type, extension) + if(create_directory){ # Add filename prefix here. if(!dir.exists(dirname(rc))){ dir.create(dirname(rc), recursive = TRUE) } diff --git a/flepimop/R_packages/inference/R/inference_slot_runner_funcs.R b/flepimop/R_packages/inference/R/inference_slot_runner_funcs.R index 6f6ca7190..9ae1fa22e 100644 --- a/flepimop/R_packages/inference/R/inference_slot_runner_funcs.R +++ b/flepimop/R_packages/inference/R/inference_slot_runner_funcs.R @@ -223,159 +223,160 @@ perform_MCMC_step_copies_global <- function(current_index, slot, block, run_id, - global_local_prefix, - gf_prefix, - global_block_prefix) { + global_intermediate_filepath_suffix, + slotblock_filename_prefix, + slot_filename_prefix + ) { rc <- list() if(current_index != 0){ #move files from global/intermediate/slot.block.run to global/final/slot rc$seed_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'seed','csv'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='seed',extension='csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='seed',extension='csv'), overwrite = TRUE ) rc$init_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'init','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'init','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='init',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='init',extension='parquet'), overwrite = TRUE ) rc$seir_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'seir','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'seir','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='seir',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='seir',extension='parquet'), overwrite = TRUE ) rc$hosp_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hosp','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'hosp','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hosp',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='hosp',extension='parquet'), overwrite = TRUE ) rc$llik_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'llik','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'llik','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='llik',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='llik',extension='parquet'), overwrite = TRUE ) rc$snpi_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'snpi','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'snpi','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='snpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='snpi',extension='parquet'), overwrite = TRUE ) rc$hnpi_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hnpi','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'hnpi','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hnpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='hnpi',extension='parquet'), overwrite = TRUE ) rc$spar_gf <-file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'spar','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'spar','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='spar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='spar',extension='parquet'), overwrite = TRUE ) rc$hpar_gf <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hpar','parquet'), - flepicommon::create_file_name(run_id,gf_prefix,slot,'hpar','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hpar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="", index=slot,type='hpar',extension='parquet'), overwrite = TRUE ) - #move files from global/intermediate/slot.block.run to global/intermediate/slot + #move files from global/intermediate/slot.block.run to global/intermediate/slot.block rc$seed_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'seed','csv'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'seed','csv') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='seed','csv') ) rc$init_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'init','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'init','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='init',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='init',extension='parquet') ) rc$seir_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'seir','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'seir','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='seir',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='seir',extension='parquet') ) rc$hosp_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hosp','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'hosp','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hosp',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hosp',extension='parquet') ) rc$llik_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'llik','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'llik','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='llik',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='llik',extension='parquet') ) rc$snpi_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'snpi','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'snpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='snpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='snpi',extension='parquet') ) rc$hnpi_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hnpi','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'hnpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hnpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hnpi',extension='parquet') ) rc$spar_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'spar','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'spar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='spar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='spar',extension='parquet') ) rc$hpar_block <- file.copy( - flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hpar','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'hpar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hpar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hpar',extension='parquet') ) } else { #move files from global/intermediate/slot.(block-1) to global/intermediate/slot.block rc$seed_prevblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1 ,'seed','csv'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'seed','csv') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1 ,type='seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='seed','csv') ) rc$init_prevblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1 ,'init','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'init','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1 ,type='init',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='init',extension='parquet') ) rc$seir_prevblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1 ,'seir','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'seir','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1 ,type='seir',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='seir',extension='parquet') ) rc$hosp_prevblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1 ,'hosp','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'hosp','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1 ,type='hosp',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hosp',extension='parquet') ) rc$llik_prevblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1,'llik','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'llik','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='llik',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='llik',extension='parquet') ) rc$snpi_prvblk <-file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1,'snpi','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'snpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='snpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='snpi',extension='parquet') ) rc$hnpi_prvblk <-file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1,'hnpi','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'hnpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='hnpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hnpi',extension='parquet') ) rc$spar_prvblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1,'spar','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'spar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='spar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='spar',extension='parquet') ) rc$hpar_prvblk <- file.copy( - flepicommon::create_file_name(run_id,global_block_prefix,block - 1,'hpar','parquet'), - flepicommon::create_file_name(run_id,global_block_prefix,block,'hpar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='hpar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hpar',extension='parquet') ) } @@ -404,147 +405,147 @@ perform_MCMC_step_copies_chimeric <- function(current_index, slot, block, run_id, - chimeric_local_prefix, - cf_prefix, - chimeric_block_prefix) { + chimeric_intermediate_filepath_suffix, + slotblock_filename_prefix, + slot_filename_prefix) { rc <- list() if(current_index != 0){ #move files from chimeric/intermediate/slot.block.run to chimeric/final/slot rc$seed_gf <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'seed','csv'), - flepicommon::create_file_name(run_id,cf_prefix,slot,'seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,'seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",index=slot,'seed','csv'), overwrite = TRUE ) # No chimeric SEIR or HOSP files, nor INIT file for now # rc$seir_gf <- file.copy( - # flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'seir','parquet'), - # flepicommon::create_file_name(run_id,cf_prefix,slot,'seir','parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_local_prefix,current_index,'seir',extension='parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",slot,'seir',extension='parquet'), # overwrite = TRUE # ) # # rc$hosp_gf <- file.copy( - # flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hosp','parquet'), - # flepicommon::create_file_name(run_id,cf_prefix,slot,'hosp','parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_local_prefix,current_index,'hosp',extension='parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",slot,'hosp',extension='parquet'), # overwrite = TRUE # ) rc$llik_gf <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'llik','parquet'), - flepicommon::create_file_name(run_id,cf_prefix,slot,'llik','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='llik',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",index=slot,type='llik',extension='parquet'), overwrite = TRUE ) rc$snpi_gf <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'snpi','parquet'), - flepicommon::create_file_name(run_id,cf_prefix,slot,'snpi','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='snpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",index=slot,type='snpi',extension='parquet'), overwrite = TRUE ) rc$hnpi_gf <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hnpi','parquet'), - flepicommon::create_file_name(run_id,cf_prefix,slot,'hnpi','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hnpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",index=slot,type='hnpi',extension='parquet'), overwrite = TRUE ) rc$spar_gf <-file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'spar','parquet'), - flepicommon::create_file_name(run_id,cf_prefix,slot,'spar','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='spar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",index=slot,type='spar',extension='parquet'), overwrite = TRUE ) rc$hpar_gf <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hpar','parquet'), - flepicommon::create_file_name(run_id,cf_prefix,slot,'hpar','parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hpar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix="global/final",filename_prefix="",index=slot,type='hpar',extension='parquet'), overwrite = TRUE ) #move files from chimeric/intermediate/slot.block.run to chimeric/intermediate/slot rc$seed_block <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'seed','csv'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'seed','csv') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='seed','csv') ) # no chimeric SEIR or HOSP files # rc$seir_block <- file.copy( - # flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'seir','parquet'), - # flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'seir','parquet') + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_local_prefix,current_index,'seir',extension='parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block,'seir',extension='parquet') # ) # # rc$hosp_block <- file.copy( - # flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hosp','parquet'), - # flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hosp','parquet') + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_local_prefix,current_index,'hosp',extension='parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block,'hosp',extension='parquet') # ) rc$llik_block <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'llik','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'llik','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='llik',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='llik',extension='parquet') ) rc$snpi_block <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'snpi','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'snpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='snpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='snpi',extension='parquet') ) rc$hnpi_block <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hnpi','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hnpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hnpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hnpi',extension='parquet') ) rc$spar_block <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'spar','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'spar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,current_index,type='spar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block,type='spar',extension='parquet') ) rc$hpar_block <- file.copy( - flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hpar','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hpar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix,index=current_index,type='hpar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hpar',extension='parquet') ) } else { #move files from chimeric/intermediate/slot.(block-1) to chimeric/intermediate/slot.block rc$seed_prevblk <- file.copy( - flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1 ,'seed','csv'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'seed','csv') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1 ,type='seed','csv'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='seed','csv') ) # Joseph: commented these as well # rc$seir_prevblk <- file.copy( - # flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1 ,'seir','parquet'), - # flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'seir','parquet') + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block - 1 ,'seir',extension='parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block,'seir',extension='parquet') # ) # rc$hosp_prevblk <- file.copy( - # flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1 ,'hosp','parquet'), - # flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hosp','parquet') + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block - 1 ,'hosp',extension='parquet'), + # flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,block,'hosp',extension='parquet') # ) rc$llik_prevblk <- file.copy( - flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1,'llik','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'llik','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='llik',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='llik',extension='parquet') ) rc$snpi_prvblk <-file.copy( - flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1,'snpi','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'snpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,'snpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,'snpi',extension='parquet') ) rc$hnpi_prvblk <-file.copy( - flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1,'hnpi','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hnpi','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='hnpi',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hnpi',extension='parquet') ) rc$spar_prvblk <- file.copy( - flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1,'spar','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'spar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='spar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='spar',extension='parquet') ) rc$hpar_prvblk <- file.copy( - flepicommon::create_file_name(run_id,chimeric_block_prefix,block - 1,'hpar','parquet'), - flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hpar','parquet') + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block - 1,type='hpar',extension='parquet'), + flepicommon::create_file_name(run_id=run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix,index=block,type='hpar',extension='parquet') ) } @@ -558,7 +559,8 @@ perform_MCMC_step_copies_chimeric <- function(current_index, create_filename_list <- function( run_id, prefix, - suffix, + filepath_suffix, + filename_prefix, index, types = c("seed", "init", "seir", "snpi", "hnpi", "spar", "hosp", "hpar", "llik"), extensions = c("csv", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet")) { @@ -570,7 +572,7 @@ create_filename_list <- function( x=types, y=extensions, function(x,y){ - flepicommon::create_file_name(run_id = run_id, prefix = prefix, suffix = suffix, index = index, type = x, extension = y, create_directory = TRUE) + flepicommon::create_file_name(run_id = run_id, prefix = prefix, filepath_suffix = filepath_suffix, filename_prefix = filename_prefix, index = index, type = x, extension = y, create_directory = TRUE) } ) names(rc) <- paste(names(rc),"filename",sep='_') @@ -589,8 +591,10 @@ create_filename_list <- function( initialize_mcmc_first_block <- function( run_id, block, - global_prefix, - chimeric_prefix, + setup_prefix, + global_intermediate_filepath_suffix, + chimeric_intermediate_filepath_suffix, + filename_prefix, gempyor_inference_runner, likelihood_calculation_function, is_resume = FALSE) { @@ -601,9 +605,11 @@ initialize_mcmc_first_block <- function( chimeric_types <- c("seed", "init", "snpi", "hnpi", "spar", "hpar", "llik") chimeric_extensions <- c("csv", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet") non_llik_types <- paste(c("seed", "init", "seir", "snpi", "hnpi", "spar", "hosp", "hpar"), "filename", sep = "_") - - global_files <- create_filename_list(run_id, global_prefix, block - 1, global_types, global_extensions) # makes file names of the form variable/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/global/intermediate/slot.(block-1).run_ID.variable.ext - chimeric_files <- create_filename_list(run_id, chimeric_prefix, block - 1, chimeric_types, chimeric_extensions) # makes file names of the form variable/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/chimeric/intermediate/slot.(block-1).run_ID.variable.ext + # create_filename_list(run_id, prefix, suffix, index, types = c("seed", "init", "seir", "snpi", "hnpi", "spar", "hosp", "hpar", "llik"), extensions = c("csv", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet", "parquet")) + # makes file names of the form variable/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/global/intermediate/slot.(block-1).run_ID.variable.ext + global_files <- create_filename_list(run_id=run_id, prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix = filename_prefix, index=block - 1, types=global_types, extension=global_extensions) + # makes file names of the form variable/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/chimeric/intermediate/slot.(block-1).run_ID.variable.ext + chimeric_files <- create_filename_list(run_id=run_id, prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix = filename_prefix, index=block - 1, types=chimeric_types, extension=chimeric_extensions) global_check <- sapply(global_files, file.exists) chimeric_check <- sapply(chimeric_files, file.exists) diff --git a/flepimop/R_packages/inference/tests/testthat/test-perform_MCMC_step_copies.R b/flepimop/R_packages/inference/tests/testthat/test-perform_MCMC_step_copies.R index 1d21ce8a7..61a117188 100644 --- a/flepimop/R_packages/inference/tests/testthat/test-perform_MCMC_step_copies.R +++ b/flepimop/R_packages/inference/tests/testthat/test-perform_MCMC_step_copies.R @@ -20,15 +20,15 @@ test_that("MCMC step copies (global) are correctly performed when we are not at dir.create("MCMC_step_copy_test") setwd("MCMC_step_copy_test") ##get file names - seed_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'seed','csv') - init_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'init','parquet') - seir_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'seir','parquet') - hosp_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hosp','parquet') - llik_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'llik','parquet') - snpi_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'snpi','parquet') - spar_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'spar','parquet') - hnpi_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hnpi','parquet') - hpar_src <- flepicommon::create_file_name(run_id,global_local_prefix,current_index,'hpar','parquet') + seed_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'seed','csv') + init_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'init','parquet') + seir_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'seir','parquet') + hosp_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'hosp','parquet') + llik_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'llik','parquet') + snpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'snpi','parquet') + spar_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'spar','parquet') + hnpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'hnpi','parquet') + hpar_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_local_prefix,current_index,'hpar','parquet') @@ -44,7 +44,7 @@ test_that("MCMC step copies (global) are correctly performed when we are not at arrow::write_parquet(data.frame(file="hpar"), hpar_src) ##print(hosp_src) - ##print(flepicommon::create_file_name(run_id,gf_prefix,slot,'hosp','parquet')) + ##print(flepicommon::create_file_name(run_id=run_id, prefix=gf_prefix,slot,'hosp','parquet')) res <- perform_MCMC_step_copies_global(current_index, slot, @@ -82,15 +82,15 @@ test_that("MCMC step copies (global) are correctly performed when we are at the dir.create("MCMC_step_copy_test") setwd("MCMC_step_copy_test") ##get file names - seed_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'seed','csv') - init_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'init','parquet') - seir_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'seir','parquet') - hosp_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'hosp','parquet') - llik_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'llik','parquet') - snpi_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'snpi','parquet') - spar_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'spar','parquet') - hnpi_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'hnpi','parquet') - hpar_src <- flepicommon::create_file_name(run_id,global_block_prefix,block-1,'hpar','parquet') + seed_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'seed','csv') + init_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'init','parquet') + seir_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'seir','parquet') + hosp_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'hosp','parquet') + llik_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'llik','parquet') + snpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'snpi','parquet') + spar_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'spar','parquet') + hnpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'hnpi','parquet') + hpar_src <- flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block-1,'hpar','parquet') ##create the copy from files readr::write_csv(data.frame(file="seed"), seed_src) @@ -104,7 +104,7 @@ test_that("MCMC step copies (global) are correctly performed when we are at the arrow::write_parquet(data.frame(file="hpar"), hpar_src) print(hosp_src) - print(flepicommon::create_file_name(run_id,global_block_prefix,block,'hosp','parquet')) + print(flepicommon::create_file_name(run_id=run_id, prefix=global_block_prefix,block,'hosp','parquet')) res <- perform_MCMC_step_copies_global(current_index, slot, @@ -142,14 +142,14 @@ test_that("MCMC step copies (chimeric) are correctly performed when we are not a dir.create("MCMC_step_copy_test") setwd("MCMC_step_copy_test") ##get file names - seed_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'seed','csv') - seir_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'seir','parquet') - hosp_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hosp','parquet') - llik_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'llik','parquet') - snpi_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'snpi','parquet') - spar_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'spar','parquet') - hnpi_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hnpi','parquet') - hpar_src <- flepicommon::create_file_name(run_id,chimeric_local_prefix,current_index,'hpar','parquet') + seed_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'seed','csv') + seir_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'seir','parquet') + hosp_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'hosp','parquet') + llik_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'llik','parquet') + snpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'snpi','parquet') + spar_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'spar','parquet') + hnpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'hnpi','parquet') + hpar_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_local_prefix,current_index,'hpar','parquet') @@ -164,7 +164,7 @@ test_that("MCMC step copies (chimeric) are correctly performed when we are not a arrow::write_parquet(data.frame(file="hpar"), hpar_src) ##print(hosp_src) - ##print(flepicommon::create_file_name(run_id,cf_prefix,slot,'hosp','parquet')) + ##print(flepicommon::create_file_name(run_id=run_id, prefix=cf_prefix,slot,'hosp','parquet')) res <- perform_MCMC_step_copies_chimeric(current_index, slot, @@ -203,14 +203,14 @@ test_that("MCMC step copies (chimeric) are correctly performed when we are at th dir.create("MCMC_step_copy_test") setwd("MCMC_step_copy_test") ##get file names - seed_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'seed','csv') - seir_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'seir','parquet') - hosp_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'hosp','parquet') - llik_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'llik','parquet') - snpi_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'snpi','parquet') - spar_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'spar','parquet') - hnpi_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'hnpi','parquet') - hpar_src <- flepicommon::create_file_name(run_id,chimeric_block_prefix,block-1,'hpar','parquet') + seed_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'seed','csv') + seir_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'seir','parquet') + hosp_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'hosp','parquet') + llik_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'llik','parquet') + snpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'snpi','parquet') + spar_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'spar','parquet') + hnpi_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'hnpi','parquet') + hpar_src <- flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block-1,'hpar','parquet') @@ -225,7 +225,7 @@ test_that("MCMC step copies (chimeric) are correctly performed when we are at th arrow::write_parquet(data.frame(file="hpar"), hpar_src) print(hosp_src) - print(flepicommon::create_file_name(run_id,chimeric_block_prefix,block,'hosp','parquet')) + print(flepicommon::create_file_name(run_id=run_id, prefix=chimeric_block_prefix,block,'hosp','parquet')) res <- perform_MCMC_step_copies_chimeric(current_index, slot, diff --git a/flepimop/gempyor_pkg/src/gempyor/file_paths.py b/flepimop/gempyor_pkg/src/gempyor/file_paths.py index ce4414351..f67340139 100644 --- a/flepimop/gempyor_pkg/src/gempyor/file_paths.py +++ b/flepimop/gempyor_pkg/src/gempyor/file_paths.py @@ -14,9 +14,7 @@ def create_file_name_without_extension(run_id, prefix, index, ftype, inference_f os.makedirs(create_dir_name(run_id, prefix, ftype, inference_filepath_suffix, inference_filename_prefix), exist_ok=True) filename = pathlib.Path("model_output", prefix, ftype, inference_filepath_suffix, f"{inference_filename_prefix}{index:>09}.{run_id}.{ftype}") - print(f" making {filename}") # old: "model_output/%s/%s%09d.%s.%s" % (ftype, prefix, index, run_id, ftype) - return filename def run_id(): diff --git a/flepimop/main_scripts/inference_slot.R b/flepimop/main_scripts/inference_slot.R index 58fcf6e85..385c171a1 100644 --- a/flepimop/main_scripts/inference_slot.R +++ b/flepimop/main_scripts/inference_slot.R @@ -384,34 +384,28 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { ## create_prefix(prefix="USA/", "inference", "med", "2022.03.04.10.18.42.CET", sep='/', trailing_separator='.') ## would be "USA/inference/med/2022.03.04.10.18.42.CET." - setup_prefix <- flepicommon::create_setup_prefix(config$setup_name, - seir_modifiers_scenario, outcome_modifiers_scenario, - trailing_separator='') - inference_prefix <- file.path(setup_prefix, opt$run_id) - + + #setup_prefix <- flepicommon::create_setup_prefix(config$setup_name, + # seir_modifiers_scenario, outcome_modifiers_scenario, + # trailing_separator='') + #inference_prefix <- file.path(setup_prefix, opt$run_id) # gf_prefix <- flepicommon::create_prefix(prefix=slot_prefix,'global','final',sep='/',trailing_separator='/') # cf_prefix <- flepicommon::create_prefix(prefix=slot_prefix,'chimeric','final',sep='/',trailing_separator='/') # ci_prefix <- flepicommon::create_prefix(prefix=slot_prefix,'chimeric','intermediate',sep='/',trailing_separator='/') # gi_prefix <- flepicommon::create_prefix(prefix=slot_prefix,'global','intermediate',sep='/',trailing_separator='/') - gf_suffix <- flepicommon::create_prefix(prefix="",'global','final',sep='/',trailing_separator='') - cf_suffix <- flepicommon::create_prefix(prefix="",'chimeric','final',sep='/',trailing_separator='') - ci_suffix <- flepicommon::create_prefix(prefix="",'chimeric','intermediate',sep='/',trailing_separator='') - gi_suffix <- flepicommon::create_prefix(prefix="",'global','intermediate',sep='/',trailing_separator='') + chimeric_intermediate_filepath_suffix <- flepicommon::create_prefix(prefix="",'chimeric','intermediate',sep='/',trailing_separator='') + global_intermediate_filepath_suffix <- flepicommon::create_prefix(prefix="",'global','intermediate',sep='/',trailing_separator='') - filename_prefix <- flepicommon::create_prefix(prefix="", slot=list(opt$this_slot,"%09d"), opt$run_id, sep='.', trailing_separator='') + #filename_prefix <- flepicommon::create_prefix(prefix="", slot=list(opt$this_slot,"%09d"), sep='.', trailing_separator='') # chimeric_block_prefix <- flepicommon::create_prefix(prefix=ci_prefix, slot=list(opt$this_slot,"%09d"), sep='.', trailing_separator='.') # chimeric_local_prefix <- flepicommon::create_prefix(prefix=chimeric_block_prefix, slot=list(opt$this_block,"%09d"), sep='.', trailing_separator='.') # global_block_prefix <- flepicommon::create_prefix(prefix=gi_prefix, slot=list(opt$this_slot,"%09d"), sep='.', trailing_separator='.') # global_local_prefix <- flepicommon::create_prefix(prefix=global_block_prefix, slot=list(opt$this_block,"%09d"), sep='.', trailing_separator='.') - - - global_local_prefix <- flepicommon::create_prefix(prefix=global_block_prefix, slot=list(opt$this_block,"%09d"), sep='.', trailing_separator='.') - - - print("prefixes created successfully.") + # global_local_prefix <- flepicommon::create_prefix(prefix=global_block_prefix, slot=list(opt$this_block,"%09d"), sep='.', trailing_separator='.') + # TODO: WHAT ABOUT BLOCS ? #swap scenarios for py_none() to pass to Gempyor @@ -422,37 +416,61 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { outcome_modifiers_scenario <- reticulate::py_none() } + slotblock_filename_prefix <- flepicommon::create_prefix(slot=list(opt$this_slot,"%09d"), block=list(opt$this_block,"%09d"), sep='.', trailing_separator='.') + + slot_filename_prefix <- flepicommon::create_prefix(slot=list(opt$this_slot,"%09d"), sep='.', trailing_separator='.') + ### Set up initial conditions ---------- ## python configuration: build simulator model initialized with compartment and all. - gempyor_inference_runner <- gempyor$GempyorSimulator( - config_path=opt$config, - seir_modifiers_scenario=seir_modifiers_scenario, - outcome_modifiers_scenario=outcome_modifiers_scenario, - stoch_traj_flag=opt$stoch_traj_flag, - initialize=TRUE, # Shall we pre-compute now things that are not pertubed by inference - run_id=opt$run_id, - prefix = inference_prefix, - suffix = gi_suffix, - index = flepicommon::create_prefix(slot=list(opt$this_slot,"%09d"), sep='.', trailing_separator='') - ) + tryCatch({ + gempyor_inference_runner <- gempyor$GempyorSimulator( + config_path=opt$config, + seir_modifiers_scenario=seir_modifiers_scenario, + outcome_modifiers_scenario=outcome_modifiers_scenario, + stoch_traj_flag=opt$stoch_traj_flag, + initialize=TRUE, # Shall we pre-compute now things that are not pertubed by inference + run_id=opt$run_id, + prefix=reticulate::py_none(), # we let gempyor create setup prefix + inference_filepath_suffix=global_intermediate_filepath_suffix, + inference_filename_prefix=slotblock_filename_prefix, + #index = + ) + }, error = function(e) { + print("GempyorSimulator failed to run (call on l. 538 of inference_slot.R).") + print("Here is all the debug information I could find:") + for(m in reticulate::py_last_error()) cat(m) + stop("GempyorSimulator failed to run... stopping") + }) + + + setup_prefix <- gempyor_inference_runner$modinf$get_setup_name() print("gempyor_inference_runner created successfully.") ## Using the prefixes, create standardized files of each type (e.g., seir) of the form ## {variable}/{prefix}{block-1}.{run_id}.{variable}.{ext} ## N.B.: prefix should end in "{slot}." - first_global_files <- inference::create_filename_list(opt$run_id, inference_prefix, gi_suffix, opt$this_block - 1) - first_chimeric_files <- inference::create_filename_list(opt$run_id, inference_prefix, ci_suffix, opt$this_block - 1) + first_global_files <- inference::create_filename_list(run_id=opt$run_id, + prefix=setup_prefix, + filepath_suffix=global_intermediate_filepath_suffix, + filename_prefix=slotblock_filename_prefix, + index=opt$this_block - 1) + first_chimeric_files <- inference::create_filename_list(run_id=opt$run_id, + prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, + filename_prefix=slotblock_filename_prefix, + index=opt$this_block - 1) ## print("RUNNING: initialization of first block") ## Functions within this function save variables to files of the form variable/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/global/intermediate/slot.(block-1),run_id.variable.ext and also copied into the /chimeric/ version, which are referenced by first_global_files and first_chimeric_files inference::initialize_mcmc_first_block( - opt$run_id, - opt$this_block, - global_block_prefix, - chimeric_block_prefix, - gempyor_inference_runner, - likelihood_calculation_fun, + run_id=opt$run_id, + block=opt$this_block, + setup_prefix=setup_prefix, + filename_prefix=slotblock_filename_prefix, + global_intermediate_filepath_suffix=global_intermediate_filepath_suffix, + chimeric_intermediate_filepath_suffix=chimeric_intermediate_filepath_suffix, + gempyor_inference_runner=gempyor_inference_runner, + likelihood_calculation_function=likelihood_calculation_fun, is_resume = opt[['is-resume']] ) print("First MCMC block initialized successfully.") @@ -469,13 +487,15 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { initial_seeding$amount <- as.integer(round(initial_seeding$amount)) } # } - initial_init <- arrow::read_parquet(first_chimeric_files[['init_filename']]) + initial_snpi <- arrow::read_parquet(first_chimeric_files[['snpi_filename']]) initial_hnpi <- arrow::read_parquet(first_chimeric_files[['hnpi_filename']]) initial_spar <- arrow::read_parquet(first_chimeric_files[['spar_filename']]) initial_hpar <- arrow::read_parquet(first_chimeric_files[['hpar_filename']]) if (!is.null(config$initial_conditions)){ initial_init <- arrow::read_parquet(first_global_files[['init_filename']]) + initial_init <- arrow::read_parquet(first_chimeric_files[['init_filename']]) + } chimeric_likelihood_data <- arrow::read_parquet(first_chimeric_files[['llik_filename']]) global_likelihood_data <- arrow::read_parquet(first_global_files[['llik_filename']]) @@ -520,8 +540,8 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { ## Using the prefixes, create standardized files of each type (e.g., seir) of the form ## {variable}/{prefix}{block-1}.{run_id}.{variable}.{ext} ## N.B.: prefix should end in "{block}." - this_global_files <- inference::create_filename_list(opt$run_id, inference_prefix, gi_suffix, this_index) - this_chimeric_files <- inference::create_filename_list(opt$run_id, inference_prefix, ci_suffix, this_index) + this_global_files <- inference::create_filename_list(run_id=opt$run_id, prefix = setup_prefix, filepath_suffix=global_intermediate_filepath_suffix,filename_prefix=slotblock_filename_prefix, index=this_index) + this_chimeric_files <- inference::create_filename_list(run_id=opt$run_id, prefix = setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix, index=this_index) ### Do perturbations from accepted parameters to get proposed parameters ---- @@ -538,24 +558,23 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { } if (infer_initial_conditions) { proposed_init <- inference::perturb_init(initial_init, config$initial_conditions$perturbation) - } else { - proposed_init <- initial_init - } + } #else { + #proposed_init <- initial_init + #} if (!is.null(config$seir_modifiers$modifiers)){ proposed_snpi <- inference::perturb_snpi(initial_snpi, config$seir_modifiers$modifiers) } if (!is.null(config$outcome_modifiers$modifiers)){ proposed_hnpi <- inference::perturb_hnpi(initial_hnpi, config$outcome_modifiers$modifiers) # NOTE: no scenarios possible right now - } + } proposed_spar <- initial_spar proposed_hpar <- inference::perturb_hpar(initial_hpar, config$outcomes$outcomes) # NOTE: no scenarios possible right now - if (!is.null(config$initial_conditions)){ - proposed_init <- initial_init - } + #if (!is.null(config$initial_conditions)){ + # proposed_init <- initial_init + #} # since the first iteration is accepted by default, we don't perturb it if ((opt$this_block == 1) && (current_index == 0)) { - proposed_init <- initial_init proposed_snpi <- initial_snpi proposed_hnpi <- initial_hnpi proposed_spar <- initial_spar @@ -578,7 +597,6 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { write.csv(proposed_seeding, this_global_files[['seed_filename']], row.names = FALSE) # } - arrow::write_parquet(proposed_init,this_global_files[['init_filename']]) arrow::write_parquet(proposed_snpi,this_global_files[['snpi_filename']]) arrow::write_parquet(proposed_hnpi,this_global_files[['hnpi_filename']]) arrow::write_parquet(proposed_spar,this_global_files[['spar_filename']]) @@ -586,9 +604,6 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { if (!is.null(config$initial_conditions)){ arrow::write_parquet(proposed_init,this_global_files[['init_filename']]) } - - ## Update the prefix - gempyor_inference_runner$update_prefix(new_prefix=global_local_prefix) ## Run the simulator tryCatch({ gempyor_inference_runner$one_simulation( @@ -663,8 +678,8 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { print("by default because it's the first iteration of a block 1") } - old_global_files <- inference::create_filename_list(opt$run_id, inference_prefix, gi_suffix, current_index) - old_chimeric_files <- inference::create_filename_list(opt$run_id, inference_prefix, ci_suffix, current_index) + old_global_files <- inference::create_filename_list(run_id=opt$run_id, prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix, index=current_index) + old_chimeric_files <- inference::create_filename_list(run_id=opt$run_id, prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix, index=current_index) #IMPORTANT: This is the index of the most recent globally accepted parameters current_index <- this_index @@ -711,6 +726,11 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { if (!reset_chimeric_files) { ## Chimeric likelihood acceptance or rejection decisions (one round) ----- # "Chimeric" means GeoID-specific + if (is.null(config$initial_conditions)){ + initial_init <- NULL + proposed_init <- NULL + } + seeding_npis_list <- inference::accept_reject_new_seeding_npis( init_orig = initial_init, @@ -729,7 +749,9 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { # Update accepted parameters to start next simulation - initial_init <- seeding_npis_list$init + if (!is.null(config$initial_conditions)){ + initial_init <- seeding_npis_list$init + } initial_seeding <- seeding_npis_list$seeding initial_snpi <- seeding_npis_list$snpi initial_hnpi <- seeding_npis_list$hnpi @@ -737,7 +759,9 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { chimeric_likelihood_data <- seeding_npis_list$ll } else { print("Resetting chimeric files to global") - initial_init <- proposed_init + if (!is.null(config$initial_conditions)){ + initial_init <- proposed_init + } initial_seeding <- proposed_seeding initial_snpi <- proposed_snpi initial_hnpi <- proposed_hnpi @@ -754,7 +778,9 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { ## Write accepted parameters to file # writes to file of the form variable/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/chimeric/intermediate/slot.block.iter.run_id.variable.ext write.csv(initial_seeding,this_chimeric_files[['seed_filename']], row.names = FALSE) - arrow::write_parquet(initial_init,this_chimeric_files[['init_filename']]) + if (!is.null(config$initial_conditions)){ + arrow::write_parquet(initial_init,this_chimeric_files[['init_filename']]) + } arrow::write_parquet(initial_snpi,this_chimeric_files[['snpi_filename']]) arrow::write_parquet(initial_hnpi,this_chimeric_files[['hnpi_filename']]) arrow::write_parquet(initial_spar,this_chimeric_files[['spar_filename']]) @@ -793,10 +819,8 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { unit = c("Gb", "Gb"), .before = 1) - this_global_memprofile <- inference::create_filename_list(opt$run_id, - inference_prefix, gi_suffix, - this_index, - types = "memprof", extensions = "parquet") + this_global_memprofile <- inference::create_filename_list(run_id=opt$run_id, + prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix, index=this_index,types = "memprof", extensions = "parquet") arrow::write_parquet(curr_obj_sizes, this_global_memprofile[['memprof_filename']]) rm(curr_obj_sizes) } @@ -816,32 +840,32 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { #####Do MCMC end copy. Fail if unsucessfull # moves the most recently globally accepted parameter values from global/intermediate file to global/final - cpy_res_global <- inference::perform_MCMC_step_copies_global(current_index, - opt$this_slot, - opt$this_block, - opt$run_id, - global_local_prefix, - gf_prefix, - global_block_prefix) - if (!prod(unlist(cpy_res_global))) {stop("File copy failed:", paste(unlist(cpy_res_global),paste(names(cpy_res_global),"|")))} + cpy_res_global <- inference::perform_MCMC_step_copies_global(current_index=current_index, + slot=opt$this_slot, + block=opt$this_block, + run_id=opt$run_id, + global_intermediate_filepath_suffix= global_intermediate_filepath_suffix, + slotblock_filename_prefix=slotblock_filename_prefix, + slot_filename_prefix=slot_filename_prefix) + #if (!prod(unlist(cpy_res_global))) {stop("File copy failed:", paste(unlist(cpy_res_global),paste(names(cpy_res_global),"|")))} # moves the most recently chimeric accepted parameter values from chimeric/intermediate file to chimeric/final - cpy_res_chimeric <- inference::perform_MCMC_step_copies_chimeric(this_index, - opt$this_slot, - opt$this_block, - opt$run_id, - chimeric_local_prefix, - cf_prefix, - chimeric_block_prefix) - if (!prod(unlist(cpy_res_chimeric))) {stop("File copy failed:", paste(unlist(cpy_res_chimeric),paste(names(cpy_res_chimeric),"|")))} + cpy_res_chimeric <- inference::perform_MCMC_step_copies_chimeric(current_index=this_index, + slot=opt$this_slot, + block=opt$this_block, + run_id=opt$run_id, + chimeric_intermediate_filepath_suffix=chimeric_intermediate_filepath_suffix, + slotblock_filename_prefix=slotblock_filename_prefix, + slot_filename_prefix=slot_filename_prefix) + #if (!prod(unlist(cpy_res_chimeric))) {stop("File copy failed:", paste(unlist(cpy_res_chimeric),paste(names(cpy_res_chimeric),"|")))} #####Write currently accepted files to disk #files of the form variables/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/chimeric/intermediate/slot.block.run_id.variable.parquet - output_chimeric_files <- inference::create_filename_list(opt$run_id, inference_prefix, ci_suffix, , opt$this_block) + output_chimeric_files <- inference::create_filename_list(run_id=opt$run_id,prefix=setup_prefix, filepath_suffix=chimeric_intermediate_filepath_suffix, filename_prefix=slot_filename_prefix, index=opt$this_block) #files of the form variables/name/seir_modifiers_scenario/outcome_modifiers_scenario/run_id/global/intermediate/slot.block.run_id.variable.parquet - output_global_files <- inference::create_filename_list(opt$run_id, inference_prefix, gi_suffix, opt$this_block) + output_global_files <- inference::create_filename_list(run_id=opt$run_id,prefix=setup_prefix,filepath_suffix=global_intermediate_filepath_suffix,filename_prefix=slot_filename_prefix, index=opt$this_block) warning("Chimeric hosp and seir files not yet supported, just using the most recently generated file of each type") - this_index_global_files <- inference::create_filename_list(opt$run_id, inference_prefix, gi_suffix, this_index) + this_index_global_files <- inference::create_filename_list(run_id=opt$run_id,prefix=setup_prefix, filepath_suffix=global_intermediate_filepath_suffix, filename_prefix=slotblock_filename_prefix, index=this_index) file.copy(this_index_global_files[['hosp_filename']],output_chimeric_files[['hosp_filename']]) file.copy(this_index_global_files[['seir_filename']],output_chimeric_files[['seir_filename']]) } From 1e5a988f6e6c5736d19e219fbc2a69222eecd03e Mon Sep 17 00:00:00 2001 From: Joseph Lemaitre Date: Mon, 16 Oct 2023 20:34:04 +0200 Subject: [PATCH 3/5] late fixes --- flepimop/gempyor_pkg/src/gempyor/simulate.py | 2 +- flepimop/main_scripts/inference_slot.R | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 5cc0c5680..ee6297aeb 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -304,7 +304,7 @@ def simulate( outcome_modifiers_scenarios = None if config["outcomes"].exists() and config["outcome_modifiers"].exists(): if config["outcome_modifiers"]["scenarios"].exists(): - outcome_modifiers_scenarios = config["outcomes"]["scenarios"].as_str_seq() + outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"].as_str_seq() outcome_modifiers_scenarios = as_list(outcome_modifiers_scenarios) seir_modifiers_scenarios = as_list(seir_modifiers_scenarios) diff --git a/flepimop/main_scripts/inference_slot.R b/flepimop/main_scripts/inference_slot.R index 385c171a1..599d769bb 100644 --- a/flepimop/main_scripts/inference_slot.R +++ b/flepimop/main_scripts/inference_slot.R @@ -564,8 +564,10 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { if (!is.null(config$seir_modifiers$modifiers)){ proposed_snpi <- inference::perturb_snpi(initial_snpi, config$seir_modifiers$modifiers) } + # TODO we need a hnpi for inference + proposed_hnpi <- inference::perturb_hnpi(initial_hnpi, config$outcome_modifiers$modifiers) if (!is.null(config$outcome_modifiers$modifiers)){ - proposed_hnpi <- inference::perturb_hnpi(initial_hnpi, config$outcome_modifiers$modifiers) # NOTE: no scenarios possible right now + proposed_hnpi <- inference::perturb_hnpi(initial_hnpi, config$outcome_modifiers$modifiers)# NOTE: no scenarios possible right now } proposed_spar <- initial_spar proposed_hpar <- inference::perturb_hpar(initial_hpar, config$outcomes$outcomes) # NOTE: no scenarios possible right now From a99ad13eab074b86ed39eccfe25e7f73115347f2 Mon Sep 17 00:00:00 2001 From: Joseph Lemaitre Date: Mon, 16 Oct 2023 22:33:00 +0200 Subject: [PATCH 4/5] fix trailing comma not supported on HPC --- flepimop/main_scripts/inference_slot.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/main_scripts/inference_slot.R b/flepimop/main_scripts/inference_slot.R index 599d769bb..cbe0a167c 100644 --- a/flepimop/main_scripts/inference_slot.R +++ b/flepimop/main_scripts/inference_slot.R @@ -433,7 +433,7 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { run_id=opt$run_id, prefix=reticulate::py_none(), # we let gempyor create setup prefix inference_filepath_suffix=global_intermediate_filepath_suffix, - inference_filename_prefix=slotblock_filename_prefix, + inference_filename_prefix=slotblock_filename_prefix #index = ) }, error = function(e) { From 9ba1854703f18c93fd0e315ec32bd944ad554210 Mon Sep 17 00:00:00 2001 From: Joseph Lemaitre Date: Mon, 16 Oct 2023 22:41:53 +0200 Subject: [PATCH 5/5] fix --- batch/inference_job_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/inference_job_launcher.py b/batch/inference_job_launcher.py index d26c89be8..c6cd39e71 100755 --- a/batch/inference_job_launcher.py +++ b/batch/inference_job_launcher.py @@ -396,7 +396,7 @@ def launch_batch( outcome_modifiers_scenarios = None if config["seir_modifiers"]["scenarios"].exists(): seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"] - if config["outcome_modifiers"]["scenarios"]: + if config["outcome_modifiers"]["scenarios"].exists(): outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"] handler.launch(job_name, config_file, seir_modifiers_scenarios, outcome_modifiers_scenarios)