From 3a4f797ac0965bd73b840ef150a67e9600c0d138 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 8 Oct 2024 16:00:51 -0400 Subject: [PATCH 01/53] initial demo for patching configs capability --- examples/tutorials/config_sample_2pop.yml | 1 + .../tutorials/config_sample_2pop_outcomes.yml | 47 ---------- flepimop/gempyor_pkg/setup.cfg | 2 + flepimop/gempyor_pkg/src/gempyor/cli.py | 85 +++++++++++++++---- .../gempyor_pkg/src/gempyor/compartments.py | 41 --------- .../src/gempyor/config_validator.py | 8 +- 6 files changed, 76 insertions(+), 108 deletions(-) diff --git a/examples/tutorials/config_sample_2pop.yml b/examples/tutorials/config_sample_2pop.yml index 75593cb23..6f65dd7d3 100644 --- a/examples/tutorials/config_sample_2pop.yml +++ b/examples/tutorials/config_sample_2pop.yml @@ -12,6 +12,7 @@ initial_conditions: method: SetInitialConditions initial_conditions_file: model_input/ic_2pop.csv allow_missing_compartments: TRUE + allow_missing_compartments: TRUE compartments: infection_stage: ["S", "E", "I", "R"] diff --git a/examples/tutorials/config_sample_2pop_outcomes.yml b/examples/tutorials/config_sample_2pop_outcomes.yml index 0044a2e96..44484403c 100644 --- a/examples/tutorials/config_sample_2pop_outcomes.yml +++ b/examples/tutorials/config_sample_2pop_outcomes.yml @@ -1,50 +1,3 @@ -name: sample_2pop -setup_name: minimal -start_date: 2020-02-01 -end_date: 2020-05-31 -nslots: 1 - -subpop_setup: - geodata: model_input/geodata_sample_2pop.csv - mobility: model_input/mobility_sample_2pop.csv - -initial_conditions: - method: SetInitialConditions - initial_conditions_file: model_input/ic_2pop.csv - allow_missing_subpops: TRUE - allow_missing_compartments: TRUE - -compartments: - infection_stage: ["S", "E", "I", "R"] - -seir: - integration: - method: rk4 - dt: 1 - parameters: - sigma: - value: 1 / 4 - gamma: - value: 1 / 5 - Ro: - value: 2.5 - transitions: - - source: ["S"] - destination: ["E"] - rate: ["Ro * gamma"] - proportional_to: [["S"],["I"]] - proportion_exponent: ["1","1"] - - source: ["E"] - destination: ["I"] - rate: ["sigma"] - proportional_to: ["E"] - proportion_exponent: ["1"] - - source: ["I"] - destination: ["R"] - rate: ["gamma"] - proportional_to: ["I"] - proportion_exponent: ["1"] - outcomes: method: delayframe outcomes: diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index e5fb0902a..cf90a2170 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -35,6 +35,7 @@ install_requires = dask scipy graphviz + pydantic # see https://stackoverflow.com/questions/58826164/dependencies-requirements-for-setting-up-testing-and-installing-a-python-lib # installed for pip install -e ".[test]" @@ -53,6 +54,7 @@ console_scripts = gempyor-seir = gempyor.simulate_seir:simulate gempyor-simulate = gempyor.simulate:simulate flepimop-calibrate = gempyor.calibrate:calibrate + flepimop-validate = gempyor.config_validator:validate [options.packages.find] where = src diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 910e78f04..9cabcd7a3 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,26 +1,81 @@ import click -from .compartments import compartments +import sys +# from .cli_shared import argument_config_files, parse_config_files, config +# TODO ...is this importing a global config object? do not like from gempyor.utils import config +from gempyor.compartments import Compartments - -@click.group() -@click.option( - "-c", - "--config", - "config_filepath", - envvar=["CONFIG_PATH"], - type=click.Path(exists=True), - help="configuration file for this simulation", +argument_config_files = click.argument( + "config_files", + nargs = -1, + type = click.Path(exists = True), + required = True ) -def cli(config_filepath): - print(config_filepath) + +def parse_config_files(config_files) -> None: config.clear() - config.read(user=False) - config.set_file(config_filepath) + for config_file in reversed(config_files): + config.set_file(config_file) + +@click.group() +def cli(): + """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" + pass +@cli.command() +@argument_config_files +def patch(config_files): + """Merge configuration files""" + parse_config_files(config_files) + print(config.dump()) -cli.add_command(compartments) +@click.group() +def compartments(): + """Commands for working with FlepiMoP compartments""" + pass + +# TODO: CLI arguments +@compartments.command() +@argument_config_files +def plot(config_files): + """Plot compartments""" + parse_config_files(config_files) + assert config["compartments"].exists() + assert config["seir"].exists() + comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) + + # TODO: this should be a command like build compartments. + ( + unique_strings, + transition_array, + proportion_array, + proportion_info, + ) = comp.get_transition_array() + + comp.plot(output_file="transition_graph", source_filters=[], destination_filters=[]) + + print("wrote file transition_graph") + + +@compartments.command() +@argument_config_files +def export(config_files): + """Export compartments""" + parse_config_files(config_files) + assert config["compartments"].exists() + assert config["seir"].exists() + comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) + ( + unique_strings, + transition_array, + proportion_array, + proportion_info, + ) = comp.get_transition_array() + comp.toFile("compartments_file.csv", "transitions_file.csv", write_parquet=False) + print("wrote files 'compartments_file.csv', 'transitions_file.csv' ") + +cli.add_command(compartments) if __name__ == "__main__": cli() diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index ec87cf7e5..8079d5798 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -2,9 +2,7 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import click from .utils import config, Timer, as_list -from . import file_paths from functools import reduce import logging @@ -744,42 +742,3 @@ def list_recursive_convert_to_string(thing): return [list_recursive_convert_to_string(x) for x in thing] return str(thing) - -@click.group() -def compartments(): - pass - - -# TODO: CLI arguments -@compartments.command() -def plot(): - assert config["compartments"].exists() - assert config["seir"].exists() - comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) - - # TODO: this should be a command like build compartments. - ( - unique_strings, - transition_array, - proportion_array, - proportion_info, - ) = comp.get_transition_array() - - comp.plot(output_file="transition_graph", source_filters=[], destination_filters=[]) - - print("wrote file transition_graph") - - -@compartments.command() -def export(): - assert config["compartments"].exists() - assert config["seir"].exists() - comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) - ( - unique_strings, - transition_array, - proportion_array, - proportion_info, - ) = comp.get_transition_array() - comp.toFile("compartments_file.csv", "transitions_file.csv", write_parquet=False) - print("wrote files 'compartments_file.csv', 'transitions_file.csv' ") diff --git a/flepimop/gempyor_pkg/src/gempyor/config_validator.py b/flepimop/gempyor_pkg/src/gempyor/config_validator.py index 95f6e3029..bcec39340 100644 --- a/flepimop/gempyor_pkg/src/gempyor/config_validator.py +++ b/flepimop/gempyor_pkg/src/gempyor/config_validator.py @@ -5,6 +5,9 @@ from functools import partial from gempyor import compartments +def validate(file_path: str): + return True + def read_yaml(file_path: str) -> dict: with open(file_path, 'r') as stream: config = yaml.safe_load(stream) @@ -14,11 +17,6 @@ def read_yaml(file_path: str) -> dict: def allowed_values(v, values): assert v in values return v - -# def parse_value(cls, values): -# value = values.get('value') -# parsed_val = compartments.Compartments.parse_parameter_strings_to_numpy_arrays_v2(value) -# return parsed_val class SubpopSetupConfig(BaseModel): geodata: str From 183c768186bf5cf3b95fc207893607d06ebe2300 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 9 Oct 2024 14:43:38 -0400 Subject: [PATCH 02/53] add config patching --- flepimop/gempyor_pkg/src/gempyor/cli.py | 163 ++++++++++++++++-- .../src/gempyor/deprecated_option.py | 62 +++++++ 2 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 flepimop/gempyor_pkg/src/gempyor/deprecated_option.py diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 9cabcd7a3..5133f9fc3 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,22 +1,154 @@ import click -import sys +from functools import reduce # from .cli_shared import argument_config_files, parse_config_files, config # TODO ...is this importing a global config object? do not like from gempyor.utils import config from gempyor.compartments import Compartments +# from gempyor.deprecated_option import DeprecatedOption, DeprecatedOptionsCommand +import multiprocessing +import warnings -argument_config_files = click.argument( - "config_files", - nargs = -1, - type = click.Path(exists = True), - required = True -) +argument_config_files = click.argument("config_files", nargs = -1, type = click.Path(exists = True), required = True) -def parse_config_files(config_files) -> None: +config_file_options = [ + click.option( + "--write-csv/--no-write-csv", + default=False, show_default=True, + help="write csv output?", + ), + click.option( + "--write-parquet/--no-write-parquet", + default=True, show_default=True, + help="write parquet output?", + ), + click.option( + "-j", "--jobs", envvar = "FLEPI_NJOBS", + type = click.IntRange(min = 1), default = multiprocessing.cpu_count(), show_default=True, + help = "the parallelization factor", + ), + click.option( + "-n", "--nslots", envvar = "FLEPI_NUM_SLOTS", + type = click.IntRange(min = 1), + help = "override the # of simulation runs in the config file", + ), + click.option( + "--in-id", "in_run_id", envvar="FLEPI_RUN_INDEX", + type=str, show_default=True, + help="Unique identifier for the run", + ), + click.option( + "--out-id", "out_run_id", envvar="FLEPI_RUN_INDEX", + type = str, show_default = True, + help = "Unique identifier for the run", + ), + click.option( + "--in-prefix", "in_prefix", envvar="FLEPI_PREFIX", + type=str, default=None, show_default=True, + help="unique identifier for the run", + ), + click.option( + "-i", "--first_sim_index", envvar = "FIRST_SIM_INDEX", + type = click.IntRange(min = 1), + default = 1, show_default = True, + help = "The index of the first simulation", + ), + click.option( + "--stochastic/--non-stochastic", "stoch_traj_flag", envvar = "FLEPI_STOCHASTIC_RUN", + default = False, + help = "Run stochastic simulations?", + ), + click.option( + "-s", "--seir_modifiers_scenario", envvar = "FLEPI_SEIR_SCENARIO", + type = str, default = [], multiple = True, + help = "override the NPI scenario(s) run for this simulation [supports multiple NPI scenarios: `-s Wuhan -s None`]", + ), + click.option( + "-d", "--outcome_modifiers_scenario", + envvar = "FLEPI_OUTCOME_SCENARIO", + type = str, default = [], multiple = True, + help = "Scenario of outcomes to run" + ), + click.option( + "-c", "--config", "config_filepath", envvar = "CONFIG_PATH", + type = click.Path(exists = True), + required = False, # deprecated = ["-c", "--config"], + # preferred = "CONFIG_FILES...", + help = "Deprecated: configuration file for this simulation" + ) +] + +def option_config_files(function): + reduce(lambda f, option: option(f), config_file_options, function) + return function + +def parse_config_files( + config_files, + config_filepath, + in_run_id, + out_run_id, + seir_modifiers_scenario, + outcome_modifiers_scenario, + in_prefix, + nslots, + jobs, + write_csv, + write_parquet, + first_sim_index, + stoch_traj_flag +) -> None: + # parse the config file(s) config.clear() + if config_filepath: + warnings.warn("The -(-c)onfig option / CONFIG_FILE environment variable invocation is deprecated. Use the positional argument CONFIG_FILES... instead; use of CONFIG_FILES... overrides this option.", DeprecationWarning) + if not len(config_files): + config_files = [config_filepath] + else: + warnings.warn("Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", DeprecationWarning) + for config_file in reversed(config_files): config.set_file(config_file) + # override the config file with any command line arguments + if seir_modifiers_scenario: + if config["seir_modifiers"].exists(): + config["seir_modifiers"]["scenarios"] = seir_modifiers_scenario + else: + config["seir_modifiers"] = {"scenarios": seir_modifiers_scenario} + + if outcome_modifiers_scenario: + if config["outcome_modifiers"].exists(): + config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenario + else: + config["outcome_modifiers"] = {"scenarios": outcome_modifiers_scenario} + + if nslots: + config["nslots"] = nslots + + if in_run_id: + config["in_run_id"] = in_run_id + + if out_run_id: + config["out_run_id"] = out_run_id + + if in_prefix: + config["in_prefix"] = in_prefix + + if jobs: + config["jobs"] = jobs + + if write_csv: + config["write_csv"] = write_csv + + if write_parquet: + config["write_parquet"] = write_parquet + + if first_sim_index: + config["first_sim_index"] = first_sim_index + + if stoch_traj_flag: + config["stoch_traj_flag"] = stoch_traj_flag + + @click.group() def cli(): """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" @@ -24,9 +156,10 @@ def cli(): @cli.command() @argument_config_files -def patch(config_files): +@option_config_files +def patch(**kwargs): """Merge configuration files""" - parse_config_files(config_files) + parse_config_files(**kwargs) print(config.dump()) @@ -38,9 +171,10 @@ def compartments(): # TODO: CLI arguments @compartments.command() @argument_config_files -def plot(config_files): +@option_config_files +def plot(**kwargs): """Plot compartments""" - parse_config_files(config_files) + parse_config_files(**kwargs) assert config["compartments"].exists() assert config["seir"].exists() comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) @@ -60,9 +194,10 @@ def plot(config_files): @compartments.command() @argument_config_files -def export(config_files): +@option_config_files +def export(**kwargs): """Export compartments""" - parse_config_files(config_files) + parse_config_files(**kwargs) assert config["compartments"].exists() assert config["seir"].exists() comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) diff --git a/flepimop/gempyor_pkg/src/gempyor/deprecated_option.py b/flepimop/gempyor_pkg/src/gempyor/deprecated_option.py new file mode 100644 index 000000000..8debe7453 --- /dev/null +++ b/flepimop/gempyor_pkg/src/gempyor/deprecated_option.py @@ -0,0 +1,62 @@ + +import click +import warnings + +# https://stackoverflow.com/a/50402799 +# CC BY-SA 4.0 https://creativecommons.org/licenses/by-sa/4.0/ +class DeprecatedOption(click.Option): + + def __init__(self, *args, **kwargs): + self.deprecated = kwargs.pop('deprecated', ()) + self.preferred = kwargs.pop('preferred', args[0][-1]) + super(DeprecatedOption, self).__init__(*args, **kwargs) + +class DeprecatedOptionsCommand(click.Command): + + def make_parser(self, ctx): + """Hook 'make_parser' and during processing check the name + used to invoke the option to see if it is preferred""" + + parser = super(DeprecatedOptionsCommand, self).make_parser(ctx) + + # get the parser options + options = set(parser._short_opt.values()) + options |= set(parser._long_opt.values()) + + for option in options: + if not isinstance(option.obj, DeprecatedOption): + continue + + def make_process(an_option): + """ Construct a closure to the parser option processor """ + + orig_process = an_option.process + deprecated = getattr(an_option.obj, 'deprecated', None) + preferred = getattr(an_option.obj, 'preferred', None) + msg = "Expected `deprecated` value for `{}`" + assert deprecated is not None, msg.format(an_option.obj.name) + + def process(value, state): + """The function above us on the stack used 'opt' to + pick option from a dict, see if it is deprecated """ + + # reach up the stack and get 'opt' + import inspect + frame = inspect.currentframe() + try: + opt = frame.f_back.f_locals.get('opt') + finally: + del frame + + if opt in deprecated: + msg = "'{}' has been deprecated, use '{}'" + warnings.warn(msg.format(opt, preferred), + FutureWarning) + + return orig_process(value, state) + + return process + + option.process = make_process(option) + + return parser From d3a6d5d19b3941613ba567e5c0bae73fc77c2fc6 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 09:55:13 -0400 Subject: [PATCH 03/53] address cleanup comments --- examples/tutorials/config_sample_2pop.yml | 1 - flepimop/gempyor_pkg/setup.cfg | 2 - flepimop/gempyor_pkg/src/gempyor/cli.py | 205 +----------------- .../gempyor_pkg/src/gempyor/compartments.py | 50 +++++ .../src/gempyor/config_validator.py | 3 - .../gempyor_pkg/src/gempyor/shared_cli.py | 149 +++++++++++++ 6 files changed, 201 insertions(+), 209 deletions(-) create mode 100644 flepimop/gempyor_pkg/src/gempyor/shared_cli.py diff --git a/examples/tutorials/config_sample_2pop.yml b/examples/tutorials/config_sample_2pop.yml index 6f65dd7d3..75593cb23 100644 --- a/examples/tutorials/config_sample_2pop.yml +++ b/examples/tutorials/config_sample_2pop.yml @@ -12,7 +12,6 @@ initial_conditions: method: SetInitialConditions initial_conditions_file: model_input/ic_2pop.csv allow_missing_compartments: TRUE - allow_missing_compartments: TRUE compartments: infection_stage: ["S", "E", "I", "R"] diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index cf90a2170..e5fb0902a 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -35,7 +35,6 @@ install_requires = dask scipy graphviz - pydantic # see https://stackoverflow.com/questions/58826164/dependencies-requirements-for-setting-up-testing-and-installing-a-python-lib # installed for pip install -e ".[test]" @@ -54,7 +53,6 @@ console_scripts = gempyor-seir = gempyor.simulate_seir:simulate gempyor-simulate = gempyor.simulate:simulate flepimop-calibrate = gempyor.calibrate:calibrate - flepimop-validate = gempyor.config_validator:validate [options.packages.find] where = src diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 5133f9fc3..8c26e46c8 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,158 +1,7 @@ import click -from functools import reduce -# from .cli_shared import argument_config_files, parse_config_files, config -# TODO ...is this importing a global config object? do not like +from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli from gempyor.utils import config -from gempyor.compartments import Compartments -# from gempyor.deprecated_option import DeprecatedOption, DeprecatedOptionsCommand -import multiprocessing -import warnings - -argument_config_files = click.argument("config_files", nargs = -1, type = click.Path(exists = True), required = True) - -config_file_options = [ - click.option( - "--write-csv/--no-write-csv", - default=False, show_default=True, - help="write csv output?", - ), - click.option( - "--write-parquet/--no-write-parquet", - default=True, show_default=True, - help="write parquet output?", - ), - click.option( - "-j", "--jobs", envvar = "FLEPI_NJOBS", - type = click.IntRange(min = 1), default = multiprocessing.cpu_count(), show_default=True, - help = "the parallelization factor", - ), - click.option( - "-n", "--nslots", envvar = "FLEPI_NUM_SLOTS", - type = click.IntRange(min = 1), - help = "override the # of simulation runs in the config file", - ), - click.option( - "--in-id", "in_run_id", envvar="FLEPI_RUN_INDEX", - type=str, show_default=True, - help="Unique identifier for the run", - ), - click.option( - "--out-id", "out_run_id", envvar="FLEPI_RUN_INDEX", - type = str, show_default = True, - help = "Unique identifier for the run", - ), - click.option( - "--in-prefix", "in_prefix", envvar="FLEPI_PREFIX", - type=str, default=None, show_default=True, - help="unique identifier for the run", - ), - click.option( - "-i", "--first_sim_index", envvar = "FIRST_SIM_INDEX", - type = click.IntRange(min = 1), - default = 1, show_default = True, - help = "The index of the first simulation", - ), - click.option( - "--stochastic/--non-stochastic", "stoch_traj_flag", envvar = "FLEPI_STOCHASTIC_RUN", - default = False, - help = "Run stochastic simulations?", - ), - click.option( - "-s", "--seir_modifiers_scenario", envvar = "FLEPI_SEIR_SCENARIO", - type = str, default = [], multiple = True, - help = "override the NPI scenario(s) run for this simulation [supports multiple NPI scenarios: `-s Wuhan -s None`]", - ), - click.option( - "-d", "--outcome_modifiers_scenario", - envvar = "FLEPI_OUTCOME_SCENARIO", - type = str, default = [], multiple = True, - help = "Scenario of outcomes to run" - ), - click.option( - "-c", "--config", "config_filepath", envvar = "CONFIG_PATH", - type = click.Path(exists = True), - required = False, # deprecated = ["-c", "--config"], - # preferred = "CONFIG_FILES...", - help = "Deprecated: configuration file for this simulation" - ) -] - -def option_config_files(function): - reduce(lambda f, option: option(f), config_file_options, function) - return function - -def parse_config_files( - config_files, - config_filepath, - in_run_id, - out_run_id, - seir_modifiers_scenario, - outcome_modifiers_scenario, - in_prefix, - nslots, - jobs, - write_csv, - write_parquet, - first_sim_index, - stoch_traj_flag -) -> None: - # parse the config file(s) - config.clear() - if config_filepath: - warnings.warn("The -(-c)onfig option / CONFIG_FILE environment variable invocation is deprecated. Use the positional argument CONFIG_FILES... instead; use of CONFIG_FILES... overrides this option.", DeprecationWarning) - if not len(config_files): - config_files = [config_filepath] - else: - warnings.warn("Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", DeprecationWarning) - - for config_file in reversed(config_files): - config.set_file(config_file) - - # override the config file with any command line arguments - if seir_modifiers_scenario: - if config["seir_modifiers"].exists(): - config["seir_modifiers"]["scenarios"] = seir_modifiers_scenario - else: - config["seir_modifiers"] = {"scenarios": seir_modifiers_scenario} - - if outcome_modifiers_scenario: - if config["outcome_modifiers"].exists(): - config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenario - else: - config["outcome_modifiers"] = {"scenarios": outcome_modifiers_scenario} - - if nslots: - config["nslots"] = nslots - - if in_run_id: - config["in_run_id"] = in_run_id - - if out_run_id: - config["out_run_id"] = out_run_id - - if in_prefix: - config["in_prefix"] = in_prefix - - if jobs: - config["jobs"] = jobs - - if write_csv: - config["write_csv"] = write_csv - - if write_parquet: - config["write_parquet"] = write_parquet - - if first_sim_index: - config["first_sim_index"] = first_sim_index - - if stoch_traj_flag: - config["stoch_traj_flag"] = stoch_traj_flag - - -@click.group() -def cli(): - """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" - pass +from gempyor.compartments import compartments @cli.command() @argument_config_files @@ -162,55 +11,5 @@ def patch(**kwargs): parse_config_files(**kwargs) print(config.dump()) - -@click.group() -def compartments(): - """Commands for working with FlepiMoP compartments""" - pass - -# TODO: CLI arguments -@compartments.command() -@argument_config_files -@option_config_files -def plot(**kwargs): - """Plot compartments""" - parse_config_files(**kwargs) - assert config["compartments"].exists() - assert config["seir"].exists() - comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) - - # TODO: this should be a command like build compartments. - ( - unique_strings, - transition_array, - proportion_array, - proportion_info, - ) = comp.get_transition_array() - - comp.plot(output_file="transition_graph", source_filters=[], destination_filters=[]) - - print("wrote file transition_graph") - - -@compartments.command() -@argument_config_files -@option_config_files -def export(**kwargs): - """Export compartments""" - parse_config_files(**kwargs) - assert config["compartments"].exists() - assert config["seir"].exists() - comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) - ( - unique_strings, - transition_array, - proportion_array, - proportion_info, - ) = comp.get_transition_array() - comp.toFile("compartments_file.csv", "transitions_file.csv", write_parquet=False) - print("wrote files 'compartments_file.csv', 'transitions_file.csv' ") - -cli.add_command(compartments) - if __name__ == "__main__": cli() diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index 8079d5798..2e04ae4e8 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -4,6 +4,9 @@ import pyarrow.parquet as pq from .utils import config, Timer, as_list from functools import reduce +import click +from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli + import logging logger = logging.getLogger(__name__) @@ -742,3 +745,50 @@ def list_recursive_convert_to_string(thing): return [list_recursive_convert_to_string(x) for x in thing] return str(thing) +@click.group() +def compartments(): + """Commands for working with FlepiMoP compartments""" + pass + +@compartments.command() +@argument_config_files +@option_config_files +def plot(**kwargs): + """Plot compartments""" + parse_config_files(**kwargs) + assert config["compartments"].exists() + assert config["seir"].exists() + comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) + + # TODO: this should be a command like build compartments. + ( + unique_strings, + transition_array, + proportion_array, + proportion_info, + ) = comp.get_transition_array() + + comp.plot(output_file="transition_graph", source_filters=[], destination_filters=[]) + + print("wrote file transition_graph") + + +@compartments.command() +@argument_config_files +@option_config_files +def export(**kwargs): + """Export compartments""" + parse_config_files(**kwargs) + assert config["compartments"].exists() + assert config["seir"].exists() + comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) + ( + unique_strings, + transition_array, + proportion_array, + proportion_info, + ) = comp.get_transition_array() + comp.toFile("compartments_file.csv", "transitions_file.csv", write_parquet=False) + print("wrote files 'compartments_file.csv', 'transitions_file.csv' ") + +cli.add_command(compartments) diff --git a/flepimop/gempyor_pkg/src/gempyor/config_validator.py b/flepimop/gempyor_pkg/src/gempyor/config_validator.py index bcec39340..2bc8202e2 100644 --- a/flepimop/gempyor_pkg/src/gempyor/config_validator.py +++ b/flepimop/gempyor_pkg/src/gempyor/config_validator.py @@ -5,9 +5,6 @@ from functools import partial from gempyor import compartments -def validate(file_path: str): - return True - def read_yaml(file_path: str) -> dict: with open(file_path, 'r') as stream: config = yaml.safe_load(stream) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py new file mode 100644 index 000000000..4f90f7e14 --- /dev/null +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -0,0 +1,149 @@ +import click +from functools import reduce +from gempyor.utils import config +import multiprocessing +import warnings + +@click.group() +def cli(): + """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" + pass + +argument_config_files = click.argument("config_files", nargs = -1, type = click.Path(exists = True), required = True) + +config_file_options = [ + click.option( + "--write-csv/--no-write-csv", + default=False, show_default=True, + help="write csv output?", + ), + click.option( + "--write-parquet/--no-write-parquet", + default=True, show_default=True, + help="write parquet output?", + ), + click.option( + "-j", "--jobs", envvar = "FLEPI_NJOBS", + type = click.IntRange(min = 1), default = multiprocessing.cpu_count(), show_default=True, + help = "the parallelization factor", + ), + click.option( + "-n", "--nslots", envvar = "FLEPI_NUM_SLOTS", + type = click.IntRange(min = 1), + help = "override the # of simulation runs in the config file", + ), + click.option( + "--in-id", "in_run_id", envvar="FLEPI_RUN_INDEX", + type=str, show_default=True, + help="Unique identifier for the run", + ), + click.option( + "--out-id", "out_run_id", envvar="FLEPI_RUN_INDEX", + type = str, show_default = True, + help = "Unique identifier for the run", + ), + click.option( + "--in-prefix", "in_prefix", envvar="FLEPI_PREFIX", + type=str, default=None, show_default=True, + help="unique identifier for the run", + ), + click.option( + "-i", "--first_sim_index", envvar = "FIRST_SIM_INDEX", + type = click.IntRange(min = 1), + default = 1, show_default = True, + help = "The index of the first simulation", + ), + click.option( + "--stochastic/--non-stochastic", "stoch_traj_flag", envvar = "FLEPI_STOCHASTIC_RUN", + default = False, + help = "Run stochastic simulations?", + ), + click.option( + "-s", "--seir_modifiers_scenario", envvar = "FLEPI_SEIR_SCENARIO", + type = str, default = [], multiple = True, + help = "override the NPI scenario(s) run for this simulation [supports multiple NPI scenarios: `-s Wuhan -s None`]", + ), + click.option( + "-d", "--outcome_modifiers_scenario", + envvar = "FLEPI_OUTCOME_SCENARIO", + type = str, default = [], multiple = True, + help = "Scenario of outcomes to run" + ), + click.option( + "-c", "--config", "config_filepath", envvar = "CONFIG_PATH", + type = click.Path(exists = True), + required = False, # deprecated = ["-c", "--config"], + # preferred = "CONFIG_FILES...", + help = "Deprecated: configuration file for this simulation" + ) +] + +def option_config_files(function): + reduce(lambda f, option: option(f), config_file_options, function) + return function + +def parse_config_files( + config_files, + config_filepath, + in_run_id, + out_run_id, + seir_modifiers_scenario, + outcome_modifiers_scenario, + in_prefix, + nslots, + jobs, + write_csv, + write_parquet, + first_sim_index, + stoch_traj_flag +) -> None: + # parse the config file(s) + config.clear() + if config_filepath: + if not len(config_files): + config_files = [config_filepath] + else: + warnings.warn("Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", DeprecationWarning) + + for config_file in reversed(config_files): + config.set_file(config_file) + + # override the config file with any command line arguments + if seir_modifiers_scenario: + if config["seir_modifiers"].exists(): + config["seir_modifiers"]["scenarios"] = seir_modifiers_scenario + else: + config["seir_modifiers"] = {"scenarios": seir_modifiers_scenario} + + if outcome_modifiers_scenario: + if config["outcome_modifiers"].exists(): + config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenario + else: + config["outcome_modifiers"] = {"scenarios": outcome_modifiers_scenario} + + if nslots: + config["nslots"] = nslots + + if in_run_id: + config["in_run_id"] = in_run_id + + if out_run_id: + config["out_run_id"] = out_run_id + + if in_prefix: + config["in_prefix"] = in_prefix + + if jobs: + config["jobs"] = jobs + + if write_csv: + config["write_csv"] = write_csv + + if write_parquet: + config["write_parquet"] = write_parquet + + if first_sim_index: + config["first_sim_index"] = first_sim_index + + if stoch_traj_flag: + config["stoch_traj_flag"] = stoch_traj_flag From 1f7109c0a684dabd8ce9ee5530b4a7c975a882cb Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 09:55:58 -0400 Subject: [PATCH 04/53] Update flepimop/gempyor_pkg/src/gempyor/cli.py --- flepimop/gempyor_pkg/src/gempyor/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 8c26e46c8..f89a82078 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,7 +1,6 @@ import click from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli from gempyor.utils import config -from gempyor.compartments import compartments @cli.command() @argument_config_files From 184b1fbfb0356a74b8ae6263c1127951355c7aee Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 10:15:24 -0400 Subject: [PATCH 05/53] further pare down tutorial configs --- .../config_sample_2pop_interventions_test.yml | 48 ------------ .../config_sample_2pop_modifiers.yml | 77 ------------------- .../tutorials/config_sample_2pop_outcomes.yml | 8 ++ 3 files changed, 8 insertions(+), 125 deletions(-) diff --git a/examples/tutorials/config_sample_2pop_interventions_test.yml b/examples/tutorials/config_sample_2pop_interventions_test.yml index 1e12f1d2d..1d5908a4c 100644 --- a/examples/tutorials/config_sample_2pop_interventions_test.yml +++ b/examples/tutorials/config_sample_2pop_interventions_test.yml @@ -1,51 +1,3 @@ -name: sample_2pop -setup_name: minimal -start_date: 2020-01-31 -end_date: 2020-05-31 -nslots: 1 - -subpop_setup: - geodata: model_input/geodata_sample_2pop.csv - mobility: model_input/mobility_sample_2pop.csv - popnodes: population - nodenames: province_name - -compartments: - infection_stage: ["S", "E", "I", "R"] - -seir: - integration: - method: rk4 - dt: 1 / 10 - parameters: - sigma: - value: - distribution: fixed - value: 1 / 4 - gamma: - value: - distribution: fixed - value: 1 / 5 - Ro: - value: - distribution: fixed - value: 3 - transitions: - - source: ["S"] - destination: ["E"] - rate: ["Ro * gamma"] - proportional_to: [["S"],["I"]] - proportion_exponent: ["1","1"] - - source: ["E"] - destination: ["I"] - rate: ["sigma"] - proportional_to: ["E"] - proportion_exponent: ["1"] - - source: ["I"] - destination: ["R"] - rate: ["gamma"] - proportional_to: ["I"] - proportion_exponent: ["1"] seeding: method: FromFile diff --git a/examples/tutorials/config_sample_2pop_modifiers.yml b/examples/tutorials/config_sample_2pop_modifiers.yml index 51237d6de..d5655564a 100644 --- a/examples/tutorials/config_sample_2pop_modifiers.yml +++ b/examples/tutorials/config_sample_2pop_modifiers.yml @@ -1,50 +1,3 @@ -name: sample_2pop -setup_name: minimal -start_date: 2020-02-01 -end_date: 2020-08-31 -nslots: 1 - -subpop_setup: - geodata: model_input/geodata_sample_2pop.csv - mobility: model_input/mobility_sample_2pop.csv - -initial_conditions: - method: SetInitialConditions - initial_conditions_file: model_input/ic_2pop.csv - allow_missing_subpops: TRUE - allow_missing_compartments: TRUE - -compartments: - infection_stage: ["S", "E", "I", "R"] - -seir: - integration: - method: rk4 - dt: 1 - parameters: - sigma: - value: 1 / 4 - gamma: - value: 1 / 5 - Ro: - value: 2.5 - transitions: - - source: ["S"] - destination: ["E"] - rate: ["Ro * gamma"] - proportional_to: [["S"],["I"]] - proportion_exponent: ["1","1"] - - source: ["E"] - destination: ["I"] - rate: ["sigma"] - proportional_to: ["E"] - proportion_exponent: ["1"] - - source: ["I"] - destination: ["R"] - rate: ["gamma"] - proportional_to: ["I"] - proportion_exponent: ["1"] - seir_modifiers: scenarios: - Ro_lockdown @@ -68,36 +21,6 @@ seir_modifiers: method: StackedModifier modifiers: ["Ro_lockdown","Ro_relax"] - -outcomes: - method: delayframe - outcomes: - incidCase: #incidence of detected cases - source: - incidence: - infection_stage: "I" - probability: - value: 0.5 - delay: - value: 5 - incidHosp: #incidence of hospitalizations - source: - incidence: - infection_stage: "I" - probability: - value: 0.05 - delay: - value: 7 - duration: - value: 10 - name: currHosp # will track number of current hospitalizations (ie prevalence) - incidDeath: #incidence of deaths - source: incidHosp - probability: - value: 0.2 - delay: - value: 14 - outcome_modifiers: scenarios: - test_limits diff --git a/examples/tutorials/config_sample_2pop_outcomes.yml b/examples/tutorials/config_sample_2pop_outcomes.yml index 44484403c..ce8ab8e57 100644 --- a/examples/tutorials/config_sample_2pop_outcomes.yml +++ b/examples/tutorials/config_sample_2pop_outcomes.yml @@ -1,6 +1,14 @@ outcomes: method: delayframe outcomes: + incidCase: #incidence of detected cases + source: + incidence: + infection_stage: "I" + probability: + value: 0.5 + delay: + value: 5 incidHosp: #incidence of hospitalizations source: incidence: From aa4c87b6bfca358f0a5cdfafa083592006d754b1 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 14:25:48 -0400 Subject: [PATCH 06/53] expand cli coverage --- flepimop/gempyor_pkg/src/gempyor/cli.py | 9 +- .../gempyor_pkg/src/gempyor/compartments.py | 2 +- flepimop/gempyor_pkg/src/gempyor/simulate.py | 156 +++--------------- 3 files changed, 28 insertions(+), 139 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index f89a82078..4a02dd826 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,7 +1,11 @@ -import click from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli from gempyor.utils import config +# Guidance for extending the CLI: +# - to add a new small command to the CLI, can just add a new function with the @cli.command() decorator here (e.g. patch below) +# - to add something with lots of module logic in it, should define that in the module (e.g. compartments for a group command, or simulate for a single command) +# - ... and then import that module here to add it to the CLI + @cli.command() @argument_config_files @option_config_files @@ -10,5 +14,8 @@ def patch(**kwargs): parse_config_files(**kwargs) print(config.dump()) +from .compartments import compartments +from .simulate import simulate + if __name__ == "__main__": cli() diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index 2e04ae4e8..2fadc6ade 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -745,7 +745,7 @@ def list_recursive_convert_to_string(thing): return [list_recursive_convert_to_string(x) for x in thing] return str(thing) -@click.group() +@cli.group() def compartments(): """Commands for working with FlepiMoP compartments""" pass diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 34fdd9d4b..2cad2d7a9 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -159,166 +159,49 @@ import multiprocessing import time, os, itertools -import click +from . import seir, outcomes, model_info, file_paths +from .utils import config, as_list, profile -from gempyor import seir, outcomes, model_info, file_paths -from gempyor.utils import config, as_list, profile +from .shared_cli import argument_config_files, option_config_files, parse_config_files, cli # from .profile import profile_options - -@click.command() -@click.option( - "-c", - "--config", - "config_filepath", - envvar=["CONFIG_PATH", "CONFIG_PATH"], - type=click.Path(exists=True), - required=True, - help="configuration file for this simulation", -) -@click.option( - "-s", - "--seir_modifiers_scenario", - "seir_modifiers_scenarios", - envvar="FLEPI_SEIR_SCENARIO", - type=str, - default=[], - multiple=True, - help="override the NPI scenario(s) run for this simulation [supports multiple NPI scenarios: `-s Wuhan -s None`]", -) -@click.option( - "-d", - "--outcome_modifiers_scenario", - "outcome_modifiers_scenarios", - envvar="FLEPI_OUTCOME_SCENARIO", - type=str, - default=[], - multiple=True, - help="Scenario of outcomes to run", -) -@click.option( - "-n", - "--nslots", - envvar="FLEPI_NUM_SLOTS", - type=click.IntRange(min=1), - help="override the # of simulation runs in the config file", -) -@click.option( - "-i", - "--first_sim_index", - envvar="FIRST_SIM_INDEX", - type=click.IntRange(min=1), - default=1, - show_default=True, - help="The index of the first simulation", -) -@click.option( - "-j", - "--jobs", - envvar="FLEPI_NJOBS", - type=click.IntRange(min=1), - default=multiprocessing.cpu_count(), - show_default=True, - help="the parallelization factor", -) -@click.option( - "--stochastic/--non-stochastic", - "--stochastic/--non-stochastic", - "stoch_traj_flag", - envvar="FLEPI_STOCHASTIC_RUN", - type=bool, - default=False, - help="Flag determining whether to run stochastic simulations or not", -) -@click.option( - "--in-id", - "--in-id", - "in_run_id", - envvar="FLEPI_RUN_INDEX", - type=str, - default=file_paths.run_id(), - show_default=True, - help="Unique identifier for the run", -) # Default does not make sense here -@click.option( - "--out-id", - "--out-id", - "out_run_id", - envvar="FLEPI_RUN_INDEX", - type=str, - default=None, - show_default=True, - help="Unique identifier for the run", -) -@click.option( - "--in-prefix", - "--in-prefix", - "in_prefix", - envvar="FLEPI_PREFIX", - type=str, - default=None, - show_default=True, - help="unique identifier for the run", -) -@click.option( - "--write-csv/--no-write-csv", - default=False, - show_default=True, - help="write CSV output at end of simulation", -) -@click.option( - "--write-parquet/--no-write-parquet", - default=True, - show_default=True, - help="write parquet file output at end of simulation", -) # @profile_options # @profile() +@cli.command() +@argument_config_files +@option_config_files def simulate( + config_files, config_filepath, in_run_id, out_run_id, - seir_modifiers_scenarios, - outcome_modifiers_scenarios, + seir_modifiers_scenario, + outcome_modifiers_scenario, in_prefix, nslots, jobs, write_csv, write_parquet, first_sim_index, - stoch_traj_flag, + stoch_traj_flag ): - config.clear() - config.read(user=False) - config.set_file(config_filepath) - print(outcome_modifiers_scenarios, seir_modifiers_scenarios) - - # Compute the list of scenarios to run. Since multiple = True, it's always a list. - if not seir_modifiers_scenarios: - seir_modifiers_scenarios = None - if config["seir_modifiers"].exists(): - if config["seir_modifiers"]["scenarios"].exists(): - seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"].as_str_seq() - # Model Info handles the case of the default scneario - if not outcome_modifiers_scenarios: - outcome_modifiers_scenarios = None - if config["outcomes"].exists() and config["outcome_modifiers"].exists(): - if config["outcome_modifiers"]["scenarios"].exists(): - outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"].as_str_seq() + """Forward simulate a model""" + largs = locals() + parse_config_files(**largs) - outcome_modifiers_scenarios = as_list(outcome_modifiers_scenarios) - seir_modifiers_scenarios = as_list(seir_modifiers_scenarios) - print(outcome_modifiers_scenarios, seir_modifiers_scenarios) + print(outcome_modifiers_scenario, seir_modifiers_scenario) + outcome_modifiers_scenario = as_list(config["outcome_modifiers_scenarios"]) + seir_modifiers_scenario = as_list(config["seir_modifiers_scenarios"]) + print(outcome_modifiers_scenario, seir_modifiers_scenario) - scenarios_combinations = [[s, d] for s in seir_modifiers_scenarios for d in outcome_modifiers_scenarios] + scenarios_combinations = [[s, d] for s in seir_modifiers_scenario for d in outcome_modifiers_scenario] print("Combination of modifiers scenarios to be run: ") print(scenarios_combinations) for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: print(f"seir_modifier: {seir_modifiers_scenario}, outcomes_modifier:{outcome_modifiers_scenario}") - if not nslots: - nslots = config["nslots"].as_number() + nslots = config["nslots"].as_number() print(f"Simulations to be run: {nslots}") for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: @@ -359,7 +242,6 @@ def simulate( f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" ) - if __name__ == "__main__": simulate() From 7107b92e51c66ff978996b025d40e3063d37def7 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 14:32:08 -0400 Subject: [PATCH 07/53] remove non-used click imports --- flepimop/gempyor_pkg/src/gempyor/compartments.py | 1 - flepimop/gempyor_pkg/src/gempyor/postprocess_inference.py | 1 - 2 files changed, 2 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index 2fadc6ade..c77432b3a 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -4,7 +4,6 @@ import pyarrow.parquet as pq from .utils import config, Timer, as_list from functools import reduce -import click from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli import logging diff --git a/flepimop/gempyor_pkg/src/gempyor/postprocess_inference.py b/flepimop/gempyor_pkg/src/gempyor/postprocess_inference.py index 2b42e2944..23cceb098 100644 --- a/flepimop/gempyor_pkg/src/gempyor/postprocess_inference.py +++ b/flepimop/gempyor_pkg/src/gempyor/postprocess_inference.py @@ -9,7 +9,6 @@ # import seaborn as sns import matplotlib._color_data as mcd import pyarrow.parquet as pq -import click import subprocess import dask.dataframe as dd import matplotlib.dates as mdates From 873ab59ee29015443109fa57fbd493a4a3f99041 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 14:32:39 -0400 Subject: [PATCH 08/53] update tutorial testing --- examples/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index 349f7219a..cba547238 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -9,7 +9,7 @@ def test_config_sample_2pop(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ['-c', 'config_sample_2pop.yml']) + result = runner.invoke(simulate, ['config_sample_2pop.yml']) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug @@ -20,7 +20,7 @@ def test_config_sample_2pop(): def test_sample_2pop_modifiers(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ['-c', 'config_sample_2pop_modifiers.yml']) + result = runner.invoke(simulate, ['config_sample_2pop.yml', 'config_sample_2pop_outcomes.yml', 'config_sample_2pop_modifiers.yml']) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug From b292b3ece1ea365bdd8822db4cadd2d3943485d6 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 15:00:47 -0400 Subject: [PATCH 09/53] adding internal comments to cli.py --- flepimop/gempyor_pkg/src/gempyor/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 4a02dd826..a9d05912b 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -6,6 +6,7 @@ # - to add something with lots of module logic in it, should define that in the module (e.g. compartments for a group command, or simulate for a single command) # - ... and then import that module here to add it to the CLI +# add some basic commands to the CLI @cli.command() @argument_config_files @option_config_files @@ -14,6 +15,7 @@ def patch(**kwargs): parse_config_files(**kwargs) print(config.dump()) +# register the commands from the other modules from .compartments import compartments from .simulate import simulate From 8483f1c171367c7bd8eec31dc66ed09484950d39 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 10 Oct 2024 15:26:04 -0400 Subject: [PATCH 10/53] non-working WIP; issue with (non)scenario parsing --- .../gempyor_pkg/src/gempyor/shared_cli.py | 33 ++++++++++--------- flepimop/gempyor_pkg/src/gempyor/simulate.py | 14 ++++---- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 4f90f7e14..d94f12071 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -59,15 +59,15 @@ def cli(): help = "Run stochastic simulations?", ), click.option( - "-s", "--seir_modifiers_scenario", envvar = "FLEPI_SEIR_SCENARIO", + "-s", "--seir_modifiers_scenarios", envvar = "FLEPI_SEIR_SCENARIO", type = str, default = [], multiple = True, - help = "override the NPI scenario(s) run for this simulation [supports multiple NPI scenarios: `-s Wuhan -s None`]", + help = "override/select the transmission scenario(s) to run", ), click.option( - "-d", "--outcome_modifiers_scenario", + "-d", "--outcome_modifiers_scenarios", envvar = "FLEPI_OUTCOME_SCENARIO", type = str, default = [], multiple = True, - help = "Scenario of outcomes to run" + help = "override/select the outcome scenario(s) to run" ), click.option( "-c", "--config", "config_filepath", envvar = "CONFIG_PATH", @@ -87,8 +87,8 @@ def parse_config_files( config_filepath, in_run_id, out_run_id, - seir_modifiers_scenario, - outcome_modifiers_scenario, + seir_modifiers_scenarios, + outcome_modifiers_scenarios, in_prefix, nslots, jobs, @@ -109,17 +109,18 @@ def parse_config_files( config.set_file(config_file) # override the config file with any command line arguments - if seir_modifiers_scenario: - if config["seir_modifiers"].exists(): - config["seir_modifiers"]["scenarios"] = seir_modifiers_scenario - else: - config["seir_modifiers"] = {"scenarios": seir_modifiers_scenario} + if config["seir_modifiers"].exists(): + if seir_modifiers_scenarios: + config["seir_modifiers"]["scenarios"] = seir_modifiers_scenarios + else: + config["seir_modifiers"] = {"scenarios": seir_modifiers_scenarios} + - if outcome_modifiers_scenario: - if config["outcome_modifiers"].exists(): - config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenario - else: - config["outcome_modifiers"] = {"scenarios": outcome_modifiers_scenario} + if config["outcome_modifiers"].exists(): + if outcome_modifiers_scenarios: + config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenarios + else: + config["outcome_modifiers"] = {"scenarios": outcome_modifiers_scenarios} if nslots: config["nslots"] = nslots diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 2cad2d7a9..d5cecf9c4 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -176,8 +176,8 @@ def simulate( config_filepath, in_run_id, out_run_id, - seir_modifiers_scenario, - outcome_modifiers_scenario, + seir_modifiers_scenarios, + outcome_modifiers_scenarios, in_prefix, nslots, jobs, @@ -190,12 +190,12 @@ def simulate( largs = locals() parse_config_files(**largs) - print(outcome_modifiers_scenario, seir_modifiers_scenario) - outcome_modifiers_scenario = as_list(config["outcome_modifiers_scenarios"]) - seir_modifiers_scenario = as_list(config["seir_modifiers_scenarios"]) - print(outcome_modifiers_scenario, seir_modifiers_scenario) + print(outcome_modifiers_scenarios, seir_modifiers_scenarios) + outcome_modifiers_scenarios = as_list(config["outcome_modifiers"]["scenarios"]) + seir_modifiers_scenarios = as_list(config["seir_modifiers"]["scenarios"]) + print(outcome_modifiers_scenarios, seir_modifiers_scenarios) - scenarios_combinations = [[s, d] for s in seir_modifiers_scenario for d in outcome_modifiers_scenario] + scenarios_combinations = [[s, d] for s in seir_modifiers_scenarios for d in outcome_modifiers_scenarios] print("Combination of modifiers scenarios to be run: ") print(scenarios_combinations) for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: From 5319316f48d4ef63bac55f25802324a965da81b9 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Fri, 11 Oct 2024 14:20:29 -0400 Subject: [PATCH 11/53] update scenario parsing, test files --- examples/tutorials/config_sample_2pop.yml | 2 +- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 10 +++++----- flepimop/gempyor_pkg/src/gempyor/simulate.py | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/tutorials/config_sample_2pop.yml b/examples/tutorials/config_sample_2pop.yml index 75593cb23..2b336303c 100644 --- a/examples/tutorials/config_sample_2pop.yml +++ b/examples/tutorials/config_sample_2pop.yml @@ -1,7 +1,7 @@ name: sample_2pop setup_name: minimal start_date: 2020-02-01 -end_date: 2020-05-31 +end_date: 2020-08-31 nslots: 1 subpop_setup: diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index d94f12071..ec352a782 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -1,6 +1,6 @@ import click from functools import reduce -from gempyor.utils import config +from gempyor.utils import config, as_list import multiprocessing import warnings @@ -60,13 +60,13 @@ def cli(): ), click.option( "-s", "--seir_modifiers_scenarios", envvar = "FLEPI_SEIR_SCENARIO", - type = str, default = [], multiple = True, + type = str, default = None, multiple = True, help = "override/select the transmission scenario(s) to run", ), click.option( "-d", "--outcome_modifiers_scenarios", envvar = "FLEPI_OUTCOME_SCENARIO", - type = str, default = [], multiple = True, + type = str, default = None, multiple = True, help = "override/select the outcome scenario(s) to run" ), click.option( @@ -113,14 +113,14 @@ def parse_config_files( if seir_modifiers_scenarios: config["seir_modifiers"]["scenarios"] = seir_modifiers_scenarios else: - config["seir_modifiers"] = {"scenarios": seir_modifiers_scenarios} + config["seir_modifiers"] = {"scenarios": as_list(seir_modifiers_scenarios)} if config["outcome_modifiers"].exists(): if outcome_modifiers_scenarios: config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenarios else: - config["outcome_modifiers"] = {"scenarios": outcome_modifiers_scenarios} + config["outcome_modifiers"] = {"scenarios": as_list(outcome_modifiers_scenarios)} if nslots: config["nslots"] = nslots diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index d5cecf9c4..a934c105a 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -189,10 +189,11 @@ def simulate( """Forward simulate a model""" largs = locals() parse_config_files(**largs) + print(config.dump()) print(outcome_modifiers_scenarios, seir_modifiers_scenarios) - outcome_modifiers_scenarios = as_list(config["outcome_modifiers"]["scenarios"]) - seir_modifiers_scenarios = as_list(config["seir_modifiers"]["scenarios"]) + outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"].as_str_seq() + seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"].as_str_seq() print(outcome_modifiers_scenarios, seir_modifiers_scenarios) scenarios_combinations = [[s, d] for s in seir_modifiers_scenarios for d in outcome_modifiers_scenarios] From fd6b1397c831570cc14cb3a6d1421ccc519a7f04 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Fri, 11 Oct 2024 14:33:16 -0400 Subject: [PATCH 12/53] incorporate backwards compat --- examples/test_cli.py | 2 +- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index cba547238..c63bfe0af 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -30,7 +30,7 @@ def test_sample_2pop_modifiers(): def test_simple_usa_statelevel(): os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") runner = CliRunner() - result = runner.invoke(simulate, ['-c', 'simple_usa_statelevel.yml', '-n', '1']) + result = runner.invoke(simulate, ['-n', '1', '-c', 'simple_usa_statelevel.yml']) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index ec352a782..ca5bc0d85 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -9,7 +9,7 @@ def cli(): """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass -argument_config_files = click.argument("config_files", nargs = -1, type = click.Path(exists = True), required = True) +argument_config_files = click.argument("config_files", nargs = -1, type = click.Path(exists = True)) config_file_options = [ click.option( @@ -105,6 +105,9 @@ def parse_config_files( else: warnings.warn("Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", DeprecationWarning) + if not len(config_files): + raise ValueError("No configuration file(s) provided") + for config_file in reversed(config_files): config.set_file(config_file) From f20d58311a1f88578892728923cde140a06045ce Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Fri, 11 Oct 2024 16:33:30 -0400 Subject: [PATCH 13/53] deal with None cases --- flepimop/gempyor_pkg/src/gempyor/model_info.py | 4 ++-- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 11 ++++++++--- flepimop/gempyor_pkg/src/gempyor/simulate.py | 5 ++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/model_info.py b/flepimop/gempyor_pkg/src/gempyor/model_info.py index 54f981d8f..b1c0f03a3 100644 --- a/flepimop/gempyor_pkg/src/gempyor/model_info.py +++ b/flepimop/gempyor_pkg/src/gempyor/model_info.py @@ -142,7 +142,7 @@ def __init__( # SEIR modifiers self.npi_config_seir = None - if config["seir_modifiers"].exists(): + if config["seir_modifiers"].exists() and self.seir_modifiers_scenario is not None: if config["seir_modifiers"]["scenarios"].exists(): self.npi_config_seir = config["seir_modifiers"]["modifiers"][seir_modifiers_scenario] self.seir_modifiers_library = config["seir_modifiers"]["modifiers"].get() @@ -165,7 +165,7 @@ def __init__( self.outcomes_config = config["outcomes"] if config["outcomes"].exists() else None if self.outcomes_config is not None: self.npi_config_outcomes = None - if config["outcome_modifiers"].exists(): + if config["outcome_modifiers"].exists() and self.outcome_modifiers_scenario is not None: if config["outcome_modifiers"]["scenarios"].exists(): self.npi_config_outcomes = config["outcome_modifiers"]["modifiers"][self.outcome_modifiers_scenario] self.outcome_modifiers_library = config["outcome_modifiers"]["modifiers"].get() diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index ca5bc0d85..b5004e603 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -116,15 +116,20 @@ def parse_config_files( if seir_modifiers_scenarios: config["seir_modifiers"]["scenarios"] = seir_modifiers_scenarios else: - config["seir_modifiers"] = {"scenarios": as_list(seir_modifiers_scenarios)} + if seir_modifiers_scenarios: + config["seir_modifiers"] = {"scenarios": as_list(seir_modifiers_scenarios)} + else: + config["seir_modifiers"] = {"scenarios": [None]} if config["outcome_modifiers"].exists(): if outcome_modifiers_scenarios: config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenarios else: - config["outcome_modifiers"] = {"scenarios": as_list(outcome_modifiers_scenarios)} - + if outcome_modifiers_scenarios: + config["outcome_modifiers"] = {"scenarios": as_list(outcome_modifiers_scenarios)} + else: + config["outcome_modifiers"] = {"scenarios": [None]} if nslots: config["nslots"] = nslots diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index a934c105a..45a20460b 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -189,11 +189,10 @@ def simulate( """Forward simulate a model""" largs = locals() parse_config_files(**largs) - print(config.dump()) print(outcome_modifiers_scenarios, seir_modifiers_scenarios) - outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"].as_str_seq() - seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"].as_str_seq() + outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"].get() + seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"].get() print(outcome_modifiers_scenarios, seir_modifiers_scenarios) scenarios_combinations = [[s, d] for s in seir_modifiers_scenarios for d in outcome_modifiers_scenarios] From aba6137470d8eade2e63f35c946f3f3b04007624 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 12:43:36 -0400 Subject: [PATCH 14/53] addressing some tw comments --- flepimop/gempyor_pkg/src/gempyor/cli.py | 25 +- .../gempyor_pkg/src/gempyor/shared_cli.py | 213 ++++++++++-------- 2 files changed, 135 insertions(+), 103 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index a9d05912b..83eaef8fb 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,23 +1,30 @@ -from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli -from gempyor.utils import config +from .shared_cli import ( + argument_config_files, + option_config_files, + parse_config_files, + cli, +) +from .utils import config + +# register the commands from the other modules +from .compartments import compartments +from .simulate import simulate # Guidance for extending the CLI: -# - to add a new small command to the CLI, can just add a new function with the @cli.command() decorator here (e.g. patch below) -# - to add something with lots of module logic in it, should define that in the module (e.g. compartments for a group command, or simulate for a single command) -# - ... and then import that module here to add it to the CLI +# - to add a new small command to the CLI, add a new function with the @cli.command() decorator here (e.g. patch below) +# - to add something with lots of module logic in it, define that in the module (e.g. .compartments, .simulate above) +# - ... and then import that module above to add it to the CLI + # add some basic commands to the CLI @cli.command() @argument_config_files @option_config_files -def patch(**kwargs): +def patch(**kwargs) -> None: """Merge configuration files""" parse_config_files(**kwargs) print(config.dump()) -# register the commands from the other modules -from .compartments import compartments -from .simulate import simulate if __name__ == "__main__": cli() diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index b5004e603..a0cea7075 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -4,106 +4,153 @@ import multiprocessing import warnings +__all__ = [] + + @click.group() def cli(): """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass -argument_config_files = click.argument("config_files", nargs = -1, type = click.Path(exists = True)) + +argument_config_files = click.argument( + "config_files", nargs=-1, type=click.Path(exists=True) +) +""" `click` Argument decorator to handle configuration file(s) """ config_file_options = [ click.option( "--write-csv/--no-write-csv", - default=False, show_default=True, + default=False, + show_default=True, help="write csv output?", ), click.option( "--write-parquet/--no-write-parquet", - default=True, show_default=True, + default=True, + show_default=True, help="write parquet output?", ), click.option( - "-j", "--jobs", envvar = "FLEPI_NJOBS", - type = click.IntRange(min = 1), default = multiprocessing.cpu_count(), show_default=True, - help = "the parallelization factor", + "-j", + "--jobs", + envvar="FLEPI_NJOBS", + type=click.IntRange(min=1), + default=multiprocessing.cpu_count(), + show_default=True, + help="the parallelization factor", ), click.option( - "-n", "--nslots", envvar = "FLEPI_NUM_SLOTS", - type = click.IntRange(min = 1), - help = "override the # of simulation runs in the config file", + "-n", + "--nslots", + envvar="FLEPI_NUM_SLOTS", + type=click.IntRange(min=1), + help="override the # of simulation runs in the config file", ), click.option( - "--in-id", "in_run_id", envvar="FLEPI_RUN_INDEX", - type=str, show_default=True, + "--in-id", + "in_run_id", + envvar="FLEPI_RUN_INDEX", + type=str, + show_default=True, help="Unique identifier for the run", ), click.option( - "--out-id", "out_run_id", envvar="FLEPI_RUN_INDEX", - type = str, show_default = True, - help = "Unique identifier for the run", + "--out-id", + "out_run_id", + envvar="FLEPI_RUN_INDEX", + type=str, + show_default=True, + help="Unique identifier for the run", ), click.option( - "--in-prefix", "in_prefix", envvar="FLEPI_PREFIX", - type=str, default=None, show_default=True, + "--in-prefix", + "in_prefix", + envvar="FLEPI_PREFIX", + type=str, + default=None, + show_default=True, help="unique identifier for the run", ), click.option( - "-i", "--first_sim_index", envvar = "FIRST_SIM_INDEX", - type = click.IntRange(min = 1), - default = 1, show_default = True, - help = "The index of the first simulation", + "-i", + "--first_sim_index", + envvar="FIRST_SIM_INDEX", + type=click.IntRange(min=1), + default=1, + show_default=True, + help="The index of the first simulation", ), click.option( - "--stochastic/--non-stochastic", "stoch_traj_flag", envvar = "FLEPI_STOCHASTIC_RUN", - default = False, - help = "Run stochastic simulations?", + "--stochastic/--non-stochastic", + "stoch_traj_flag", + envvar="FLEPI_STOCHASTIC_RUN", + default=False, + help="Run stochastic simulations?", ), click.option( - "-s", "--seir_modifiers_scenarios", envvar = "FLEPI_SEIR_SCENARIO", - type = str, default = None, multiple = True, - help = "override/select the transmission scenario(s) to run", + "-s", + "--seir_modifiers_scenarios", + envvar="FLEPI_SEIR_SCENARIO", + type=str, + default=None, + multiple=True, + help="override/select the transmission scenario(s) to run", ), click.option( - "-d", "--outcome_modifiers_scenarios", - envvar = "FLEPI_OUTCOME_SCENARIO", - type = str, default = None, multiple = True, - help = "override/select the outcome scenario(s) to run" + "-d", + "--outcome_modifiers_scenarios", + envvar="FLEPI_OUTCOME_SCENARIO", + type=str, + default=None, + multiple=True, + help="override/select the outcome scenario(s) to run", ), click.option( - "-c", "--config", "config_filepath", envvar = "CONFIG_PATH", - type = click.Path(exists = True), - required = False, # deprecated = ["-c", "--config"], + "-c", + "--config", + "config_filepath", + envvar="CONFIG_PATH", + type=click.Path(exists=True), + required=False, # deprecated = ["-c", "--config"], # preferred = "CONFIG_FILES...", - help = "Deprecated: configuration file for this simulation" - ) + help="Deprecated: configuration file for this simulation", + ), ] +""" List of `click` options that will be applied by `option_config_files` """ + -def option_config_files(function): +def option_config_files(function: callable) -> callable: + """`click` Option decorator to apply handlers for all options""" reduce(lambda f, option: option(f), config_file_options, function) return function + def parse_config_files( - config_files, - config_filepath, - in_run_id, - out_run_id, - seir_modifiers_scenarios, - outcome_modifiers_scenarios, - in_prefix, - nslots, - jobs, - write_csv, - write_parquet, - first_sim_index, - stoch_traj_flag + config_files: list[str], + config_filepath: str, + in_run_id: str, + out_run_id: str, + seir_modifiers_scenarios: list[str], + outcome_modifiers_scenarios: list[str], + in_prefix: str, + nslots: int, + jobs: int, + write_csv: bool, + write_parquet: bool, + first_sim_index: int, + stoch_traj_flag: bool, ) -> None: - # parse the config file(s) + """Parse the configuration file(s) and override with command line arguments""" config.clear() if config_filepath: if not len(config_files): config_files = [config_filepath] else: - warnings.warn("Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", DeprecationWarning) + warnings.warn( + "Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", + DeprecationWarning, + ) if not len(config_files): raise ValueError("No configuration file(s) provided") @@ -111,48 +158,26 @@ def parse_config_files( for config_file in reversed(config_files): config.set_file(config_file) - # override the config file with any command line arguments - if config["seir_modifiers"].exists(): - if seir_modifiers_scenarios: - config["seir_modifiers"]["scenarios"] = seir_modifiers_scenarios - else: - if seir_modifiers_scenarios: - config["seir_modifiers"] = {"scenarios": as_list(seir_modifiers_scenarios)} + for option in ("seir_modifiers", "outcome_modifiers"): + if config[option].exists(): + if (value := locals()[f"{option}_scenarios"]) is not None: + config[option]["scenarios"] = value else: - config["seir_modifiers"] = {"scenarios": [None]} - + if (value := locals()[f"{option}_scenarios"]) is not None: + config[option] = {"scenarios": as_list(value)} + else: + config[option] = {"scenarios": [None]} - if config["outcome_modifiers"].exists(): - if outcome_modifiers_scenarios: - config["outcome_modifiers"]["scenarios"] = outcome_modifiers_scenarios - else: - if outcome_modifiers_scenarios: - config["outcome_modifiers"] = {"scenarios": as_list(outcome_modifiers_scenarios)} - else: - config["outcome_modifiers"] = {"scenarios": [None]} - if nslots: - config["nslots"] = nslots - - if in_run_id: - config["in_run_id"] = in_run_id - - if out_run_id: - config["out_run_id"] = out_run_id - - if in_prefix: - config["in_prefix"] = in_prefix - - if jobs: - config["jobs"] = jobs - - if write_csv: - config["write_csv"] = write_csv - - if write_parquet: - config["write_parquet"] = write_parquet - - if first_sim_index: - config["first_sim_index"] = first_sim_index - - if stoch_traj_flag: - config["stoch_traj_flag"] = stoch_traj_flag + for option in ( + "nslots", + "in_run_id", + "out_run_id", + "in_prefix", + "jobs", + "write_csv", + "write_parquet", + "first_sim_index", + "stoch_traj_flag", + ): + if (value := locals()[option]) is not None: + config[option] = value From edf87a4284afc56fe52a6309f49f7d655f9fc0d6 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 12:48:11 -0400 Subject: [PATCH 15/53] add back in commented out --- flepimop/gempyor_pkg/src/gempyor/config_validator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/config_validator.py b/flepimop/gempyor_pkg/src/gempyor/config_validator.py index 2bc8202e2..eb96e5eef 100644 --- a/flepimop/gempyor_pkg/src/gempyor/config_validator.py +++ b/flepimop/gempyor_pkg/src/gempyor/config_validator.py @@ -14,7 +14,12 @@ def read_yaml(file_path: str) -> dict: def allowed_values(v, values): assert v in values return v - + +# def parse_value(cls, values): +# value = values.get('value') +# parsed_val = compartments.Compartments.parse_parameter_strings_to_numpy_arrays_v2(value) +# return parsed_val + class SubpopSetupConfig(BaseModel): geodata: str mobility: Optional[str] From e115bd73f7e5d0247d2112473e9d43168b4186ef Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 12:48:44 -0400 Subject: [PATCH 16/53] add back in commented out + whitespace --- flepimop/gempyor_pkg/src/gempyor/config_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/config_validator.py b/flepimop/gempyor_pkg/src/gempyor/config_validator.py index eb96e5eef..95f6e3029 100644 --- a/flepimop/gempyor_pkg/src/gempyor/config_validator.py +++ b/flepimop/gempyor_pkg/src/gempyor/config_validator.py @@ -19,7 +19,7 @@ def allowed_values(v, values): # value = values.get('value') # parsed_val = compartments.Compartments.parse_parameter_strings_to_numpy_arrays_v2(value) # return parsed_val - + class SubpopSetupConfig(BaseModel): geodata: str mobility: Optional[str] From 4855349365fa0f4d19abaa96c3bc41e5d98101a3 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 12:55:39 -0400 Subject: [PATCH 17/53] reorder imports in shared cli --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index a0cea7075..66b824232 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -1,11 +1,12 @@ -import click -from functools import reduce -from gempyor.utils import config, as_list import multiprocessing import warnings +from functools import reduce -__all__ = [] +import click + +from .utils import config, as_list +__all__ = [] @click.group() def cli(): From 73b850739d39eca555926789a5d0299cba2dd36b Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 15:37:37 -0400 Subject: [PATCH 18/53] fix scenario parsing issues, re-route gempyor-simulate calls --- examples/test_cli.py | 14 +++++++++++-- flepimop/gempyor_pkg/setup.cfg | 6 +++--- .../gempyor_pkg/src/gempyor/model_info.py | 4 ++-- .../gempyor_pkg/src/gempyor/shared_cli.py | 20 ++++++++++--------- flepimop/gempyor_pkg/src/gempyor/simulate.py | 19 +++++++++--------- 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index c63bfe0af..68af466f6 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -1,7 +1,9 @@ +import os +import subprocess from click.testing import CliRunner + from gempyor.simulate import simulate -import os # See here to test click application https://click.palletsprojects.com/en/8.1.x/testing/ # would be useful to also call the command directly @@ -35,4 +37,12 @@ def test_simple_usa_statelevel(): print(result.exit_code) # useful for debug print(result.exception) # useful for debug assert result.exit_code == 0 - assert 'completed in' in result.output \ No newline at end of file + assert 'completed in' in result.output + +def test_simple_usa_statelevel_deprecated(): + os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") + result = subprocess.run(["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"]) + print(result.stdout) # useful for debug + print(result.stderr) # useful for debug + print(result.returncode) # useful for debug + assert result.returncode == 0 diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index e5fb0902a..0be7ae364 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -48,10 +48,10 @@ aws = [options.entry_points] console_scripts = - gempyor-outcomes = gempyor.simulate_outcome:simulate + gempyor-outcomes = gempyor.simulate_outcome:_deprecation_simulate flepimop = gempyor.cli:cli - gempyor-seir = gempyor.simulate_seir:simulate - gempyor-simulate = gempyor.simulate:simulate + gempyor-seir = gempyor.simulate_seir:_deprecation_simulate + gempyor-simulate = gempyor.simulate:_deprecation_simulate flepimop-calibrate = gempyor.calibrate:calibrate [options.packages.find] diff --git a/flepimop/gempyor_pkg/src/gempyor/model_info.py b/flepimop/gempyor_pkg/src/gempyor/model_info.py index b1c0f03a3..54f981d8f 100644 --- a/flepimop/gempyor_pkg/src/gempyor/model_info.py +++ b/flepimop/gempyor_pkg/src/gempyor/model_info.py @@ -142,7 +142,7 @@ def __init__( # SEIR modifiers self.npi_config_seir = None - if config["seir_modifiers"].exists() and self.seir_modifiers_scenario is not None: + if config["seir_modifiers"].exists(): if config["seir_modifiers"]["scenarios"].exists(): self.npi_config_seir = config["seir_modifiers"]["modifiers"][seir_modifiers_scenario] self.seir_modifiers_library = config["seir_modifiers"]["modifiers"].get() @@ -165,7 +165,7 @@ def __init__( self.outcomes_config = config["outcomes"] if config["outcomes"].exists() else None if self.outcomes_config is not None: self.npi_config_outcomes = None - if config["outcome_modifiers"].exists() and self.outcome_modifiers_scenario is not None: + if config["outcome_modifiers"].exists(): if config["outcome_modifiers"]["scenarios"].exists(): self.npi_config_outcomes = config["outcome_modifiers"]["modifiers"][self.outcome_modifiers_scenario] self.outcome_modifiers_library = config["outcome_modifiers"]["modifiers"].get() diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 66b824232..4b1f021e6 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -6,6 +6,10 @@ from .utils import config, as_list +""" +An internal module to share common CLI elements. +""" + __all__ = [] @click.group() @@ -94,7 +98,7 @@ def cli(): "--seir_modifiers_scenarios", envvar="FLEPI_SEIR_SCENARIO", type=str, - default=None, + default=[], multiple=True, help="override/select the transmission scenario(s) to run", ), @@ -103,7 +107,7 @@ def cli(): "--outcome_modifiers_scenarios", envvar="FLEPI_OUTCOME_SCENARIO", type=str, - default=None, + default=[], multiple=True, help="override/select the outcome scenario(s) to run", ), @@ -160,14 +164,12 @@ def parse_config_files( config.set_file(config_file) for option in ("seir_modifiers", "outcome_modifiers"): - if config[option].exists(): - if (value := locals()[f"{option}_scenarios"]) is not None: - config[option]["scenarios"] = value - else: - if (value := locals()[f"{option}_scenarios"]) is not None: - config[option] = {"scenarios": as_list(value)} + value = locals()[f"{option}_scenarios"] + if value: + if config[option].exists(): + config[option]["scenarios"] = as_list(value) else: - config[option] = {"scenarios": [None]} + raise ValueError(f"Specified {option}_scenarios when no {option} in configuration file(s): {value}") for option in ( "nslots", diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 45a20460b..5b432a7c9 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -156,8 +156,7 @@ ## @cond -import multiprocessing -import time, os, itertools +import time, os, itertools, warnings from . import seir, outcomes, model_info, file_paths from .utils import config, as_list, profile @@ -190,12 +189,10 @@ def simulate( largs = locals() parse_config_files(**largs) - print(outcome_modifiers_scenarios, seir_modifiers_scenarios) - outcome_modifiers_scenarios = config["outcome_modifiers"]["scenarios"].get() - seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"].get() - print(outcome_modifiers_scenarios, seir_modifiers_scenarios) - - scenarios_combinations = [[s, d] for s in seir_modifiers_scenarios for d in outcome_modifiers_scenarios] + scenarios_combinations = [ + [s, d] for s in (config["seir_modifiers"]["scenarios"].as_str_seq() if config["seir_modifiers"].exists() else [None]) + for d in (config["outcome_modifiers"]["scenarios"].as_str_seq() if config["outcome_modifiers"].exists() else [None])] + print("Combination of modifiers scenarios to be run: ") print(scenarios_combinations) for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: @@ -242,7 +239,11 @@ def simulate( f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" ) +def _deprecation_simulate(**args): + warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) + cli(["simulate"] + args, standalone_mode=False) + if __name__ == "__main__": - simulate() + _deprecation_simulate() ## @endcond From 2d22ee755b060915cf81328bde53019749319bcc Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 15:41:39 -0400 Subject: [PATCH 19/53] re-add trailing comma --- flepimop/gempyor_pkg/src/gempyor/simulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 5b432a7c9..4dd06d91f 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -183,7 +183,7 @@ def simulate( write_csv, write_parquet, first_sim_index, - stoch_traj_flag + stoch_traj_flag, ): """Forward simulate a model""" largs = locals() From d4769f763130b886b75483ec1f3cfd7c5f771a71 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 16:14:57 -0400 Subject: [PATCH 20/53] revert tutorial pruning --- examples/test_cli.py | 12 ++- .../config_sample_2pop_modifiers.part | 36 +++++++++ .../config_sample_2pop_modifiers.yml | 77 +++++++++++++++++++ .../config_sample_2pop_outcomes.part | 29 +++++++ .../tutorials/config_sample_2pop_outcomes.yml | 55 +++++++++++-- 5 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 examples/tutorials/config_sample_2pop_modifiers.part create mode 100644 examples/tutorials/config_sample_2pop_outcomes.part diff --git a/examples/test_cli.py b/examples/test_cli.py index 68af466f6..992afa7f0 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -22,7 +22,17 @@ def test_config_sample_2pop(): def test_sample_2pop_modifiers(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ['config_sample_2pop.yml', 'config_sample_2pop_outcomes.yml', 'config_sample_2pop_modifiers.yml']) + result = runner.invoke(simulate, ['config_sample_2pop.yml', 'config_sample_2pop_outcomes.part', 'config_sample_2pop_modifiers.part']) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert 'completed in' in result.output + +def test_sample_2pop_modifiers_combined(): + os.chdir(os.path.dirname(__file__) + "/tutorials") + runner = CliRunner() + result = runner.invoke(simulate, ['config_sample_2pop_modifiers.yml']) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug diff --git a/examples/tutorials/config_sample_2pop_modifiers.part b/examples/tutorials/config_sample_2pop_modifiers.part new file mode 100644 index 000000000..d5655564a --- /dev/null +++ b/examples/tutorials/config_sample_2pop_modifiers.part @@ -0,0 +1,36 @@ +seir_modifiers: + scenarios: + - Ro_lockdown + - Ro_all + modifiers: + Ro_lockdown: + method: SinglePeriodModifier + parameter: Ro + period_start_date: 2020-03-15 + period_end_date: 2020-05-01 + subpop: "all" + value: 0.4 + Ro_relax: + method: SinglePeriodModifier + parameter: Ro + period_start_date: 2020-05-01 + period_end_date: 2020-07-01 + subpop: "all" + value: 0.8 + Ro_all: + method: StackedModifier + modifiers: ["Ro_lockdown","Ro_relax"] + +outcome_modifiers: + scenarios: + - test_limits + modifiers: + # assume that due to limitations in testing, initially the case detection probability was lower + test_limits: + method: SinglePeriodModifier + parameter: incidCase + subpop: "all" + period_start_date: 2020-02-01 + period_end_date: 2020-06-01 + value: 0.5 + diff --git a/examples/tutorials/config_sample_2pop_modifiers.yml b/examples/tutorials/config_sample_2pop_modifiers.yml index d5655564a..51237d6de 100644 --- a/examples/tutorials/config_sample_2pop_modifiers.yml +++ b/examples/tutorials/config_sample_2pop_modifiers.yml @@ -1,3 +1,50 @@ +name: sample_2pop +setup_name: minimal +start_date: 2020-02-01 +end_date: 2020-08-31 +nslots: 1 + +subpop_setup: + geodata: model_input/geodata_sample_2pop.csv + mobility: model_input/mobility_sample_2pop.csv + +initial_conditions: + method: SetInitialConditions + initial_conditions_file: model_input/ic_2pop.csv + allow_missing_subpops: TRUE + allow_missing_compartments: TRUE + +compartments: + infection_stage: ["S", "E", "I", "R"] + +seir: + integration: + method: rk4 + dt: 1 + parameters: + sigma: + value: 1 / 4 + gamma: + value: 1 / 5 + Ro: + value: 2.5 + transitions: + - source: ["S"] + destination: ["E"] + rate: ["Ro * gamma"] + proportional_to: [["S"],["I"]] + proportion_exponent: ["1","1"] + - source: ["E"] + destination: ["I"] + rate: ["sigma"] + proportional_to: ["E"] + proportion_exponent: ["1"] + - source: ["I"] + destination: ["R"] + rate: ["gamma"] + proportional_to: ["I"] + proportion_exponent: ["1"] + seir_modifiers: scenarios: - Ro_lockdown @@ -21,6 +68,36 @@ seir_modifiers: method: StackedModifier modifiers: ["Ro_lockdown","Ro_relax"] + +outcomes: + method: delayframe + outcomes: + incidCase: #incidence of detected cases + source: + incidence: + infection_stage: "I" + probability: + value: 0.5 + delay: + value: 5 + incidHosp: #incidence of hospitalizations + source: + incidence: + infection_stage: "I" + probability: + value: 0.05 + delay: + value: 7 + duration: + value: 10 + name: currHosp # will track number of current hospitalizations (ie prevalence) + incidDeath: #incidence of deaths + source: incidHosp + probability: + value: 0.2 + delay: + value: 14 + outcome_modifiers: scenarios: - test_limits diff --git a/examples/tutorials/config_sample_2pop_outcomes.part b/examples/tutorials/config_sample_2pop_outcomes.part new file mode 100644 index 000000000..ce8ab8e57 --- /dev/null +++ b/examples/tutorials/config_sample_2pop_outcomes.part @@ -0,0 +1,29 @@ +outcomes: + method: delayframe + outcomes: + incidCase: #incidence of detected cases + source: + incidence: + infection_stage: "I" + probability: + value: 0.5 + delay: + value: 5 + incidHosp: #incidence of hospitalizations + source: + incidence: + infection_stage: "I" + probability: + value: 0.05 + delay: + value: 7 + duration: + value: 10 + name: currHosp # will track number of current hospitalizations (ie prevalence) + incidDeath: #incidence of deaths + source: incidHosp + probability: + value: 0.2 + delay: + value: 14 + diff --git a/examples/tutorials/config_sample_2pop_outcomes.yml b/examples/tutorials/config_sample_2pop_outcomes.yml index ce8ab8e57..0044a2e96 100644 --- a/examples/tutorials/config_sample_2pop_outcomes.yml +++ b/examples/tutorials/config_sample_2pop_outcomes.yml @@ -1,14 +1,53 @@ +name: sample_2pop +setup_name: minimal +start_date: 2020-02-01 +end_date: 2020-05-31 +nslots: 1 + +subpop_setup: + geodata: model_input/geodata_sample_2pop.csv + mobility: model_input/mobility_sample_2pop.csv + +initial_conditions: + method: SetInitialConditions + initial_conditions_file: model_input/ic_2pop.csv + allow_missing_subpops: TRUE + allow_missing_compartments: TRUE + +compartments: + infection_stage: ["S", "E", "I", "R"] + +seir: + integration: + method: rk4 + dt: 1 + parameters: + sigma: + value: 1 / 4 + gamma: + value: 1 / 5 + Ro: + value: 2.5 + transitions: + - source: ["S"] + destination: ["E"] + rate: ["Ro * gamma"] + proportional_to: [["S"],["I"]] + proportion_exponent: ["1","1"] + - source: ["E"] + destination: ["I"] + rate: ["sigma"] + proportional_to: ["E"] + proportion_exponent: ["1"] + - source: ["I"] + destination: ["R"] + rate: ["gamma"] + proportional_to: ["I"] + proportion_exponent: ["1"] + outcomes: method: delayframe outcomes: - incidCase: #incidence of detected cases - source: - incidence: - infection_stage: "I" - probability: - value: 0.5 - delay: - value: 5 incidHosp: #incidence of hospitalizations source: incidence: From abeb2b14ab62eff01846bad7297b0a1cb4486b69 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 16:18:03 -0400 Subject: [PATCH 21/53] revert more tutorial pruning --- ...config_sample_2pop_interventions_test.part | 18 +++++++ .../config_sample_2pop_interventions_test.yml | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 examples/tutorials/config_sample_2pop_interventions_test.part diff --git a/examples/tutorials/config_sample_2pop_interventions_test.part b/examples/tutorials/config_sample_2pop_interventions_test.part new file mode 100644 index 000000000..1d5908a4c --- /dev/null +++ b/examples/tutorials/config_sample_2pop_interventions_test.part @@ -0,0 +1,18 @@ + +seeding: + method: FromFile + seeding_file: model_input/seeding_2pop.csv + +modifiers: + scenarios: + - None + settings: + None: + template: Reduce + parameter: r0 + period_start_date: 2020-04-01 + period_end_date: 2020-05-15 + value: + distribution: fixed + value: 0 + diff --git a/examples/tutorials/config_sample_2pop_interventions_test.yml b/examples/tutorials/config_sample_2pop_interventions_test.yml index 1d5908a4c..1e12f1d2d 100644 --- a/examples/tutorials/config_sample_2pop_interventions_test.yml +++ b/examples/tutorials/config_sample_2pop_interventions_test.yml @@ -1,3 +1,51 @@ +name: sample_2pop +setup_name: minimal +start_date: 2020-01-31 +end_date: 2020-05-31 +nslots: 1 + +subpop_setup: + geodata: model_input/geodata_sample_2pop.csv + mobility: model_input/mobility_sample_2pop.csv + popnodes: population + nodenames: province_name + +compartments: + infection_stage: ["S", "E", "I", "R"] + +seir: + integration: + method: rk4 + dt: 1 / 10 + parameters: + sigma: + value: + distribution: fixed + value: 1 / 4 + gamma: + value: + distribution: fixed + value: 1 / 5 + Ro: + value: + distribution: fixed + value: 3 + transitions: + - source: ["S"] + destination: ["E"] + rate: ["Ro * gamma"] + proportional_to: [["S"],["I"]] + proportion_exponent: ["1","1"] + - source: ["E"] + destination: ["I"] + rate: ["sigma"] + proportional_to: ["E"] + proportion_exponent: ["1"] + - source: ["I"] + destination: ["R"] + rate: ["gamma"] + proportional_to: ["I"] + proportion_exponent: ["1"] seeding: method: FromFile From 9bfe250c7d429e3316b677b5ae8d8bbcec022389 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 16:53:34 -0400 Subject: [PATCH 22/53] maybe fix CI? --- flepimop/gempyor_pkg/src/gempyor/simulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 4dd06d91f..bc6c638c3 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -239,7 +239,7 @@ def simulate( f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" ) -def _deprecation_simulate(**args): +def _deprecation_simulate(*args): warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) cli(["simulate"] + args, standalone_mode=False) From d45c65f90f41ed5cb238051d57db8be117355969 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 17:06:37 -0400 Subject: [PATCH 23/53] another attempt to fix dispatch --- flepimop/gempyor_pkg/src/gempyor/simulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index bc6c638c3..216ead572 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -241,7 +241,7 @@ def simulate( def _deprecation_simulate(*args): warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) - cli(["simulate"] + args, standalone_mode=False) + cli(["simulate"].extend(args), standalone_mode=False) if __name__ == "__main__": _deprecation_simulate() From 23e5f04c7a7b56730f70fe7f1d2410c6c82e7f0d Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 14 Oct 2024 17:56:59 -0400 Subject: [PATCH 24/53] address cli testing issue --- examples/test_cli.py | 2 +- flepimop/gempyor_pkg/setup.cfg | 6 +++--- flepimop/gempyor_pkg/src/gempyor/simulate.py | 7 ++----- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index 992afa7f0..598533abc 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -51,7 +51,7 @@ def test_simple_usa_statelevel(): def test_simple_usa_statelevel_deprecated(): os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") - result = subprocess.run(["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"]) + result = subprocess.run(["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"], capture_output=True, text=True) print(result.stdout) # useful for debug print(result.stderr) # useful for debug print(result.returncode) # useful for debug diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index 0be7ae364..e5fb0902a 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -48,10 +48,10 @@ aws = [options.entry_points] console_scripts = - gempyor-outcomes = gempyor.simulate_outcome:_deprecation_simulate + gempyor-outcomes = gempyor.simulate_outcome:simulate flepimop = gempyor.cli:cli - gempyor-seir = gempyor.simulate_seir:_deprecation_simulate - gempyor-simulate = gempyor.simulate:_deprecation_simulate + gempyor-seir = gempyor.simulate_seir:simulate + gempyor-simulate = gempyor.simulate:simulate flepimop-calibrate = gempyor.calibrate:calibrate [options.packages.find] diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 216ead572..0c1e57ce7 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -239,11 +239,8 @@ def simulate( f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" ) -def _deprecation_simulate(*args): - warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) - cli(["simulate"].extend(args), standalone_mode=False) - if __name__ == "__main__": - _deprecation_simulate() + warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) + simulate() ## @endcond From 70ef13991f2a0e748fa56f3d6f2d6e6c83504723 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 15 Oct 2024 09:27:47 -0400 Subject: [PATCH 25/53] update examples and cli testing --- examples/test_cli.py | 113 ++++++++++++------ ...g_sample_2pop_interventions_test_part.yml} | 0 ... => config_sample_2pop_modifiers_part.yml} | 0 ...t => config_sample_2pop_outcomes_part.yml} | 0 4 files changed, 75 insertions(+), 38 deletions(-) rename examples/tutorials/{config_sample_2pop_interventions_test.part => config_sample_2pop_interventions_test_part.yml} (100%) rename examples/tutorials/{config_sample_2pop_modifiers.part => config_sample_2pop_modifiers_part.yml} (100%) rename examples/tutorials/{config_sample_2pop_outcomes.part => config_sample_2pop_outcomes_part.yml} (100%) diff --git a/examples/test_cli.py b/examples/test_cli.py index 598533abc..339b07b1c 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -8,51 +8,88 @@ # See here to test click application https://click.palletsprojects.com/en/8.1.x/testing/ # would be useful to also call the command directly + def test_config_sample_2pop(): - os.chdir(os.path.dirname(__file__) + "/tutorials") - runner = CliRunner() - result = runner.invoke(simulate, ['config_sample_2pop.yml']) - print(result.output) # useful for debug - print(result.exit_code) # useful for debug - print(result.exception) # useful for debug - assert result.exit_code == 0 - assert 'completed in' in result.output + os.chdir(os.path.dirname(__file__) + "/tutorials") + runner = CliRunner() + result = runner.invoke(simulate, ["config_sample_2pop.yml"]) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert "completed in" in result.output + + +def test_config_sample_2pop(): + os.chdir(os.path.dirname(__file__) + "/tutorials") + runner = CliRunner() + result = runner.invoke(simulate, ["-c", "config_sample_2pop.yml"]) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert "completed in" in result.output def test_sample_2pop_modifiers(): - os.chdir(os.path.dirname(__file__) + "/tutorials") - runner = CliRunner() - result = runner.invoke(simulate, ['config_sample_2pop.yml', 'config_sample_2pop_outcomes.part', 'config_sample_2pop_modifiers.part']) - print(result.output) # useful for debug - print(result.exit_code) # useful for debug - print(result.exception) # useful for debug - assert result.exit_code == 0 - assert 'completed in' in result.output + os.chdir(os.path.dirname(__file__) + "/tutorials") + runner = CliRunner() + result = runner.invoke( + simulate, + [ + "config_sample_2pop.yml", + "config_sample_2pop_outcomes_part.yml", + "config_sample_2pop_modifiers_part.yml", + ], + ) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert "completed in" in result.output + def test_sample_2pop_modifiers_combined(): - os.chdir(os.path.dirname(__file__) + "/tutorials") - runner = CliRunner() - result = runner.invoke(simulate, ['config_sample_2pop_modifiers.yml']) - print(result.output) # useful for debug - print(result.exit_code) # useful for debug - print(result.exception) # useful for debug - assert result.exit_code == 0 - assert 'completed in' in result.output + os.chdir(os.path.dirname(__file__) + "/tutorials") + runner = CliRunner() + result = runner.invoke(simulate, ["config_sample_2pop_modifiers.yml"]) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert "completed in" in result.output + + +def test_sample_2pop_modifiers_combined(): + os.chdir(os.path.dirname(__file__) + "/tutorials") + runner = CliRunner() + result = runner.invoke(simulate, ["-c", "config_sample_2pop_modifiers.yml"]) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert "completed in" in result.output + def test_simple_usa_statelevel(): - os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") - runner = CliRunner() - result = runner.invoke(simulate, ['-n', '1', '-c', 'simple_usa_statelevel.yml']) - print(result.output) # useful for debug - print(result.exit_code) # useful for debug - print(result.exception) # useful for debug - assert result.exit_code == 0 - assert 'completed in' in result.output + os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") + runner = CliRunner() + result = runner.invoke(simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"]) + print(result.output) # useful for debug + print(result.exit_code) # useful for debug + print(result.exception) # useful for debug + assert result.exit_code == 0 + assert "completed in" in result.output + def test_simple_usa_statelevel_deprecated(): - os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") - result = subprocess.run(["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"], capture_output=True, text=True) - print(result.stdout) # useful for debug - print(result.stderr) # useful for debug - print(result.returncode) # useful for debug - assert result.returncode == 0 + os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") + result = subprocess.run( + ["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"], + capture_output=True, + text=True, + ) + print(result.stdout) # useful for debug + print(result.stderr) # useful for debug + print(result.returncode) # useful for debug + assert result.returncode == 0 diff --git a/examples/tutorials/config_sample_2pop_interventions_test.part b/examples/tutorials/config_sample_2pop_interventions_test_part.yml similarity index 100% rename from examples/tutorials/config_sample_2pop_interventions_test.part rename to examples/tutorials/config_sample_2pop_interventions_test_part.yml diff --git a/examples/tutorials/config_sample_2pop_modifiers.part b/examples/tutorials/config_sample_2pop_modifiers_part.yml similarity index 100% rename from examples/tutorials/config_sample_2pop_modifiers.part rename to examples/tutorials/config_sample_2pop_modifiers_part.yml diff --git a/examples/tutorials/config_sample_2pop_outcomes.part b/examples/tutorials/config_sample_2pop_outcomes_part.yml similarity index 100% rename from examples/tutorials/config_sample_2pop_outcomes.part rename to examples/tutorials/config_sample_2pop_outcomes_part.yml From e1feefb513c97c42585ce9d2af6042c17dffeb22 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 15 Oct 2024 10:36:29 -0400 Subject: [PATCH 26/53] prune deprecated, some shared_cli reorg --- .../src/gempyor/deprecated_option.py | 62 --------- .../gempyor_pkg/src/gempyor/shared_cli.py | 121 +++++++++--------- 2 files changed, 64 insertions(+), 119 deletions(-) delete mode 100644 flepimop/gempyor_pkg/src/gempyor/deprecated_option.py diff --git a/flepimop/gempyor_pkg/src/gempyor/deprecated_option.py b/flepimop/gempyor_pkg/src/gempyor/deprecated_option.py deleted file mode 100644 index 8debe7453..000000000 --- a/flepimop/gempyor_pkg/src/gempyor/deprecated_option.py +++ /dev/null @@ -1,62 +0,0 @@ - -import click -import warnings - -# https://stackoverflow.com/a/50402799 -# CC BY-SA 4.0 https://creativecommons.org/licenses/by-sa/4.0/ -class DeprecatedOption(click.Option): - - def __init__(self, *args, **kwargs): - self.deprecated = kwargs.pop('deprecated', ()) - self.preferred = kwargs.pop('preferred', args[0][-1]) - super(DeprecatedOption, self).__init__(*args, **kwargs) - -class DeprecatedOptionsCommand(click.Command): - - def make_parser(self, ctx): - """Hook 'make_parser' and during processing check the name - used to invoke the option to see if it is preferred""" - - parser = super(DeprecatedOptionsCommand, self).make_parser(ctx) - - # get the parser options - options = set(parser._short_opt.values()) - options |= set(parser._long_opt.values()) - - for option in options: - if not isinstance(option.obj, DeprecatedOption): - continue - - def make_process(an_option): - """ Construct a closure to the parser option processor """ - - orig_process = an_option.process - deprecated = getattr(an_option.obj, 'deprecated', None) - preferred = getattr(an_option.obj, 'preferred', None) - msg = "Expected `deprecated` value for `{}`" - assert deprecated is not None, msg.format(an_option.obj.name) - - def process(value, state): - """The function above us on the stack used 'opt' to - pick option from a dict, see if it is deprecated """ - - # reach up the stack and get 'opt' - import inspect - frame = inspect.currentframe() - try: - opt = frame.f_back.f_locals.get('opt') - finally: - del frame - - if opt in deprecated: - msg = "'{}' has been deprecated, use '{}'" - warnings.warn(msg.format(opt, preferred), - FutureWarning) - - return orig_process(value, state) - - return process - - option.process = make_process(option) - - return parser diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 4b1f021e6..ebfee8389 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -1,15 +1,17 @@ +""" +An internal module to share common CLI elements. Defines the overall cli group, +supported options for config file overrides, and custom click decorators. +""" + +from functools import reduce import multiprocessing +import pathlib import warnings -from functools import reduce import click from .utils import config, as_list -""" -An internal module to share common CLI elements. -""" - __all__ = [] @click.group() @@ -17,24 +19,42 @@ def cli(): """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass - +# click decorator to handle configuration file(s) as arguments +# use as `@argument_config_files` before a cli command definition argument_config_files = click.argument( "config_files", nargs=-1, type=click.Path(exists=True) ) -""" `click` Argument decorator to handle configuration file(s) """ +# List of `click` options that will be applied by `option_config_files` +# n.b., the help for these options will be presented in the order defined here config_file_options = [ click.option( - "--write-csv/--no-write-csv", - default=False, - show_default=True, - help="write csv output?", + "-c", + "--config", + "config_filepath", + envvar="CONFIG_PATH", + type=click.Path(exists=True), + required=False, # deprecated = ["-c", "--config"], + # preferred = "CONFIG_FILES...", + help="Deprecated: configuration file for this simulation", ), click.option( - "--write-parquet/--no-write-parquet", - default=True, - show_default=True, - help="write parquet output?", + "-s", + "--seir_modifiers_scenarios", + envvar="FLEPI_SEIR_SCENARIO", + type=str, + default=[], + multiple=True, + help="override/select the transmission scenario(s) to run", + ), + click.option( + "-d", + "--outcome_modifiers_scenarios", + envvar="FLEPI_OUTCOME_SCENARIO", + type=str, + default=[], + multiple=True, + help="override/select the outcome scenario(s) to run", ), click.option( "-j", @@ -94,59 +114,46 @@ def cli(): help="Run stochastic simulations?", ), click.option( - "-s", - "--seir_modifiers_scenarios", - envvar="FLEPI_SEIR_SCENARIO", - type=str, - default=[], - multiple=True, - help="override/select the transmission scenario(s) to run", - ), - click.option( - "-d", - "--outcome_modifiers_scenarios", - envvar="FLEPI_OUTCOME_SCENARIO", - type=str, - default=[], - multiple=True, - help="override/select the outcome scenario(s) to run", + "--write-csv/--no-write-csv", + default=False, + show_default=True, + help="write csv output?", ), click.option( - "-c", - "--config", - "config_filepath", - envvar="CONFIG_PATH", - type=click.Path(exists=True), - required=False, # deprecated = ["-c", "--config"], - # preferred = "CONFIG_FILES...", - help="Deprecated: configuration file for this simulation", + "--write-parquet/--no-write-parquet", + default=True, + show_default=True, + help="write parquet output?", ), ] -""" List of `click` options that will be applied by `option_config_files` """ - def option_config_files(function: callable) -> callable: - """`click` Option decorator to apply handlers for all options""" - reduce(lambda f, option: option(f), config_file_options, function) + """ + `click` Option decorator to apply handlers for all options + use as `@option_config_files` before a cli command definition + """ + reduce(lambda f, option: option(f), reversed(config_file_options), function) return function def parse_config_files( - config_files: list[str], - config_filepath: str, - in_run_id: str, - out_run_id: str, - seir_modifiers_scenarios: list[str], - outcome_modifiers_scenarios: list[str], - in_prefix: str, - nslots: int, - jobs: int, - write_csv: bool, - write_parquet: bool, - first_sim_index: int, - stoch_traj_flag: bool, + config_files: list[pathlib.Path], + config_filepath: pathlib.Path = None, + seir_modifiers_scenarios: list[str] = [], + outcome_modifiers_scenarios: list[str] = [], + in_run_id: str = None, + out_run_id: str = None, + in_prefix: str = None, + nslots: int = None, + jobs: int = None, + write_csv: bool = False, + write_parquet: bool = True, + first_sim_index: int = 1, + stoch_traj_flag: bool = False, ) -> None: - """Parse the configuration file(s) and override with command line arguments""" + """ + Parse configuration file(s) and override with command line arguments + """ config.clear() if config_filepath: if not len(config_files): From 5a9f92db12d248122357474b44c5015c95cb0df5 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 15 Oct 2024 15:57:51 -0400 Subject: [PATCH 27/53] WIP on kwargs approach to cli [skip ci] --- flepimop/gempyor_pkg/src/gempyor/cli.py | 8 +- .../gempyor_pkg/src/gempyor/compartments.py | 17 ++--- .../gempyor_pkg/src/gempyor/shared_cli.py | 76 +++++++------------ flepimop/gempyor_pkg/src/gempyor/simulate.py | 7 +- 4 files changed, 41 insertions(+), 67 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 83eaef8fb..b2693f6da 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,6 +1,6 @@ from .shared_cli import ( - argument_config_files, - option_config_files, + config_files_argument, + config_file_options, parse_config_files, cli, ) @@ -17,9 +17,7 @@ # add some basic commands to the CLI -@cli.command() -@argument_config_files -@option_config_files +@cli.command(params=[config_files_argument].extend(config_file_options.values())) def patch(**kwargs) -> None: """Merge configuration files""" parse_config_files(**kwargs) diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index c77432b3a..2e7268b29 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -1,13 +1,14 @@ +from functools import reduce + import numpy as np import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from .utils import config, Timer, as_list -from functools import reduce -from gempyor.shared_cli import argument_config_files, option_config_files, parse_config_files, cli - import logging +from .utils import config, Timer, as_list +from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli + logger = logging.getLogger(__name__) @@ -749,9 +750,7 @@ def compartments(): """Commands for working with FlepiMoP compartments""" pass -@compartments.command() -@argument_config_files -@option_config_files +@compartments.command(params=[config_files_argument].extend(config_file_options.values())) def plot(**kwargs): """Plot compartments""" parse_config_files(**kwargs) @@ -772,9 +771,7 @@ def plot(**kwargs): print("wrote file transition_graph") -@compartments.command() -@argument_config_files -@option_config_files +@compartments.command(params=[config_files_argument].extend(config_file_options.values())) def export(**kwargs): """Export compartments""" parse_config_files(**kwargs) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index ebfee8389..b66b9b442 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -21,120 +21,102 @@ def cli(): # click decorator to handle configuration file(s) as arguments # use as `@argument_config_files` before a cli command definition -argument_config_files = click.argument( - "config_files", nargs=-1, type=click.Path(exists=True) +config_files_argument = click.Argument( + ["config_files"], nargs=-1, type=click.Path(exists=True) ) # List of `click` options that will be applied by `option_config_files` # n.b., the help for these options will be presented in the order defined here -config_file_options = [ - click.option( - "-c", - "--config", - "config_filepath", +config_file_options = { + "config_filepath" : click.Option( + ["-c", "--config", "config_filepath"], envvar="CONFIG_PATH", type=click.Path(exists=True), required=False, # deprecated = ["-c", "--config"], # preferred = "CONFIG_FILES...", help="Deprecated: configuration file for this simulation", ), - click.option( - "-s", - "--seir_modifiers_scenarios", + "seir_modifiers_scenarios": click.Option( + ["-s", "--seir_modifiers_scenarios"], envvar="FLEPI_SEIR_SCENARIO", type=str, default=[], multiple=True, help="override/select the transmission scenario(s) to run", ), - click.option( - "-d", - "--outcome_modifiers_scenarios", + "outcome_modifiers_scenarios": click.Option( + ["-d", "--outcome_modifiers_scenarios"], envvar="FLEPI_OUTCOME_SCENARIO", type=str, default=[], multiple=True, help="override/select the outcome scenario(s) to run", ), - click.option( - "-j", - "--jobs", + "jobs": click.Option( + ["-j", "--jobs"], envvar="FLEPI_NJOBS", type=click.IntRange(min=1), default=multiprocessing.cpu_count(), show_default=True, help="the parallelization factor", ), - click.option( - "-n", - "--nslots", + "nslots": click.Option( + ["-n", "--nslots"], envvar="FLEPI_NUM_SLOTS", type=click.IntRange(min=1), help="override the # of simulation runs in the config file", ), - click.option( - "--in-id", - "in_run_id", + "in_run_id": click.Option( + ["--in-id", "in_run_id"], envvar="FLEPI_RUN_INDEX", type=str, show_default=True, help="Unique identifier for the run", ), - click.option( - "--out-id", - "out_run_id", + "out_run_id": click.Option( + ["--out-id", "out_run_id"], envvar="FLEPI_RUN_INDEX", type=str, show_default=True, help="Unique identifier for the run", ), - click.option( - "--in-prefix", - "in_prefix", + "in_prefix": click.Option( + ["--in-prefix"], envvar="FLEPI_PREFIX", type=str, default=None, show_default=True, help="unique identifier for the run", ), - click.option( - "-i", - "--first_sim_index", + "first_sim_index": click.Option( + ["-i", "--first_sim_index"], envvar="FIRST_SIM_INDEX", type=click.IntRange(min=1), default=1, show_default=True, help="The index of the first simulation", ), - click.option( - "--stochastic/--non-stochastic", - "stoch_traj_flag", + "stoch_traj_flag": click.Option( + ["--stochastic/--non-stochastic", "stoch_traj_flag"], envvar="FLEPI_STOCHASTIC_RUN", default=False, help="Run stochastic simulations?", ), - click.option( - "--write-csv/--no-write-csv", + "write_csv": click.Option( + ["--write-csv/--no-write-csv"], default=False, show_default=True, help="write csv output?", ), - click.option( - "--write-parquet/--no-write-parquet", + "write_parquet": click.Option( + ["--write-parquet/--no-write-parquet"], default=True, show_default=True, help="write parquet output?", ), -] - -def option_config_files(function: callable) -> callable: - """ - `click` Option decorator to apply handlers for all options - use as `@option_config_files` before a cli command definition - """ - reduce(lambda f, option: option(f), reversed(config_file_options), function) - return function +} +# TODO: create a custom command decorator cls ala: https://click.palletsprojects.com/en/8.1.x/advanced/#command-aliases def parse_config_files( config_files: list[pathlib.Path], diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 0c1e57ce7..d5a8f6210 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -160,16 +160,13 @@ from . import seir, outcomes, model_info, file_paths from .utils import config, as_list, profile - -from .shared_cli import argument_config_files, option_config_files, parse_config_files, cli +from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli # from .profile import profile_options # @profile_options # @profile() -@cli.command() -@argument_config_files -@option_config_files +@cli.command(params=[config_files_argument].extend(config_file_options.values())) def simulate( config_files, config_filepath, From 5b87d7ab67ffa588aac0546150210ba36869536f Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 09:35:00 -0400 Subject: [PATCH 28/53] dynamically generated docstrings --- .../gempyor_pkg/src/gempyor/shared_cli.py | 40 ++++++++++++++++++- flepimop/gempyor_pkg/src/gempyor/simulate.py | 13 ++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index b66b9b442..a29aa1f34 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -3,10 +3,11 @@ supported options for config file overrides, and custom click decorators. """ -from functools import reduce import multiprocessing import pathlib import warnings +from typing import List, Union + import click @@ -116,8 +117,41 @@ def cli(): ), } + +# adapted from https://stackoverflow.com/a/78533451 +def click_helpstring(params: Union[click.Parameter, List[click.Parameter]]): + """ + A decorator that dynamically appends `click.Parameter`s to the docstring of the decorated function. + + Args: + params (Union[click.Parameter, List[click.Parameter]]): A parameter or a list of parameters whose corresponding argument values and help strings will be appended to the docstring of the decorated function. + + Returns: + Callable: The original function with an updated docstring. + """ + if not isinstance(params, list): + params = [params] + + def decorator(func): + # Generate the additional docstring with args from the specified functions + additional_doc = "\n\nCommand Line Interface arguments:\n" + for param in params: + paraminfo = param.to_info_dict() + additional_doc += f"\n{paraminfo['name']}: {paraminfo['type']['param_type']}\n" + + if func.__doc__ is None: + func.__doc__ = "" + func.__doc__ += additional_doc + + return func + + return decorator + # TODO: create a custom command decorator cls ala: https://click.palletsprojects.com/en/8.1.x/advanced/#command-aliases +# to also apply the `@click_helpstring` decorator to the command. Possibly to also default the params argument, assuming +# enough commands have consistent option set? +@click_helpstring([config_files_argument] + list(config_file_options.values())) def parse_config_files( config_files: list[pathlib.Path], config_filepath: pathlib.Path = None, @@ -135,6 +169,10 @@ def parse_config_files( ) -> None: """ Parse configuration file(s) and override with command line arguments + + Args: (see auto generated CLI items below) + + Returns: None (side effect: updates the global configuration object) """ config.clear() if config_filepath: diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index d5a8f6210..b60b80b9a 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -160,13 +160,14 @@ from . import seir, outcomes, model_info, file_paths from .utils import config, as_list, profile -from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli +from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli, click_helpstring # from .profile import profile_options # @profile_options # @profile() -@cli.command(params=[config_files_argument].extend(config_file_options.values())) +@cli.command(params=[config_files_argument] + list(config_file_options.values())) +@click_helpstring([config_files_argument] + list(config_file_options.values())) def simulate( config_files, config_filepath, @@ -182,7 +183,13 @@ def simulate( first_sim_index, stoch_traj_flag, ): - """Forward simulate a model""" + """ + Forward simulate a model using gempyor. + + Args: (see auto generated CLI items below) + + Returns: None (side effect: writes output to disk) + """ largs = locals() parse_config_files(**largs) From 59f7815600c33bac929a2b73560387fcf35c88b8 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 12:01:58 -0400 Subject: [PATCH 29/53] kwargs-ify parse_config_files --- .../gempyor_pkg/src/gempyor/shared_cli.py | 100 ++++++++---------- 1 file changed, 45 insertions(+), 55 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index a29aa1f34..325c3cb9b 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -7,7 +7,7 @@ import pathlib import warnings from typing import List, Union - +from functools import reduce import click @@ -26,16 +26,19 @@ def cli(): ["config_files"], nargs=-1, type=click.Path(exists=True) ) -# List of `click` options that will be applied by `option_config_files` +# List of standard `click` options that override/update config settings # n.b., the help for these options will be presented in the order defined here config_file_options = { "config_filepath" : click.Option( ["-c", "--config", "config_filepath"], envvar="CONFIG_PATH", type=click.Path(exists=True), - required=False, # deprecated = ["-c", "--config"], + required=False, + default=[], + # deprecated = ["-c", "--config"], # preferred = "CONFIG_FILES...", - help="Deprecated: configuration file for this simulation", + multiple=True, + help="Deprecated: configuration file(s) for this simulation", ), "seir_modifiers_scenarios": click.Option( ["-s", "--seir_modifiers_scenarios"], @@ -152,62 +155,49 @@ def decorator(func): # enough commands have consistent option set? @click_helpstring([config_files_argument] + list(config_file_options.values())) -def parse_config_files( - config_files: list[pathlib.Path], - config_filepath: pathlib.Path = None, - seir_modifiers_scenarios: list[str] = [], - outcome_modifiers_scenarios: list[str] = [], - in_run_id: str = None, - out_run_id: str = None, - in_prefix: str = None, - nslots: int = None, - jobs: int = None, - write_csv: bool = False, - write_parquet: bool = True, - first_sim_index: int = 1, - stoch_traj_flag: bool = False, -) -> None: +def parse_config_files(**kwargs) -> None: """ Parse configuration file(s) and override with command line arguments - Args: (see auto generated CLI items below) + Args: + **kwargs: see auto generated CLI items below. unmatched keys will be ignored + a warning will be issued Returns: None (side effect: updates the global configuration object) """ - config.clear() - if config_filepath: - if not len(config_files): - config_files = [config_filepath] + parsed_args = {config_files_argument.name}.union({option.name for option in config_file_options.values()}) + + # warn re unrecognized arguments + if parsed_args.difference(kwargs.keys()): + warnings.warn(f"Unused arguments: {parsed_args.difference(kwargs.keys())}") + + # initialize the config, including handling missing / double-specified config files + config_args = {k for k in parsed_args if k.startswith("config")} + found_configs = [k for k in config_args if kwargs[k]] + config_src = [] + if len(found_configs) != 1: + if not found_configs: + raise ValueError(f"No config files provided.") + else: + error_dict = {k: kwargs[k] for k in found_configs} + raise ValueError(f"Exactly one config file source option must be provided; got {error_dict}.") + else: + config_src = kwargs[found_configs[0]] + config.clear() + for config_file in reversed(config_src): + config.set_file(config_file) + + # deal with the scenario overrides + scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs[k]} + for option in scen_args: + key = option.replace("_scenarios", "") + value = kwargs[option] + if config[key].exists(): + config[key]["scenarios"] = as_list(value) else: - warnings.warn( - "Found CONFIG_FILES... ignoring -(-c)onfig option / CONFIG_FILE environment variable.", - DeprecationWarning, - ) - - if not len(config_files): - raise ValueError("No configuration file(s) provided") - - for config_file in reversed(config_files): - config.set_file(config_file) - - for option in ("seir_modifiers", "outcome_modifiers"): - value = locals()[f"{option}_scenarios"] - if value: - if config[option].exists(): - config[option]["scenarios"] = as_list(value) - else: - raise ValueError(f"Specified {option}_scenarios when no {option} in configuration file(s): {value}") - - for option in ( - "nslots", - "in_run_id", - "out_run_id", - "in_prefix", - "jobs", - "write_csv", - "write_parquet", - "first_sim_index", - "stoch_traj_flag", - ): - if (value := locals()[option]) is not None: + raise ValueError(f"Specified {option} when no {key} in configuration file(s): {config_src}") + + # update the config with the remaining options + other_args = parsed_args - config_args - scen_args + for option in other_args: + if (value := kwargs[option]) is not None: config[option] = value From bd5a2783599a9ab2022d5ada774ae55ba07b83a8 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 12:10:55 -0400 Subject: [PATCH 30/53] Update flepimop/gempyor_pkg/src/gempyor/shared_cli.py Co-authored-by: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com> --- .../gempyor_pkg/src/gempyor/shared_cli.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 325c3cb9b..e38688b36 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -122,15 +122,36 @@ def cli(): # adapted from https://stackoverflow.com/a/78533451 -def click_helpstring(params: Union[click.Parameter, List[click.Parameter]]): +def click_helpstring( + params: click.Parameter | list[click.Parameter], +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: """ - A decorator that dynamically appends `click.Parameter`s to the docstring of the decorated function. + Dynamically append `click.Parameter`s to the docstring of a decorated function. Args: - params (Union[click.Parameter, List[click.Parameter]]): A parameter or a list of parameters whose corresponding argument values and help strings will be appended to the docstring of the decorated function. + params: A parameter or a list of parameters whose corresponding argument + values and help strings will be appended to the docstring of the + decorated function. Returns: - Callable: The original function with an updated docstring. + The original function with an updated docstring. + + Notes: + Adapted from https://stackoverflow.com/a/78533451. + + Examples: + >>> import click + >>> opt = click.Option("-n", "--number", type=int, help="Your favorite number.") + >>> @click_helpstring(opt) + ... def test_cli(num: int) -> None: + ... print(f"Your favorite number is {num}!") + ... + >>> print(test_cli.__doc__) + + + Command Line Interface arguments: + + n: Int """ if not isinstance(params, list): params = [params] From b0a5769eb7beabb725a0cd503e7d63f2d9bb10f2 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 13:24:21 -0400 Subject: [PATCH 31/53] kwargs-ify simulate --- examples/test_cli.py | 8 +-- .../gempyor_pkg/src/gempyor/shared_cli.py | 12 ++--- flepimop/gempyor_pkg/src/gempyor/simulate.py | 49 +++++++------------ 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index 339b07b1c..1f9bfdebb 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -20,7 +20,7 @@ def test_config_sample_2pop(): assert "completed in" in result.output -def test_config_sample_2pop(): +def test_config_sample_2pop_deprecated(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() result = runner.invoke(simulate, ["-c", "config_sample_2pop.yml"]) @@ -60,7 +60,7 @@ def test_sample_2pop_modifiers_combined(): assert "completed in" in result.output -def test_sample_2pop_modifiers_combined(): +def test_sample_2pop_modifiers_combined_deprecated(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() result = runner.invoke(simulate, ["-c", "config_sample_2pop_modifiers.yml"]) @@ -71,7 +71,7 @@ def test_sample_2pop_modifiers_combined(): assert "completed in" in result.output -def test_simple_usa_statelevel(): +def test_simple_usa_statelevel_deprecated(): os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") runner = CliRunner() result = runner.invoke(simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"]) @@ -82,7 +82,7 @@ def test_simple_usa_statelevel(): assert "completed in" in result.output -def test_simple_usa_statelevel_deprecated(): +def test_simple_usa_statelevel_more_deprecated(): os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") result = subprocess.run( ["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"], diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index e38688b36..8ff889bc3 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -6,8 +6,7 @@ import multiprocessing import pathlib import warnings -from typing import List, Union -from functools import reduce +from typing import Callable, Any import click @@ -120,8 +119,6 @@ def cli(): ), } - -# adapted from https://stackoverflow.com/a/78533451 def click_helpstring( params: click.Parameter | list[click.Parameter], ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: @@ -158,10 +155,10 @@ def click_helpstring( def decorator(func): # Generate the additional docstring with args from the specified functions - additional_doc = "\n\nCommand Line Interface arguments:\n" + additional_doc = "\n\tCommand Line Interface arguments:\n" for param in params: paraminfo = param.to_info_dict() - additional_doc += f"\n{paraminfo['name']}: {paraminfo['type']['param_type']}\n" + additional_doc += f"\n\t\t{paraminfo['name']}: {paraminfo['type']['param_type']}\n" if func.__doc__ is None: func.__doc__ = "" @@ -181,7 +178,7 @@ def parse_config_files(**kwargs) -> None: Parse configuration file(s) and override with command line arguments Args: - **kwargs: see auto generated CLI items below. unmatched keys will be ignored + a warning will be issued + **kwargs: see auto generated CLI items below. Unmatched keys will be ignored + a warning will be issued Returns: None (side effect: updates the global configuration object) """ @@ -206,6 +203,7 @@ def parse_config_files(**kwargs) -> None: config.clear() for config_file in reversed(config_src): config.set_file(config_file) + config["config_src"] = config_src # deal with the scenario overrides scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs[k]} diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index b60b80b9a..1c4926cf2 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -156,10 +156,10 @@ ## @cond -import time, os, itertools, warnings +import time, warnings -from . import seir, outcomes, model_info, file_paths -from .utils import config, as_list, profile +from . import seir, outcomes, model_info +from .utils import config #, profile from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli, click_helpstring # from .profile import profile_options @@ -168,21 +168,7 @@ # @profile() @cli.command(params=[config_files_argument] + list(config_file_options.values())) @click_helpstring([config_files_argument] + list(config_file_options.values())) -def simulate( - config_files, - config_filepath, - in_run_id, - out_run_id, - seir_modifiers_scenarios, - outcome_modifiers_scenarios, - in_prefix, - nslots, - jobs, - write_csv, - write_parquet, - first_sim_index, - stoch_traj_flag, -): +def simulate(**kwargs) -> None: """ Forward simulate a model using gempyor. @@ -190,8 +176,7 @@ def simulate( Returns: None (side effect: writes output to disk) """ - largs = locals() - parse_config_files(**largs) + parse_config_files(**kwargs) scenarios_combinations = [ [s, d] for s in (config["seir_modifiers"]["scenarios"].as_str_seq() if config["seir_modifiers"].exists() else [None]) @@ -214,31 +199,31 @@ def simulate( nslots=nslots, seir_modifiers_scenario=seir_modifiers_scenario, outcome_modifiers_scenario=outcome_modifiers_scenario, - write_csv=write_csv, - write_parquet=write_parquet, - first_sim_index=first_sim_index, - in_run_id=in_run_id, + write_csv=config["write_csv"].get(bool), + write_parquet=config["write_parquet"].get(bool), + first_sim_index=config["first_sim_index"].get(int), + in_run_id=config["in_run_id"].get(str) if config["in_run_id"].exists() else None, # in_prefix=config["name"].get() + "/", - out_run_id=out_run_id, + out_run_id=config["out_run_id"].get(str) if config["out_run_id"].exists() else None, # out_prefix=config["name"].get() + "/" + str(seir_modifiers_scenario) + "/" + out_run_id + "/", - stoch_traj_flag=stoch_traj_flag, - config_filepath=config_filepath, + stoch_traj_flag=config["stoch_traj_flag"].get(bool), + config_filepath=config["config_src"].as_str_seq(), ) print( f""" - >> Running from config {config_filepath} - >> Starting {modinf.nslots} model runs beginning from {modinf.first_sim_index} on {jobs} processes + >> Running from config {config["config_src"].as_str_seq()} + >> Starting {modinf.nslots} model runs beginning from {modinf.first_sim_index} on {config["jobs"].get(int)} processes >> ModelInfo *** {modinf.setup_name} *** from {modinf.ti} to {modinf.tf} >> Running scenario {seir_modifiers_scenario}_{outcome_modifiers_scenario} - >> running ***{'STOCHASTIC' if stoch_traj_flag else 'DETERMINISTIC'}*** trajectories + >> running ***{'STOCHASTIC' if config["stoch_traj_flag"].get(bool) else 'DETERMINISTIC'}*** trajectories """ ) # (there should be a run function) if config["seir"].exists(): - seir.run_parallel_SEIR(modinf, config=config, n_jobs=jobs) + seir.run_parallel_SEIR(modinf, config=config, n_jobs=config["jobs"].get(int)) if config["outcomes"].exists(): - outcomes.run_parallel_outcomes(sim_id2write=first_sim_index, modinf=modinf, nslots=nslots, n_jobs=jobs) + outcomes.run_parallel_outcomes(sim_id2write=config["first_sim_index"].get(int), modinf=modinf, nslots=nslots, n_jobs=config["jobs"].get(int)) print( f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" ) From dfa380cb7215c47abc226c5467e84d0e7fa761c0 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 13:33:14 -0400 Subject: [PATCH 32/53] adding gempyor-simulate deprecation --- flepimop/gempyor_pkg/setup.cfg | 6 +++--- flepimop/gempyor_pkg/src/gempyor/simulate.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index e5fb0902a..356bf5794 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -48,10 +48,10 @@ aws = [options.entry_points] console_scripts = - gempyor-outcomes = gempyor.simulate_outcome:simulate + gempyor-outcomes = gempyor.simulate_outcome:_deprecated_simulate flepimop = gempyor.cli:cli - gempyor-seir = gempyor.simulate_seir:simulate - gempyor-simulate = gempyor.simulate:simulate + gempyor-seir = gempyor.simulate_seir:_deprecated_simulate + gempyor-simulate = gempyor.simulate:_deprecated_simulate flepimop-calibrate = gempyor.calibrate:calibrate [options.packages.find] diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 1c4926cf2..5303e4ffa 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -168,13 +168,13 @@ # @profile() @cli.command(params=[config_files_argument] + list(config_file_options.values())) @click_helpstring([config_files_argument] + list(config_file_options.values())) -def simulate(**kwargs) -> None: +def simulate(**kwargs) -> int: """ Forward simulate a model using gempyor. Args: (see auto generated CLI items below) - Returns: None (side effect: writes output to disk) + Returns: exit code (side effect: writes output to disk) """ parse_config_files(**kwargs) @@ -227,9 +227,14 @@ def simulate(**kwargs) -> None: print( f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" ) + return 0 -if __name__ == "__main__": +def _deprecated_simulate(*args: list[str]) -> int: warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) - simulate() + cli(['simulate'].extend(args), standalone_mode=False) +_deprecated_simulate.__doc__ = simulate.__doc__ + +if __name__ == "__main__": + _deprecated_simulate() ## @endcond From bbf03770fc70a1903b5c9f7e7c01f634b15dc6d0 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 14:30:21 -0400 Subject: [PATCH 33/53] deprecate gempyor-simulate --- flepimop/gempyor_pkg/setup.cfg | 4 ++-- flepimop/gempyor_pkg/src/gempyor/simulate.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index 356bf5794..4b377001a 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -48,9 +48,9 @@ aws = [options.entry_points] console_scripts = - gempyor-outcomes = gempyor.simulate_outcome:_deprecated_simulate + gempyor-outcomes = gempyor.simulate_outcome:simulate flepimop = gempyor.cli:cli - gempyor-seir = gempyor.simulate_seir:_deprecated_simulate + gempyor-seir = gempyor.simulate_seir:simulate gempyor-simulate = gempyor.simulate:_deprecated_simulate flepimop-calibrate = gempyor.calibrate:calibrate diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 5303e4ffa..5140ee5dc 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -156,7 +156,7 @@ ## @cond -import time, warnings +import time, warnings, sys from . import seir, outcomes, model_info from .utils import config #, profile @@ -229,12 +229,16 @@ def simulate(**kwargs) -> int: ) return 0 -def _deprecated_simulate(*args: list[str]) -> int: +def _deprecated_simulate(argv : list[str] = []) -> int: warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) - cli(['simulate'].extend(args), standalone_mode=False) + if not argv: + argv = sys.argv[1:] + clickcmd = ['simulate'] + argv + cli(clickcmd, standalone_mode=False) + _deprecated_simulate.__doc__ = simulate.__doc__ if __name__ == "__main__": - _deprecated_simulate() + _deprecated_simulate(sys.argv[1:]) ## @endcond From 62bc7f0b90203d7200428c800f5befca6e429651 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 15:47:29 -0400 Subject: [PATCH 34/53] stub in parse config [skip ci] --- .../shared_cli/test_parse_config_files.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py new file mode 100644 index 000000000..c79f885e5 --- /dev/null +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -0,0 +1,47 @@ + +import yaml +import pathlib +from typing import Callable, Any + +import pytest + +from gempyor.shared_cli import parse_config_files + +class TestParseConfigFiles: + + def test_deprecated_config( + self, + tmp_path: pathlib.Path, + factory: Callable[[pathlib.Path], Any], + ) -> None: + pass + + def test_preferred_config( + self, + tmp_path: pathlib.Path, + factory: Callable[[pathlib.Path], Any], + ) -> None: + pass + + def test_conflict_config_opts_error( + self, + tmp_path: pathlib.Path, + factory: Callable[[pathlib.Path], Any], + ) -> None: + pass + + def test_multifile_config( + self, + tmp_path: pathlib.Path, + factory: Callable[[pathlib.Path], Any], + ) -> None: + pass + + # for all the options: + # - test the default + # - test the envvar + # - test invalid values => error + # - test valid values => present in config + # - test override: if not present in config => assigned + # - test override: if present in config => not default (i.e. actually provided) overridden + # - test override: if present in config => default => not overridden From 5c4bff8715d5059a2bcc4cf7cbe9e10d22ee0e3f Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 16 Oct 2024 16:08:30 -0400 Subject: [PATCH 35/53] leverage click.Parameter typecasting / validation --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 8ff889bc3..8ac9643ee 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -172,8 +172,13 @@ def decorator(func): # to also apply the `@click_helpstring` decorator to the command. Possibly to also default the params argument, assuming # enough commands have consistent option set? +# TODO: have parse_config_files check with the click.Parameter validators: +# https://stackoverflow.com/questions/59096020/how-to-unit-test-function-that-requires-an-active-click-context-in-python + +mock_context = click.Context(click.Command('mock'), info_name="Mock context for non-click use of parse_config_files") + @click_helpstring([config_files_argument] + list(config_file_options.values())) -def parse_config_files(**kwargs) -> None: +def parse_config_files(ctx = mock_context, **kwargs) -> None: """ Parse configuration file(s) and override with command line arguments @@ -199,7 +204,9 @@ def parse_config_files(**kwargs) -> None: error_dict = {k: kwargs[k] for k in found_configs} raise ValueError(f"Exactly one config file source option must be provided; got {error_dict}.") else: - config_src = kwargs[found_configs[0]] + config_key = found_configs[0] + config_validator = config_file_options[config_key] if config_key in config_file_options else config_files_argument + config_src = config_validator.type_cast_value(ctx, kwargs[config_key]) config.clear() for config_file in reversed(config_src): config.set_file(config_file) @@ -209,7 +216,7 @@ def parse_config_files(**kwargs) -> None: scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs[k]} for option in scen_args: key = option.replace("_scenarios", "") - value = kwargs[option] + value = config_file_options[option].type_cast_value(ctx, kwargs[option]) if config[key].exists(): config[key]["scenarios"] = as_list(value) else: @@ -219,4 +226,4 @@ def parse_config_files(**kwargs) -> None: other_args = parsed_args - config_args - scen_args for option in other_args: if (value := kwargs[option]) is not None: - config[option] = value + config[option] = config_file_options[option].type_cast_value(ctx, value) From 214f9581b2c884107cd2877dcce67e1b32bca4b9 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Fri, 18 Oct 2024 16:18:47 -0400 Subject: [PATCH 36/53] introduce passing click context if available --- examples/test_cli.py | 14 ++++----- flepimop/gempyor_pkg/setup.cfg | 2 -- flepimop/gempyor_pkg/src/gempyor/cli.py | 13 ++++---- .../gempyor_pkg/src/gempyor/compartments.py | 25 +++++++++------- .../gempyor_pkg/src/gempyor/shared_cli.py | 17 ++++++++--- flepimop/gempyor_pkg/src/gempyor/simulate.py | 30 ++++++++++++++----- 6 files changed, 64 insertions(+), 37 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index 1f9bfdebb..628210457 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -3,7 +3,7 @@ from click.testing import CliRunner -from gempyor.simulate import simulate +from gempyor.simulate import _click_simulate # See here to test click application https://click.palletsprojects.com/en/8.1.x/testing/ # would be useful to also call the command directly @@ -12,7 +12,7 @@ def test_config_sample_2pop(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ["config_sample_2pop.yml"]) + result = runner.invoke(_click_simulate, ["config_sample_2pop.yml"]) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug @@ -23,7 +23,7 @@ def test_config_sample_2pop(): def test_config_sample_2pop_deprecated(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ["-c", "config_sample_2pop.yml"]) + result = runner.invoke(_click_simulate, ["-c", "config_sample_2pop.yml"]) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug @@ -35,7 +35,7 @@ def test_sample_2pop_modifiers(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() result = runner.invoke( - simulate, + _click_simulate, [ "config_sample_2pop.yml", "config_sample_2pop_outcomes_part.yml", @@ -52,7 +52,7 @@ def test_sample_2pop_modifiers(): def test_sample_2pop_modifiers_combined(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ["config_sample_2pop_modifiers.yml"]) + result = runner.invoke(_click_simulate, ["config_sample_2pop_modifiers.yml"]) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug @@ -63,7 +63,7 @@ def test_sample_2pop_modifiers_combined(): def test_sample_2pop_modifiers_combined_deprecated(): os.chdir(os.path.dirname(__file__) + "/tutorials") runner = CliRunner() - result = runner.invoke(simulate, ["-c", "config_sample_2pop_modifiers.yml"]) + result = runner.invoke(_click_simulate, ["-c", "config_sample_2pop_modifiers.yml"]) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug @@ -74,7 +74,7 @@ def test_sample_2pop_modifiers_combined_deprecated(): def test_simple_usa_statelevel_deprecated(): os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") runner = CliRunner() - result = runner.invoke(simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"]) + result = runner.invoke(_click_simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"]) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index 4b377001a..e7dea1d0e 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -48,9 +48,7 @@ aws = [options.entry_points] console_scripts = - gempyor-outcomes = gempyor.simulate_outcome:simulate flepimop = gempyor.cli:cli - gempyor-seir = gempyor.simulate_seir:simulate gempyor-simulate = gempyor.simulate:_deprecated_simulate flepimop-calibrate = gempyor.calibrate:calibrate diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index b2693f6da..6c77e99b9 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -1,14 +1,17 @@ +from click import pass_context, Context + from .shared_cli import ( config_files_argument, config_file_options, parse_config_files, cli, + mock_context, ) from .utils import config # register the commands from the other modules -from .compartments import compartments -from .simulate import simulate +from . import compartments +from . import simulate # Guidance for extending the CLI: # - to add a new small command to the CLI, add a new function with the @cli.command() decorator here (e.g. patch below) @@ -18,11 +21,11 @@ # add some basic commands to the CLI @cli.command(params=[config_files_argument].extend(config_file_options.values())) -def patch(**kwargs) -> None: +@pass_context +def patch(ctx : Context = mock_context, **kwargs) -> None: """Merge configuration files""" - parse_config_files(**kwargs) + parse_config_files(ctx, **kwargs) print(config.dump()) - if __name__ == "__main__": cli() diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index 2e7268b29..f5dfd4c23 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -5,6 +5,7 @@ import pyarrow as pa import pyarrow.parquet as pq import logging +from click import pass_context, Context from .utils import config, Timer, as_list from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli @@ -697,8 +698,9 @@ def filter_func(lst, this_filter=[]): ) + "\n}" ) + src = graphviz.Source(graph_description) - src.render(output_file, view=True) + src.render(output_file) def get_list_dimension(thing): @@ -746,14 +748,16 @@ def list_recursive_convert_to_string(thing): return str(thing) @cli.group() -def compartments(): +@pass_context +def compartments(ctx: Context): """Commands for working with FlepiMoP compartments""" pass -@compartments.command(params=[config_files_argument].extend(config_file_options.values())) -def plot(**kwargs): +@compartments.command(params=[config_files_argument] + list(config_file_options.values())) +@pass_context +def plot(ctx : Context, **kwargs): """Plot compartments""" - parse_config_files(**kwargs) + parse_config_files(ctx, **kwargs) assert config["compartments"].exists() assert config["seir"].exists() comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) @@ -768,13 +772,12 @@ def plot(**kwargs): comp.plot(output_file="transition_graph", source_filters=[], destination_filters=[]) - print("wrote file transition_graph") - -@compartments.command(params=[config_files_argument].extend(config_file_options.values())) -def export(**kwargs): +@compartments.command(params=[config_files_argument] + list(config_file_options.values())) +@pass_context +def export(ctx : Context, **kwargs): """Export compartments""" - parse_config_files(**kwargs) + parse_config_files(ctx, **kwargs) assert config["compartments"].exists() assert config["seir"].exists() comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) @@ -787,4 +790,4 @@ def export(**kwargs): comp.toFile("compartments_file.csv", "transitions_file.csv", write_parquet=False) print("wrote files 'compartments_file.csv', 'transitions_file.csv' ") -cli.add_command(compartments) +cli.add_command(compartments) \ No newline at end of file diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 8ac9643ee..cf50ad8c7 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -15,10 +15,19 @@ __all__ = [] @click.group() -def cli(): +@click.pass_context +def cli(ctx: click.Context) -> None: """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass +output_option = click.Option( + ["-o", "--output-file"], + type=click.Path(allow_dash=True), + is_flag=False, flag_value="-", + default="transition_graph.pdf", show_default=True, + help="output file path", +) + # click decorator to handle configuration file(s) as arguments # use as `@argument_config_files` before a cli command definition config_files_argument = click.Argument( @@ -195,7 +204,7 @@ def parse_config_files(ctx = mock_context, **kwargs) -> None: # initialize the config, including handling missing / double-specified config files config_args = {k for k in parsed_args if k.startswith("config")} - found_configs = [k for k in config_args if kwargs[k]] + found_configs = [k for k in config_args if kwargs.get(k)] config_src = [] if len(found_configs) != 1: if not found_configs: @@ -213,7 +222,7 @@ def parse_config_files(ctx = mock_context, **kwargs) -> None: config["config_src"] = config_src # deal with the scenario overrides - scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs[k]} + scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs.get(k)} for option in scen_args: key = option.replace("_scenarios", "") value = config_file_options[option].type_cast_value(ctx, kwargs[option]) @@ -225,5 +234,5 @@ def parse_config_files(ctx = mock_context, **kwargs) -> None: # update the config with the remaining options other_args = parsed_args - config_args - scen_args for option in other_args: - if (value := kwargs[option]) is not None: + if (value := kwargs.get(option)) is not None: config[option] = config_file_options[option].type_cast_value(ctx, value) diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 5140ee5dc..a0d500f6f 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -158,17 +158,18 @@ import time, warnings, sys +from click import Context, pass_context + from . import seir, outcomes, model_info from .utils import config #, profile -from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli, click_helpstring +from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli, click_helpstring, mock_context # from .profile import profile_options # @profile_options # @profile() -@cli.command(params=[config_files_argument] + list(config_file_options.values())) @click_helpstring([config_files_argument] + list(config_file_options.values())) -def simulate(**kwargs) -> int: +def simulate(ctx : Context = mock_context, **kwargs) -> int: """ Forward simulate a model using gempyor. @@ -176,7 +177,7 @@ def simulate(**kwargs) -> int: Returns: exit code (side effect: writes output to disk) """ - parse_config_files(**kwargs) + parse_config_files(ctx, **kwargs) scenarios_combinations = [ [s, d] for s in (config["seir_modifiers"]["scenarios"].as_str_seq() if config["seir_modifiers"].exists() else [None]) @@ -229,16 +230,29 @@ def simulate(**kwargs) -> int: ) return 0 +@cli.command(name="simulate", params=[config_files_argument] + list(config_file_options.values())) +@pass_context +def _click_simulate(ctx : Context, **kwargs) -> int: + return simulate(ctx, **kwargs) + + +# will all be removed upon deprecated endpoint removal + +import subprocess + def _deprecated_simulate(argv : list[str] = []) -> int: - warnings.warn("This function is deprecated, use the CLI instead: `flepimop simulate ...`", DeprecationWarning) if not argv: argv = sys.argv[1:] - clickcmd = ['simulate'] + argv - cli(clickcmd, standalone_mode=False) + clickcmd = ' '.join(['flepimop', 'simulate'] + argv) + warnings.warn(f"This command is deprecated, use the CLI instead: `{clickcmd}`", DeprecationWarning) + return subprocess.run(clickcmd, shell=True).returncode _deprecated_simulate.__doc__ = simulate.__doc__ if __name__ == "__main__": - _deprecated_simulate(sys.argv[1:]) + argv = sys.argv[1:] + clickcmd = ' '.join(['flepimop', 'simulate'] + argv) + warnings.warn(f"Use the CLI instead: `{clickcmd}`", DeprecationWarning) + _deprecated_simulate(argv) ## @endcond From f8e6052aa6d2fa5e8e99e80ee0d876bd556be0d2 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 10:59:27 -0400 Subject: [PATCH 37/53] add unit testing for parse config --- flepimop/gempyor_pkg/src/gempyor/cli.py | 2 +- .../gempyor_pkg/src/gempyor/compartments.py | 4 +- .../gempyor_pkg/src/gempyor/shared_cli.py | 49 ++++++------ flepimop/gempyor_pkg/src/gempyor/simulate.py | 2 +- flepimop/gempyor_pkg/src/gempyor/testing.py | 49 ++++++++++++ .../shared_cli/test_parse_config_files.py | 79 +++++++++++++++++-- 6 files changed, 147 insertions(+), 38 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 6c77e99b9..84ad05503 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -24,7 +24,7 @@ @pass_context def patch(ctx : Context = mock_context, **kwargs) -> None: """Merge configuration files""" - parse_config_files(ctx, **kwargs) + parse_config_files(config, ctx, **kwargs) print(config.dump()) if __name__ == "__main__": diff --git a/flepimop/gempyor_pkg/src/gempyor/compartments.py b/flepimop/gempyor_pkg/src/gempyor/compartments.py index f5dfd4c23..0a3ee5332 100644 --- a/flepimop/gempyor_pkg/src/gempyor/compartments.py +++ b/flepimop/gempyor_pkg/src/gempyor/compartments.py @@ -757,7 +757,7 @@ def compartments(ctx: Context): @pass_context def plot(ctx : Context, **kwargs): """Plot compartments""" - parse_config_files(ctx, **kwargs) + parse_config_files(config, ctx, **kwargs) assert config["compartments"].exists() assert config["seir"].exists() comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) @@ -777,7 +777,7 @@ def plot(ctx : Context, **kwargs): @pass_context def export(ctx : Context, **kwargs): """Export compartments""" - parse_config_files(ctx, **kwargs) + parse_config_files(config, ctx, **kwargs) assert config["compartments"].exists() assert config["seir"].exists() comp = Compartments(seir_config=config["seir"], compartments_config=config["compartments"]) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index cf50ad8c7..706be231c 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -9,6 +9,7 @@ from typing import Callable, Any import click +import confuse from .utils import config, as_list @@ -20,18 +21,10 @@ def cli(ctx: click.Context) -> None: """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass -output_option = click.Option( - ["-o", "--output-file"], - type=click.Path(allow_dash=True), - is_flag=False, flag_value="-", - default="transition_graph.pdf", show_default=True, - help="output file path", -) - # click decorator to handle configuration file(s) as arguments # use as `@argument_config_files` before a cli command definition config_files_argument = click.Argument( - ["config_files"], nargs=-1, type=click.Path(exists=True) + ["config_files"], nargs=-1, type=click.Path(exists=True, path_type=pathlib.Path) ) # List of standard `click` options that override/update config settings @@ -40,9 +33,8 @@ def cli(ctx: click.Context) -> None: "config_filepath" : click.Option( ["-c", "--config", "config_filepath"], envvar="CONFIG_PATH", - type=click.Path(exists=True), + type=click.Path(exists=True, path_type=pathlib.Path), required=False, - default=[], # deprecated = ["-c", "--config"], # preferred = "CONFIG_FILES...", multiple=True, @@ -181,26 +173,30 @@ def decorator(func): # to also apply the `@click_helpstring` decorator to the command. Possibly to also default the params argument, assuming # enough commands have consistent option set? -# TODO: have parse_config_files check with the click.Parameter validators: -# https://stackoverflow.com/questions/59096020/how-to-unit-test-function-that-requires-an-active-click-context-in-python - mock_context = click.Context(click.Command('mock'), info_name="Mock context for non-click use of parse_config_files") +def _parse_option(ctx: click.Context, param : click.Parameter, value: Any) -> Any: + if ((param.multiple or param.nargs == -1) and not isinstance(value, list)): + value = [value] + return param.type_cast_value(ctx, value) + @click_helpstring([config_files_argument] + list(config_file_options.values())) -def parse_config_files(ctx = mock_context, **kwargs) -> None: +def parse_config_files(cfg : confuse.Configuration = config, ctx : click.Context = mock_context, **kwargs) -> None: """ Parse configuration file(s) and override with command line arguments Args: + cfg: the configuration object to update; defaults to global `.utils.config` + ctx: the click context (used for type casting); defaults to a mock context for non-click use; pass actual context if using in a click command **kwargs: see auto generated CLI items below. Unmatched keys will be ignored + a warning will be issued - Returns: None (side effect: updates the global configuration object) + Returns: None; side effect: updates the `cfg` argument """ parsed_args = {config_files_argument.name}.union({option.name for option in config_file_options.values()}) # warn re unrecognized arguments - if parsed_args.difference(kwargs.keys()): - warnings.warn(f"Unused arguments: {parsed_args.difference(kwargs.keys())}") + if (unknownargs := [k for k in parsed_args.difference(kwargs.keys()) if kwargs.get(k) is not None]): + warnings.warn(f"Unused arguments: {unknownargs}") # initialize the config, including handling missing / double-specified config files config_args = {k for k in parsed_args if k.startswith("config")} @@ -215,19 +211,19 @@ def parse_config_files(ctx = mock_context, **kwargs) -> None: else: config_key = found_configs[0] config_validator = config_file_options[config_key] if config_key in config_file_options else config_files_argument - config_src = config_validator.type_cast_value(ctx, kwargs[config_key]) - config.clear() + config_src = _parse_option(ctx, config_validator, kwargs[config_key]) + cfg.clear() for config_file in reversed(config_src): - config.set_file(config_file) - config["config_src"] = config_src + cfg.set_file(config_file) + cfg["config_src"] = [str(k) for k in config_src] # deal with the scenario overrides scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs.get(k)} for option in scen_args: key = option.replace("_scenarios", "") - value = config_file_options[option].type_cast_value(ctx, kwargs[option]) - if config[key].exists(): - config[key]["scenarios"] = as_list(value) + value = _parse_option(ctx, config_file_options[option], kwargs[option]) + if cfg[key].exists(): + cfg[key]["scenarios"] = as_list(value) else: raise ValueError(f"Specified {option} when no {key} in configuration file(s): {config_src}") @@ -235,4 +231,5 @@ def parse_config_files(ctx = mock_context, **kwargs) -> None: other_args = parsed_args - config_args - scen_args for option in other_args: if (value := kwargs.get(option)) is not None: - config[option] = config_file_options[option].type_cast_value(ctx, value) + # auto box the value if the option expects a multiple + cfg[option] = _parse_option(ctx, config_file_options[option], value) diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index a0d500f6f..c46ce34b0 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -177,7 +177,7 @@ def simulate(ctx : Context = mock_context, **kwargs) -> int: Returns: exit code (side effect: writes output to disk) """ - parse_config_files(ctx, **kwargs) + parse_config_files(config, ctx, **kwargs) scenarios_combinations = [ [s, d] for s in (config["seir_modifiers"]["scenarios"].as_str_seq() if config["seir_modifiers"].exists() else [None]) diff --git a/flepimop/gempyor_pkg/src/gempyor/testing.py b/flepimop/gempyor_pkg/src/gempyor/testing.py index 8e7e4d3c2..e6b4b7046 100644 --- a/flepimop/gempyor_pkg/src/gempyor/testing.py +++ b/flepimop/gempyor_pkg/src/gempyor/testing.py @@ -7,7 +7,10 @@ __all__ = [ "change_directory_to_temp_directory", + "mock_empty_config", + "create_confuse_config_from_file", "create_confuse_configview_from_dict", + "create_confuse_config_from_dict", "partials_are_similar", "sample_fits_distribution", ] @@ -17,6 +20,7 @@ import os from tempfile import TemporaryDirectory from typing import Any, Literal +from pathlib import Path import confuse import numpy as np @@ -43,6 +47,33 @@ def change_directory_to_temp_directory() -> Generator[None, None, None]: temp_dir.cleanup() +def mock_empty_config() -> confuse.Configuration: + """ + Create a `confuse.Configuration` (akin to `gempyor.utils.config`) with no data for + unit testing configurations. + + Returns: + A `confuse.Configuration`. + """ + return confuse.Configuration("flepiMoPMock", read=False) + +def create_confuse_config_from_file( + data_file: Path, +) -> confuse.Configuration: + """ + Create a `confuse.Configuration` (akin to `gempyor.utils.config`) from a file for + unit testing configurations. + + Args: + data_file: The file to populate the confuse ConfigView with. + + Returns: + A `confuse.Configuration`. + """ + cv = mock_empty_config() + cv.set_file(data_file) + return cv + def create_confuse_configview_from_dict( data: dict[str, Any], name: None | str = None ) -> confuse.ConfigView: @@ -102,6 +133,24 @@ def create_confuse_configview_from_dict( cv = cv[name] if name is not None else cv return cv +def create_confuse_config_from_dict( + data: dict[str, Any] +) -> confuse.Configuration: + """ + Create a Configuration from a dictionary for unit testing confuse parameters. + + Args: + data: The data to populate the confuse ConfigView with. + + Returns: + confuse Configuration + + + """ + cfg = mock_empty_config() + cfg.set_args(data) + return cfg + def partials_are_similar( f: functools.partial, diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py index c79f885e5..04e8b891a 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -7,35 +7,98 @@ from gempyor.shared_cli import parse_config_files +from gempyor.testing import * + +def config_file( + tmp_path: pathlib.Path, + config_dict: dict[str, Any], + filename: str = "config.yaml", +) -> pathlib.Path: + config_file = tmp_path / filename + with open(config_file, "w") as f: + f.write(create_confuse_config_from_dict(config_dict).dump()) + return config_file + +# collection of bad option values +def bad_opt_args(opt : str) -> Any: + return { + "write_csv": "foo", "write_parquet": "bar", "first_sim_index": -1, + "config_filepath": -1, "seir_modifiers_scenario": 1, "outcome_modifiers_scenario": 1, + "jobs": -1, "nslots": -1, "stoch_traj_flag": "foo", "in_run_id": 42, "out_run_id": 42, + "in_prefix": 42, + }[opt] + +# collection of good option values +def good_opt_args(opt : str) -> Any: + return { + "write_csv": False, "write_parquet": False, "first_sim_index": 42, + "config_filepath": "foo", "seir_modifiers_scenario": "example", "outcome_modifiers_scenario": "example", + "jobs": 10, "nslots": 10, "stoch_traj_flag": False, "in_run_id": "foo", "out_run_id": "foo", + "in_prefix": "foo", + }[opt] + +# collection of good configuration entries, to be overwritten by good_opt_args +def ref_cfg_kvs(opt : str) -> Any: + return { opt: { + "write_csv": True, "write_parquet": True, "first_sim_index": 1, + "config_filepath": "notfoo", "seir_modifiers_scenario": ["example", "ibid"], "outcome_modifiers_scenario": ["example", "ibid"], + "jobs": 1, "nslots": 1, "stoch_traj_flag": True, "in_run_id": "bar", "out_run_id": "bar", + "in_prefix": "bar", + }[opt]} + class TestParseConfigFiles: def test_deprecated_config( self, tmp_path: pathlib.Path, - factory: Callable[[pathlib.Path], Any], ) -> None: - pass + """Check that a -c config file work.""" + testdict = {"foo": "bar", "test": 123 } + tmpconfigfile = config_file(tmp_path, testdict) + mockconfig = mock_empty_config() + parse_config_files(mockconfig, config_filepath=tmpconfigfile) + for k, v in testdict.items(): + assert mockconfig[k].get(v) == v + assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile)] def test_preferred_config( self, tmp_path: pathlib.Path, - factory: Callable[[pathlib.Path], Any], ) -> None: - pass + """Check that a -c config file work.""" + testdict = {"foo": "bar", "test": 123 } + tmpconfigfile = config_file(tmp_path, testdict) + mockconfig = mock_empty_config() + parse_config_files(mockconfig, config_files=tmpconfigfile) + for k, v in testdict.items(): + assert mockconfig[k].get(v) == v + assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile)] def test_conflict_config_opts_error( self, tmp_path: pathlib.Path, - factory: Callable[[pathlib.Path], Any], ) -> None: - pass + """Check that both -c and argument style config file raise an error.""" + testdict = {"foo": "bar", "test": 123 } + tmpconfigfile = config_file(tmp_path, testdict) + mockconfig = mock_empty_config() + with pytest.raises(ValueError): + parse_config_files(mockconfig, config_filepath=tmpconfigfile, config_files=tmpconfigfile) def test_multifile_config( self, tmp_path: pathlib.Path, - factory: Callable[[pathlib.Path], Any], ) -> None: - pass + """Check that multiple config files are merged.""" + testdict1 = {"foo": "bar", "test": 123 } + testdict2 = {"bar": "baz" } + tmpconfigfile1 = config_file(tmp_path, testdict1, "config1.yaml") + tmpconfigfile2 = config_file(tmp_path, testdict2, "config2.yaml") + mockconfig = mock_empty_config() + parse_config_files(mockconfig, config_files=[tmpconfigfile1, tmpconfigfile2]) + for k, v in (testdict1 | testdict2).items(): + assert mockconfig[k].get(v) == v + assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile1), str(tmpconfigfile2)] # for all the options: # - test the default From 70149381d01e87eae68d75d599897f9fe39d88da Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 11:39:09 -0400 Subject: [PATCH 38/53] handle tuple case in shared_cli --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 706be231c..dc0e05e79 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -175,11 +175,6 @@ def decorator(func): mock_context = click.Context(click.Command('mock'), info_name="Mock context for non-click use of parse_config_files") -def _parse_option(ctx: click.Context, param : click.Parameter, value: Any) -> Any: - if ((param.multiple or param.nargs == -1) and not isinstance(value, list)): - value = [value] - return param.type_cast_value(ctx, value) - @click_helpstring([config_files_argument] + list(config_file_options.values())) def parse_config_files(cfg : confuse.Configuration = config, ctx : click.Context = mock_context, **kwargs) -> None: """ @@ -192,6 +187,12 @@ def parse_config_files(cfg : confuse.Configuration = config, ctx : click.Context Returns: None; side effect: updates the `cfg` argument """ + def _parse_option(param : click.Parameter, value: Any) -> Any: + """internal parser to autobox values""" + if ((param.multiple or param.nargs == -1) and not isinstance(value, (list, tuple))): + value = [value] + return param.type_cast_value(ctx, value) + parsed_args = {config_files_argument.name}.union({option.name for option in config_file_options.values()}) # warn re unrecognized arguments From c9f917e585421814a9a44c38b45a1ef9d549255f Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 13:34:02 -0400 Subject: [PATCH 39/53] full option testing --- .../gempyor_pkg/src/gempyor/shared_cli.py | 34 ++++++-- .../shared_cli/test_parse_config_files.py | 78 +++++++++++++------ 2 files changed, 81 insertions(+), 31 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index dc0e05e79..cd200dfb6 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -7,6 +7,7 @@ import pathlib import warnings from typing import Callable, Any +import re import click import confuse @@ -21,6 +22,23 @@ def cli(ctx: click.Context) -> None: """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass + +class AlphanumericParamType(click.ParamType): + """A custom click parameter type for alphanumeric strings""" + name = "alphanumeric" + an_pattern = re.compile("^[a-zA-Z0-9]+$") + + def convert(self, value, param, ctx): + if not isinstance(value, str): + value = str(value) + if not self.an_pattern.match(value): + self.fail(f"{value!r} is not a valid alphanumeric value; must only have [a-zA-Z0-9] elements.", param, ctx) + else: + return value + +AN_STR = AlphanumericParamType() + + # click decorator to handle configuration file(s) as arguments # use as `@argument_config_files` before a cli command definition config_files_argument = click.Argument( @@ -43,7 +61,7 @@ def cli(ctx: click.Context) -> None: "seir_modifiers_scenarios": click.Option( ["-s", "--seir_modifiers_scenarios"], envvar="FLEPI_SEIR_SCENARIO", - type=str, + type=AN_STR, default=[], multiple=True, help="override/select the transmission scenario(s) to run", @@ -51,7 +69,7 @@ def cli(ctx: click.Context) -> None: "outcome_modifiers_scenarios": click.Option( ["-d", "--outcome_modifiers_scenarios"], envvar="FLEPI_OUTCOME_SCENARIO", - type=str, + type=AN_STR, default=[], multiple=True, help="override/select the outcome scenario(s) to run", @@ -73,21 +91,21 @@ def cli(ctx: click.Context) -> None: "in_run_id": click.Option( ["--in-id", "in_run_id"], envvar="FLEPI_RUN_INDEX", - type=str, + type=AN_STR, show_default=True, help="Unique identifier for the run", ), "out_run_id": click.Option( ["--out-id", "out_run_id"], envvar="FLEPI_RUN_INDEX", - type=str, + type=AN_STR, show_default=True, help="Unique identifier for the run", ), "in_prefix": click.Option( ["--in-prefix"], envvar="FLEPI_PREFIX", - type=str, + type=AN_STR, default=None, show_default=True, help="unique identifier for the run", @@ -212,7 +230,7 @@ def _parse_option(param : click.Parameter, value: Any) -> Any: else: config_key = found_configs[0] config_validator = config_file_options[config_key] if config_key in config_file_options else config_files_argument - config_src = _parse_option(ctx, config_validator, kwargs[config_key]) + config_src = _parse_option(config_validator, kwargs[config_key]) cfg.clear() for config_file in reversed(config_src): cfg.set_file(config_file) @@ -222,7 +240,7 @@ def _parse_option(param : click.Parameter, value: Any) -> Any: scen_args = {k for k in parsed_args if k.endswith("scenarios") and kwargs.get(k)} for option in scen_args: key = option.replace("_scenarios", "") - value = _parse_option(ctx, config_file_options[option], kwargs[option]) + value = _parse_option(config_file_options[option], kwargs[option]) if cfg[key].exists(): cfg[key]["scenarios"] = as_list(value) else: @@ -233,4 +251,4 @@ def _parse_option(param : click.Parameter, value: Any) -> Any: for option in other_args: if (value := kwargs.get(option)) is not None: # auto box the value if the option expects a multiple - cfg[option] = _parse_option(ctx, config_file_options[option], value) + cfg[option] = _parse_option(config_file_options[option], value) diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py index 04e8b891a..66179545b 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -4,14 +4,14 @@ from typing import Callable, Any import pytest +import click -from gempyor.shared_cli import parse_config_files - +from gempyor.shared_cli import parse_config_files, config_file_options from gempyor.testing import * def config_file( tmp_path: pathlib.Path, - config_dict: dict[str, Any], + config_dict: dict[str, Any] = {}, filename: str = "config.yaml", ) -> pathlib.Path: config_file = tmp_path / filename @@ -19,29 +19,34 @@ def config_file( f.write(create_confuse_config_from_dict(config_dict).dump()) return config_file +other_single_opt_args = [ + "write_csv", "write_parquet", "first_sim_index", "jobs", "nslots", "stoch_traj_flag", + "in_run_id", "out_run_id", "in_prefix" +] + # collection of bad option values -def bad_opt_args(opt : str) -> Any: - return { +def bad_opt_args(opt : str) -> dict[str, Any]: + return { opt: { "write_csv": "foo", "write_parquet": "bar", "first_sim_index": -1, - "config_filepath": -1, "seir_modifiers_scenario": 1, "outcome_modifiers_scenario": 1, - "jobs": -1, "nslots": -1, "stoch_traj_flag": "foo", "in_run_id": 42, "out_run_id": 42, - "in_prefix": 42, - }[opt] + "seir_modifiers_scenario": 1, "outcome_modifiers_scenario": 1, + "jobs": -1, "nslots": -1, "stoch_traj_flag": "foo", "in_run_id": -1, "out_run_id": -1, + "in_prefix": [42], + }[opt] } # collection of good option values -def good_opt_args(opt : str) -> Any: - return { +def good_opt_args(opt : str) -> dict[str, Any]: + return { opt: { "write_csv": False, "write_parquet": False, "first_sim_index": 42, - "config_filepath": "foo", "seir_modifiers_scenario": "example", "outcome_modifiers_scenario": "example", + "seir_modifiers_scenario": "example", "outcome_modifiers_scenario": "example", "jobs": 10, "nslots": 10, "stoch_traj_flag": False, "in_run_id": "foo", "out_run_id": "foo", "in_prefix": "foo", - }[opt] + }[opt] } # collection of good configuration entries, to be overwritten by good_opt_args -def ref_cfg_kvs(opt : str) -> Any: +def ref_cfg_kvs(opt : str) -> dict[str, Any]: return { opt: { "write_csv": True, "write_parquet": True, "first_sim_index": 1, - "config_filepath": "notfoo", "seir_modifiers_scenario": ["example", "ibid"], "outcome_modifiers_scenario": ["example", "ibid"], + "seir_modifiers_scenario": ["example", "ibid"], "outcome_modifiers_scenario": ["example", "ibid"], "jobs": 1, "nslots": 1, "stoch_traj_flag": True, "in_run_id": "bar", "out_run_id": "bar", "in_prefix": "bar", }[opt]} @@ -100,11 +105,38 @@ def test_multifile_config( assert mockconfig[k].get(v) == v assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile1), str(tmpconfigfile2)] - # for all the options: - # - test the default - # - test the envvar - # - test invalid values => error - # - test valid values => present in config - # - test override: if not present in config => assigned - # - test override: if present in config => not default (i.e. actually provided) overridden - # - test override: if present in config => default => not overridden + @pytest.mark.parametrize("opt", [(k) for k in other_single_opt_args]) + def test_other_opts(self, tmp_path: pathlib.Path, opt: str) -> None: + """for the non-scenario modifier parameters, test default, envvar, invalid values, valid values, override""" + + goodopt = good_opt_args(opt) + badopt = bad_opt_args(opt) + refopt = ref_cfg_kvs(opt) + + # the config file, with option set + tmpconfigfile_wi_ref = config_file(tmp_path, refopt, "withref.yaml") + # the config, without option set + tmpconfigfile_wo_ref = config_file(tmp_path, filename="noref.yaml") + mockconfig = mock_empty_config() + + for cfg in [tmpconfigfile_wi_ref, tmpconfigfile_wo_ref]: + # both versions error on bad values + with pytest.raises(click.exceptions.BadParameter): + parse_config_files(mockconfig, config_files=cfg, **badopt) + # when supplied an override, both should have the override + parse_config_files(mockconfig, config_files=cfg, **goodopt) + for k, v in goodopt.items(): + assert mockconfig[k].get(v) == v + mockconfig.clear() + + # the config file with the option set should override the default + parse_config_files(mockconfig, config_files=tmpconfigfile_wi_ref) + for k, v in refopt.items(): + assert mockconfig[k].get(v) == v + mockconfig.clear() + + # the config file without the option set should adopt the default + parse_config_files(mockconfig, config_files=tmpconfigfile_wo_ref) + defopt = config_file_options[opt].default + if defopt is not None: + assert mockconfig[opt].get(defopt) == defopt From 0f879fdcb6dd9c2aaa68b8ca8b58ab0b584daacb Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 13:37:11 -0400 Subject: [PATCH 40/53] apply black formatting [skip ci] --- examples/test_cli.py | 4 +- .../gempyor_pkg/src/gempyor/shared_cli.py | 58 +++++++-- .../shared_cli/test_parse_config_files.py | 114 ++++++++++++------ 3 files changed, 127 insertions(+), 49 deletions(-) diff --git a/examples/test_cli.py b/examples/test_cli.py index 628210457..78234b610 100644 --- a/examples/test_cli.py +++ b/examples/test_cli.py @@ -74,7 +74,9 @@ def test_sample_2pop_modifiers_combined_deprecated(): def test_simple_usa_statelevel_deprecated(): os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel") runner = CliRunner() - result = runner.invoke(_click_simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"]) + result = runner.invoke( + _click_simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"] + ) print(result.output) # useful for debug print(result.exit_code) # useful for debug print(result.exception) # useful for debug diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index cd200dfb6..f71ec7f77 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -16,6 +16,7 @@ __all__ = [] + @click.group() @click.pass_context def cli(ctx: click.Context) -> None: @@ -25,6 +26,7 @@ def cli(ctx: click.Context) -> None: class AlphanumericParamType(click.ParamType): """A custom click parameter type for alphanumeric strings""" + name = "alphanumeric" an_pattern = re.compile("^[a-zA-Z0-9]+$") @@ -32,9 +34,14 @@ def convert(self, value, param, ctx): if not isinstance(value, str): value = str(value) if not self.an_pattern.match(value): - self.fail(f"{value!r} is not a valid alphanumeric value; must only have [a-zA-Z0-9] elements.", param, ctx) + self.fail( + f"{value!r} is not a valid alphanumeric value; must only have [a-zA-Z0-9] elements.", + param, + ctx, + ) else: - return value + return value + AN_STR = AlphanumericParamType() @@ -48,7 +55,7 @@ def convert(self, value, param, ctx): # List of standard `click` options that override/update config settings # n.b., the help for these options will be presented in the order defined here config_file_options = { - "config_filepath" : click.Option( + "config_filepath": click.Option( ["-c", "--config", "config_filepath"], envvar="CONFIG_PATH", type=click.Path(exists=True, path_type=pathlib.Path), @@ -138,6 +145,7 @@ def convert(self, value, param, ctx): ), } + def click_helpstring( params: click.Parameter | list[click.Parameter], ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: @@ -177,7 +185,9 @@ def decorator(func): additional_doc = "\n\tCommand Line Interface arguments:\n" for param in params: paraminfo = param.to_info_dict() - additional_doc += f"\n\t\t{paraminfo['name']}: {paraminfo['type']['param_type']}\n" + additional_doc += ( + f"\n\t\t{paraminfo['name']}: {paraminfo['type']['param_type']}\n" + ) if func.__doc__ is None: func.__doc__ = "" @@ -187,14 +197,21 @@ def decorator(func): return decorator + # TODO: create a custom command decorator cls ala: https://click.palletsprojects.com/en/8.1.x/advanced/#command-aliases # to also apply the `@click_helpstring` decorator to the command. Possibly to also default the params argument, assuming # enough commands have consistent option set? -mock_context = click.Context(click.Command('mock'), info_name="Mock context for non-click use of parse_config_files") +mock_context = click.Context( + click.Command("mock"), + info_name="Mock context for non-click use of parse_config_files", +) + @click_helpstring([config_files_argument] + list(config_file_options.values())) -def parse_config_files(cfg : confuse.Configuration = config, ctx : click.Context = mock_context, **kwargs) -> None: +def parse_config_files( + cfg: confuse.Configuration = config, ctx: click.Context = mock_context, **kwargs +) -> None: """ Parse configuration file(s) and override with command line arguments @@ -205,16 +222,23 @@ def parse_config_files(cfg : confuse.Configuration = config, ctx : click.Context Returns: None; side effect: updates the `cfg` argument """ - def _parse_option(param : click.Parameter, value: Any) -> Any: + + def _parse_option(param: click.Parameter, value: Any) -> Any: """internal parser to autobox values""" - if ((param.multiple or param.nargs == -1) and not isinstance(value, (list, tuple))): + if (param.multiple or param.nargs == -1) and not isinstance( + value, (list, tuple) + ): value = [value] return param.type_cast_value(ctx, value) - parsed_args = {config_files_argument.name}.union({option.name for option in config_file_options.values()}) + parsed_args = {config_files_argument.name}.union( + {option.name for option in config_file_options.values()} + ) # warn re unrecognized arguments - if (unknownargs := [k for k in parsed_args.difference(kwargs.keys()) if kwargs.get(k) is not None]): + if unknownargs := [ + k for k in parsed_args.difference(kwargs.keys()) if kwargs.get(k) is not None + ]: warnings.warn(f"Unused arguments: {unknownargs}") # initialize the config, including handling missing / double-specified config files @@ -226,10 +250,16 @@ def _parse_option(param : click.Parameter, value: Any) -> Any: raise ValueError(f"No config files provided.") else: error_dict = {k: kwargs[k] for k in found_configs} - raise ValueError(f"Exactly one config file source option must be provided; got {error_dict}.") + raise ValueError( + f"Exactly one config file source option must be provided; got {error_dict}." + ) else: config_key = found_configs[0] - config_validator = config_file_options[config_key] if config_key in config_file_options else config_files_argument + config_validator = ( + config_file_options[config_key] + if config_key in config_file_options + else config_files_argument + ) config_src = _parse_option(config_validator, kwargs[config_key]) cfg.clear() for config_file in reversed(config_src): @@ -244,7 +274,9 @@ def _parse_option(param : click.Parameter, value: Any) -> Any: if cfg[key].exists(): cfg[key]["scenarios"] = as_list(value) else: - raise ValueError(f"Specified {option} when no {key} in configuration file(s): {config_src}") + raise ValueError( + f"Specified {option} when no {key} in configuration file(s): {config_src}" + ) # update the config with the remaining options other_args = parsed_args - config_args - scen_args diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py index 66179545b..11fbcbcbe 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -1,4 +1,3 @@ - import yaml import pathlib from typing import Callable, Any @@ -9,6 +8,7 @@ from gempyor.shared_cli import parse_config_files, config_file_options from gempyor.testing import * + def config_file( tmp_path: pathlib.Path, config_dict: dict[str, Any] = {}, @@ -19,51 +19,90 @@ def config_file( f.write(create_confuse_config_from_dict(config_dict).dump()) return config_file + other_single_opt_args = [ - "write_csv", "write_parquet", "first_sim_index", "jobs", "nslots", "stoch_traj_flag", - "in_run_id", "out_run_id", "in_prefix" + "write_csv", + "write_parquet", + "first_sim_index", + "jobs", + "nslots", + "stoch_traj_flag", + "in_run_id", + "out_run_id", + "in_prefix", ] + # collection of bad option values -def bad_opt_args(opt : str) -> dict[str, Any]: - return { opt: { - "write_csv": "foo", "write_parquet": "bar", "first_sim_index": -1, - "seir_modifiers_scenario": 1, "outcome_modifiers_scenario": 1, - "jobs": -1, "nslots": -1, "stoch_traj_flag": "foo", "in_run_id": -1, "out_run_id": -1, - "in_prefix": [42], - }[opt] } +def bad_opt_args(opt: str) -> dict[str, Any]: + return { + opt: { + "write_csv": "foo", + "write_parquet": "bar", + "first_sim_index": -1, + "seir_modifiers_scenario": 1, + "outcome_modifiers_scenario": 1, + "jobs": -1, + "nslots": -1, + "stoch_traj_flag": "foo", + "in_run_id": -1, + "out_run_id": -1, + "in_prefix": [42], + }[opt] + } + # collection of good option values -def good_opt_args(opt : str) -> dict[str, Any]: - return { opt: { - "write_csv": False, "write_parquet": False, "first_sim_index": 42, - "seir_modifiers_scenario": "example", "outcome_modifiers_scenario": "example", - "jobs": 10, "nslots": 10, "stoch_traj_flag": False, "in_run_id": "foo", "out_run_id": "foo", - "in_prefix": "foo", - }[opt] } +def good_opt_args(opt: str) -> dict[str, Any]: + return { + opt: { + "write_csv": False, + "write_parquet": False, + "first_sim_index": 42, + "seir_modifiers_scenario": "example", + "outcome_modifiers_scenario": "example", + "jobs": 10, + "nslots": 10, + "stoch_traj_flag": False, + "in_run_id": "foo", + "out_run_id": "foo", + "in_prefix": "foo", + }[opt] + } + # collection of good configuration entries, to be overwritten by good_opt_args -def ref_cfg_kvs(opt : str) -> dict[str, Any]: - return { opt: { - "write_csv": True, "write_parquet": True, "first_sim_index": 1, - "seir_modifiers_scenario": ["example", "ibid"], "outcome_modifiers_scenario": ["example", "ibid"], - "jobs": 1, "nslots": 1, "stoch_traj_flag": True, "in_run_id": "bar", "out_run_id": "bar", - "in_prefix": "bar", - }[opt]} +def ref_cfg_kvs(opt: str) -> dict[str, Any]: + return { + opt: { + "write_csv": True, + "write_parquet": True, + "first_sim_index": 1, + "seir_modifiers_scenario": ["example", "ibid"], + "outcome_modifiers_scenario": ["example", "ibid"], + "jobs": 1, + "nslots": 1, + "stoch_traj_flag": True, + "in_run_id": "bar", + "out_run_id": "bar", + "in_prefix": "bar", + }[opt] + } + class TestParseConfigFiles: - + def test_deprecated_config( self, tmp_path: pathlib.Path, ) -> None: """Check that a -c config file work.""" - testdict = {"foo": "bar", "test": 123 } + testdict = {"foo": "bar", "test": 123} tmpconfigfile = config_file(tmp_path, testdict) mockconfig = mock_empty_config() parse_config_files(mockconfig, config_filepath=tmpconfigfile) for k, v in testdict.items(): - assert mockconfig[k].get(v) == v + assert mockconfig[k].get(v) == v assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile)] def test_preferred_config( @@ -71,12 +110,12 @@ def test_preferred_config( tmp_path: pathlib.Path, ) -> None: """Check that a -c config file work.""" - testdict = {"foo": "bar", "test": 123 } + testdict = {"foo": "bar", "test": 123} tmpconfigfile = config_file(tmp_path, testdict) mockconfig = mock_empty_config() parse_config_files(mockconfig, config_files=tmpconfigfile) for k, v in testdict.items(): - assert mockconfig[k].get(v) == v + assert mockconfig[k].get(v) == v assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile)] def test_conflict_config_opts_error( @@ -84,26 +123,31 @@ def test_conflict_config_opts_error( tmp_path: pathlib.Path, ) -> None: """Check that both -c and argument style config file raise an error.""" - testdict = {"foo": "bar", "test": 123 } + testdict = {"foo": "bar", "test": 123} tmpconfigfile = config_file(tmp_path, testdict) mockconfig = mock_empty_config() with pytest.raises(ValueError): - parse_config_files(mockconfig, config_filepath=tmpconfigfile, config_files=tmpconfigfile) + parse_config_files( + mockconfig, config_filepath=tmpconfigfile, config_files=tmpconfigfile + ) def test_multifile_config( self, tmp_path: pathlib.Path, ) -> None: """Check that multiple config files are merged.""" - testdict1 = {"foo": "bar", "test": 123 } - testdict2 = {"bar": "baz" } + testdict1 = {"foo": "bar", "test": 123} + testdict2 = {"bar": "baz"} tmpconfigfile1 = config_file(tmp_path, testdict1, "config1.yaml") tmpconfigfile2 = config_file(tmp_path, testdict2, "config2.yaml") mockconfig = mock_empty_config() parse_config_files(mockconfig, config_files=[tmpconfigfile1, tmpconfigfile2]) for k, v in (testdict1 | testdict2).items(): - assert mockconfig[k].get(v) == v - assert mockconfig["config_src"].as_str_seq() == [str(tmpconfigfile1), str(tmpconfigfile2)] + assert mockconfig[k].get(v) == v + assert mockconfig["config_src"].as_str_seq() == [ + str(tmpconfigfile1), + str(tmpconfigfile2), + ] @pytest.mark.parametrize("opt", [(k) for k in other_single_opt_args]) def test_other_opts(self, tmp_path: pathlib.Path, opt: str) -> None: From bf58e9443c6aef7527115612ed517379a83a1f83 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 15:59:02 -0400 Subject: [PATCH 41/53] WIP test patch [skip ci] --- examples/tutorials/config_sample_2pop.yml | 1 + flepimop/gempyor_pkg/src/gempyor/cli.py | 2 +- .../gempyor_pkg/src/gempyor/shared_cli.py | 4 +- .../gempyor_pkg/tests/shared_cli/test_cli.py | 67 +++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 flepimop/gempyor_pkg/tests/shared_cli/test_cli.py diff --git a/examples/tutorials/config_sample_2pop.yml b/examples/tutorials/config_sample_2pop.yml index 2b336303c..3d7d064db 100644 --- a/examples/tutorials/config_sample_2pop.yml +++ b/examples/tutorials/config_sample_2pop.yml @@ -11,6 +11,7 @@ subpop_setup: initial_conditions: method: SetInitialConditions initial_conditions_file: model_input/ic_2pop.csv + allow_missing_subpops: TRUE allow_missing_compartments: TRUE compartments: diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py index 84ad05503..a8a21b0ba 100644 --- a/flepimop/gempyor_pkg/src/gempyor/cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/cli.py @@ -20,7 +20,7 @@ # add some basic commands to the CLI -@cli.command(params=[config_files_argument].extend(config_file_options.values())) +@cli.command(params=[config_files_argument] + list(config_file_options.values())) @pass_context def patch(ctx : Context = mock_context, **kwargs) -> None: """Merge configuration files""" diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index f71ec7f77..c718f9605 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -211,7 +211,7 @@ def decorator(func): @click_helpstring([config_files_argument] + list(config_file_options.values())) def parse_config_files( cfg: confuse.Configuration = config, ctx: click.Context = mock_context, **kwargs -) -> None: +) -> confuse.Configuration: """ Parse configuration file(s) and override with command line arguments @@ -284,3 +284,5 @@ def _parse_option(param: click.Parameter, value: Any) -> Any: if (value := kwargs.get(option)) is not None: # auto box the value if the option expects a multiple cfg[option] = _parse_option(config_file_options[option], value) + + return cfg diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py b/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py new file mode 100644 index 000000000..1bd82d9b3 --- /dev/null +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py @@ -0,0 +1,67 @@ +import os +import subprocess +from pathlib import Path + +from click.testing import CliRunner + +from gempyor.simulate import _click_simulate +from gempyor.testing import * +from gempyor.shared_cli import parse_config_files +from gempyor.cli import patch + +# See here to test click application https://click.palletsprojects.com/en/8.1.x/testing/ +# would be useful to also call the command directly + +tutorialpath = os.path.dirname(__file__) + "/../../../../examples/tutorials" + + +def test_config_sample_2pop(): + os.chdir(tutorialpath) + runner = CliRunner() + result = runner.invoke(_click_simulate, ["config_sample_2pop.yml"]) + assert result.exit_code == 0 + +def test_config_sample_2pop_deprecated(): + os.chdir(tutorialpath) + runner = CliRunner() + result = runner.invoke(_click_simulate, ["-c", "config_sample_2pop.yml"]) + assert result.exit_code == 0 + + +def test_sample_2pop_modifiers(): + os.chdir(tutorialpath) + runner = CliRunner() + result = runner.invoke( + _click_simulate, + [ + "config_sample_2pop.yml", + "config_sample_2pop_outcomes_part.yml", + "config_sample_2pop_modifiers_part.yml", + ], + ) + assert result.exit_code == 0 + + +def test_sample_2pop_modifiers_combined(tmp_path : Path): + os.chdir(tutorialpath) + tmp_cfg = tmp_path / "patch_modifiers.yml" + runner = CliRunner() + result = runner.invoke(patch, ["config_sample_2pop.yml", + "config_sample_2pop_outcomes_part.yml", + "config_sample_2pop_modifiers_part.yml"]) + assert result.exit_code == 0 + with open(tmp_cfg, "w") as f: + f.write(result.output) + tmpconfig1 = create_confuse_config_from_file(str(tmp_cfg)).flatten() + tmpconfig2 = parse_config_files(cfg = mock_empty_config(), config_files = "config_sample_2pop_modifiers.yml").flatten() + + assert { k: v for k, v in tmpconfig1.items() if k != "config_src" } == { k: v for k, v in tmpconfig2.items() if k != "config_src" } + +def test_simple_usa_statelevel_more_deprecated(): + os.chdir(tutorialpath + "/../simple_usa_statelevel") + result = subprocess.run( + ["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"], + capture_output=True, + text=True, + ) + assert result.returncode == 0 From d7beb69c82d170427765c557e148164d96a1c1e1 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 16:03:52 -0400 Subject: [PATCH 42/53] working patch test --- .../gempyor_pkg/tests/shared_cli/test_cli.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py b/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py index 1bd82d9b3..c1190fccb 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_cli.py @@ -44,16 +44,25 @@ def test_sample_2pop_modifiers(): def test_sample_2pop_modifiers_combined(tmp_path : Path): os.chdir(tutorialpath) - tmp_cfg = tmp_path / "patch_modifiers.yml" + tmp_cfg1 = tmp_path / "patch_modifiers.yml" + tmp_cfg2 = tmp_path / "nopatch_modifiers.yml" runner = CliRunner() + result = runner.invoke(patch, ["config_sample_2pop.yml", "config_sample_2pop_outcomes_part.yml", "config_sample_2pop_modifiers_part.yml"]) assert result.exit_code == 0 - with open(tmp_cfg, "w") as f: + with open(tmp_cfg1, "w") as f: f.write(result.output) - tmpconfig1 = create_confuse_config_from_file(str(tmp_cfg)).flatten() - tmpconfig2 = parse_config_files(cfg = mock_empty_config(), config_files = "config_sample_2pop_modifiers.yml").flatten() + + result = runner.invoke(patch, ["config_sample_2pop_modifiers.yml"]) + assert result.exit_code == 0 + with open(tmp_cfg2, "w") as f: + f.write(result.output) + + + tmpconfig1 = create_confuse_config_from_file(str(tmp_cfg1)).flatten() + tmpconfig2 = create_confuse_config_from_file(str(tmp_cfg2)).flatten() assert { k: v for k, v in tmpconfig1.items() if k != "config_src" } == { k: v for k, v in tmpconfig2.items() if k != "config_src" } From 558ce4ba8f0edd6b859d70dc1cc08287f1dc972f Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 21:56:37 -0400 Subject: [PATCH 43/53] Update flepimop/gempyor_pkg/src/gempyor/shared_cli.py Co-authored-by: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com> --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index c718f9605..6ed09348a 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -28,7 +28,7 @@ class AlphanumericParamType(click.ParamType): """A custom click parameter type for alphanumeric strings""" name = "alphanumeric" - an_pattern = re.compile("^[a-zA-Z0-9]+$") + an_pattern = re.compile("^[a-zA-Z0-9_]+$") def convert(self, value, param, ctx): if not isinstance(value, str): From 814f8a4b7d7cb133a029c02d136eb071c3a66075 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 21 Oct 2024 22:08:26 -0400 Subject: [PATCH 44/53] Update flepimop/gempyor_pkg/src/gempyor/shared_cli.py --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 6ed09348a..2a61ee858 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -35,7 +35,7 @@ def convert(self, value, param, ctx): value = str(value) if not self.an_pattern.match(value): self.fail( - f"{value!r} is not a valid alphanumeric value; must only have [a-zA-Z0-9] elements.", + f"{value!r} is not a valid alphanumeric value; must only have [a-zA-Z0-9_] elements.", param, ctx, ) From 3e6edb3e077d8cf76c60679ed963e11d9705e14d Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 22 Oct 2024 09:16:02 -0400 Subject: [PATCH 45/53] prune yaml import from test --- .../gempyor_pkg/tests/shared_cli/test_parse_config_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py index 11fbcbcbe..404862c85 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -1,6 +1,6 @@ -import yaml + import pathlib -from typing import Callable, Any +from typing import Any import pytest import click From 64243826cf0252a23706363b770a1aef20bd5222 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 22 Oct 2024 09:26:28 -0400 Subject: [PATCH 46/53] docstring generation tweaking --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 2a61ee858..729f998c6 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -180,18 +180,19 @@ def click_helpstring( if not isinstance(params, list): params = [params] - def decorator(func): + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: # Generate the additional docstring with args from the specified functions additional_doc = "\n\tCommand Line Interface arguments:\n" for param in params: paraminfo = param.to_info_dict() additional_doc += ( - f"\n\t\t{paraminfo['name']}: {paraminfo['type']['param_type']}\n" + f"\n\t{paraminfo['name']}: {paraminfo['type']['param_type']}" ) if func.__doc__ is None: func.__doc__ = "" - func.__doc__ += additional_doc + + func.__doc__ += additional_doc.lstrip() return func From 96cec5baf3651a40827e75879d0c9d94b249ec14 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 24 Oct 2024 14:32:41 -0400 Subject: [PATCH 47/53] reconstitute simulate interface [skip ci] --- .../gempyor_pkg/src/gempyor/shared_cli.py | 4 +- flepimop/gempyor_pkg/src/gempyor/simulate.py | 129 ++++++++++++------ 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 729f998c6..6ff3c6087 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -217,11 +217,11 @@ def parse_config_files( Parse configuration file(s) and override with command line arguments Args: - cfg: the configuration object to update; defaults to global `.utils.config` + cfg: the configuration object to update; defaults to global `utils.config` ctx: the click context (used for type casting); defaults to a mock context for non-click use; pass actual context if using in a click command **kwargs: see auto generated CLI items below. Unmatched keys will be ignored + a warning will be issued - Returns: None; side effect: updates the `cfg` argument + Returns: returns the object passed via `cfg`; n.b. this object is also side-effected """ def _parse_option(param: click.Parameter, value: Any) -> Any: diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index c46ce34b0..1a95cdfcc 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -158,82 +158,125 @@ import time, warnings, sys +from pathlib import Path +from collections.abc import Iterable + +from confuse import Configuration from click import Context, pass_context -from . import seir, outcomes, model_info -from .utils import config #, profile +from . import seir, outcomes, model_info, utils from .shared_cli import config_files_argument, config_file_options, parse_config_files, cli, click_helpstring, mock_context # from .profile import profile_options # @profile_options # @profile() -@click_helpstring([config_files_argument] + list(config_file_options.values())) -def simulate(ctx : Context = mock_context, **kwargs) -> int: +def simulate( + config_filepath: Configuration | Path | Iterable[Path], + id_run_id: str = None, + out_run_id: str = None, + seir_modifiers_scenarios: str | Iterable[str] = [], + outcome_modifiers_scenarios: str | Iterable[str] = [], + in_prefix: str = None, + nslots: int = None, + jobs: int = None, + write_csv: bool = False, + write_parquet: bool = True, + first_sim_index: int = 1, + stoch_traj_flag: bool = False, + verbose : bool = True, +) -> int: """ Forward simulate a model using gempyor. - Args: (see auto generated CLI items below) + Args: + config_filepath: Either a Configuration (in which case: ALL other arguments will be silently ignored) OR + file path(s) for configuration file(s) (in which case other arguments will be used to override the configuration) + id_run_id: run_id for the simulation + out_run_id: run_id for the output + seir_modifiers_scenarios: scenarios for the SEIR model - if present, used to subset the scenarios in the configuration + outcome_modifiers_scenarios: scenarios for the outcomes model - if present, used to subset the scenarios in the configuration + in_prefix: prefix for the input files + nslots: number of simulation chains + jobs: amount of parallelization + write_csv: write output to csv? + write_parquet: write output to parquet? + first_sim_index: index of the first simulation + stoch_traj_flag: stochastic trajectories? + verbose: print output to console? Returns: exit code (side effect: writes output to disk) """ - parse_config_files(config, ctx, **kwargs) + if not isinstance(config_filepath, Configuration): + largs = locals() + cfg = parse_config_files(**largs) + else: + cfg = config_filepath scenarios_combinations = [ - [s, d] for s in (config["seir_modifiers"]["scenarios"].as_str_seq() if config["seir_modifiers"].exists() else [None]) - for d in (config["outcome_modifiers"]["scenarios"].as_str_seq() if config["outcome_modifiers"].exists() else [None])] + [s, d] for s in (cfg["seir_modifiers"]["scenarios"].as_str_seq() if cfg["seir_modifiers"].exists() else [None]) + for d in (cfg["outcome_modifiers"]["scenarios"].as_str_seq() if cfg["outcome_modifiers"].exists() else [None])] - print("Combination of modifiers scenarios to be run: ") - print(scenarios_combinations) - for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: - print(f"seir_modifier: {seir_modifiers_scenario}, outcomes_modifier:{outcome_modifiers_scenario}") + if verbose: + print("Combination of modifiers scenarios to be run: ") + print(scenarios_combinations) + for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: + print(f"seir_modifier: {seir_modifiers_scenario}, outcomes_modifier: {outcome_modifiers_scenario}") - nslots = config["nslots"].as_number() - print(f"Simulations to be run: {nslots}") + nchains = cfg["nslots"].as_number() + + if verbose: + print(f"Simulations to be run: {nchains}") for seir_modifiers_scenario, outcome_modifiers_scenario in scenarios_combinations: start = time.monotonic() - print(f"Running {seir_modifiers_scenario}_{outcome_modifiers_scenario}") + if verbose: + print(f"Running {seir_modifiers_scenario}_{outcome_modifiers_scenario}") modinf = model_info.ModelInfo( - config=config, - nslots=nslots, + config=cfg, + nslots=nchains, seir_modifiers_scenario=seir_modifiers_scenario, outcome_modifiers_scenario=outcome_modifiers_scenario, - write_csv=config["write_csv"].get(bool), - write_parquet=config["write_parquet"].get(bool), - first_sim_index=config["first_sim_index"].get(int), - in_run_id=config["in_run_id"].get(str) if config["in_run_id"].exists() else None, + write_csv=cfg["write_csv"].get(bool), + write_parquet=cfg["write_parquet"].get(bool), + first_sim_index=cfg["first_sim_index"].get(int), + in_run_id=cfg["in_run_id"].get(str) if cfg["in_run_id"].exists() else None, # in_prefix=config["name"].get() + "/", - out_run_id=config["out_run_id"].get(str) if config["out_run_id"].exists() else None, + out_run_id=cfg["out_run_id"].get(str) if cfg["out_run_id"].exists() else None, # out_prefix=config["name"].get() + "/" + str(seir_modifiers_scenario) + "/" + out_run_id + "/", - stoch_traj_flag=config["stoch_traj_flag"].get(bool), - config_filepath=config["config_src"].as_str_seq(), + stoch_traj_flag=cfg["stoch_traj_flag"].get(bool), + config_filepath=cfg["config_src"].as_str_seq(), ) - print( - f""" - >> Running from config {config["config_src"].as_str_seq()} - >> Starting {modinf.nslots} model runs beginning from {modinf.first_sim_index} on {config["jobs"].get(int)} processes - >> ModelInfo *** {modinf.setup_name} *** from {modinf.ti} to {modinf.tf} - >> Running scenario {seir_modifiers_scenario}_{outcome_modifiers_scenario} - >> running ***{'STOCHASTIC' if config["stoch_traj_flag"].get(bool) else 'DETERMINISTIC'}*** trajectories - """ - ) + if verbose: + print( + f""" + >> Running from config {cfg["config_src"].as_str_seq()} + >> Starting {modinf.nslots} model runs beginning from {modinf.first_sim_index} on {cfg["jobs"].get(int)} processes + >> ModelInfo *** {modinf.setup_name} *** from {modinf.ti} to {modinf.tf} + >> Running scenario {seir_modifiers_scenario}_{outcome_modifiers_scenario} + >> running ***{'STOCHASTIC' if cfg["stoch_traj_flag"].get(bool) else 'DETERMINISTIC'}*** trajectories + """ + ) # (there should be a run function) - if config["seir"].exists(): - seir.run_parallel_SEIR(modinf, config=config, n_jobs=config["jobs"].get(int)) - if config["outcomes"].exists(): - outcomes.run_parallel_outcomes(sim_id2write=config["first_sim_index"].get(int), modinf=modinf, nslots=nslots, n_jobs=config["jobs"].get(int)) - print( - f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" - ) - return 0 + if cfg["seir"].exists(): + seir.run_parallel_SEIR(modinf, config=cfg, n_jobs=cfg["jobs"].get(int)) + if cfg["outcomes"].exists(): + outcomes.run_parallel_outcomes(sim_id2write=cfg["first_sim_index"].get(int), modinf=modinf, nslots=nchains, n_jobs=cfg["jobs"].get(int)) + if verbose: + print( + f">>> {seir_modifiers_scenario}_{outcome_modifiers_scenario} completed in {time.monotonic() - start:.1f} seconds" + ) + + return 0 @cli.command(name="simulate", params=[config_files_argument] + list(config_file_options.values())) @pass_context def _click_simulate(ctx : Context, **kwargs) -> int: - return simulate(ctx, **kwargs) + """Forward simulate a model using gempyor.""" + cfg = parse_config_files(utils.config, ctx, **kwargs) + return simulate(cfg) # will all be removed upon deprecated endpoint removal @@ -247,8 +290,6 @@ def _deprecated_simulate(argv : list[str] = []) -> int: warnings.warn(f"This command is deprecated, use the CLI instead: `{clickcmd}`", DeprecationWarning) return subprocess.run(clickcmd, shell=True).returncode -_deprecated_simulate.__doc__ = simulate.__doc__ - if __name__ == "__main__": argv = sys.argv[1:] clickcmd = ' '.join(['flepimop', 'simulate'] + argv) From 51a48608cb6caf9e49afba567d07ebb478845209 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 24 Oct 2024 15:07:55 -0400 Subject: [PATCH 48/53] issue warnings for config files with duplicate keys --- flepimop/gempyor_pkg/src/gempyor/shared_cli.py | 8 +++++++- flepimop/gempyor_pkg/src/gempyor/simulate.py | 2 ++ .../tests/shared_cli/test_parse_config_files.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 6ff3c6087..341227ca6 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -263,7 +263,13 @@ def _parse_option(param: click.Parameter, value: Any) -> Any: ) config_src = _parse_option(config_validator, kwargs[config_key]) cfg.clear() - for config_file in reversed(config_src): + for config_file in config_src: + tmp = confuse.Configuration("tmp") + tmp.set_file(config_file) + if intersect := set(tmp.keys()) & set(cfg.keys()): + warnings.warn( + f"Configuration files contain overlapping keys: {intersect}." + ) cfg.set_file(config_file) cfg["config_src"] = [str(k) for k in config_src] diff --git a/flepimop/gempyor_pkg/src/gempyor/simulate.py b/flepimop/gempyor_pkg/src/gempyor/simulate.py index 1a95cdfcc..ad0485f14 100644 --- a/flepimop/gempyor_pkg/src/gempyor/simulate.py +++ b/flepimop/gempyor_pkg/src/gempyor/simulate.py @@ -209,6 +209,8 @@ def simulate( """ if not isinstance(config_filepath, Configuration): largs = locals() + largs.pop("verbose") + largs["config_files"] = largs.pop("config_filepath") cfg = parse_config_files(**largs) else: cfg = config_filepath diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py index 404862c85..b19ebcb95 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -149,6 +149,21 @@ def test_multifile_config( str(tmpconfigfile2), ] + def test_multifile_config_collision( + self, + tmp_path: pathlib.Path, + ) -> None: + """Check that multiple config overlapping keys are warned.""" + testdict1 = {"foo": "notthis", "test": 123} + testdict2 = {"foo": "this"} + tmpconfigfile1 = config_file(tmp_path, testdict1, "config1.yaml") + tmpconfigfile2 = config_file(tmp_path, testdict2, "config2.yaml") + mockconfig = mock_empty_config() + with pytest.warns(UserWarning, match=r'foo'): + parse_config_files(mockconfig, config_files=[tmpconfigfile1, tmpconfigfile2]) + for k, v in (testdict1 | testdict2).items(): + assert mockconfig[k].get(v) == v + @pytest.mark.parametrize("opt", [(k) for k in other_single_opt_args]) def test_other_opts(self, tmp_path: pathlib.Path, opt: str) -> None: """for the non-scenario modifier parameters, test default, envvar, invalid values, valid values, override""" From 0dd4c5925b405d15f34ae37cb18bd92602e87dba Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 28 Oct 2024 09:39:08 -0400 Subject: [PATCH 49/53] clarify click version requirement and remove string filtering --- flepimop/gempyor_pkg/setup.cfg | 2 +- .../gempyor_pkg/src/gempyor/shared_cli.py | 33 +++---------------- .../shared_cli/test_parse_config_files.py | 3 -- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index e7dea1d0e..48a2d3111 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -28,7 +28,7 @@ install_requires = matplotlib xarray emcee - click + click >= 8.7.1 confuse pyarrow sympy diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py index 341227ca6..ffc1089fa 100644 --- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py +++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py @@ -23,29 +23,6 @@ def cli(ctx: click.Context) -> None: """Flexible Epidemic Modeling Platform (FlepiMoP) Command Line Interface""" pass - -class AlphanumericParamType(click.ParamType): - """A custom click parameter type for alphanumeric strings""" - - name = "alphanumeric" - an_pattern = re.compile("^[a-zA-Z0-9_]+$") - - def convert(self, value, param, ctx): - if not isinstance(value, str): - value = str(value) - if not self.an_pattern.match(value): - self.fail( - f"{value!r} is not a valid alphanumeric value; must only have [a-zA-Z0-9_] elements.", - param, - ctx, - ) - else: - return value - - -AN_STR = AlphanumericParamType() - - # click decorator to handle configuration file(s) as arguments # use as `@argument_config_files` before a cli command definition config_files_argument = click.Argument( @@ -68,7 +45,7 @@ def convert(self, value, param, ctx): "seir_modifiers_scenarios": click.Option( ["-s", "--seir_modifiers_scenarios"], envvar="FLEPI_SEIR_SCENARIO", - type=AN_STR, + type=click.STRING, default=[], multiple=True, help="override/select the transmission scenario(s) to run", @@ -76,7 +53,7 @@ def convert(self, value, param, ctx): "outcome_modifiers_scenarios": click.Option( ["-d", "--outcome_modifiers_scenarios"], envvar="FLEPI_OUTCOME_SCENARIO", - type=AN_STR, + type=click.STRING, default=[], multiple=True, help="override/select the outcome scenario(s) to run", @@ -98,21 +75,21 @@ def convert(self, value, param, ctx): "in_run_id": click.Option( ["--in-id", "in_run_id"], envvar="FLEPI_RUN_INDEX", - type=AN_STR, + type=click.STRING, show_default=True, help="Unique identifier for the run", ), "out_run_id": click.Option( ["--out-id", "out_run_id"], envvar="FLEPI_RUN_INDEX", - type=AN_STR, + type=click.STRING, show_default=True, help="Unique identifier for the run", ), "in_prefix": click.Option( ["--in-prefix"], envvar="FLEPI_PREFIX", - type=AN_STR, + type=click.STRING, default=None, show_default=True, help="unique identifier for the run", diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py index b19ebcb95..0a73555d9 100644 --- a/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py +++ b/flepimop/gempyor_pkg/tests/shared_cli/test_parse_config_files.py @@ -169,7 +169,6 @@ def test_other_opts(self, tmp_path: pathlib.Path, opt: str) -> None: """for the non-scenario modifier parameters, test default, envvar, invalid values, valid values, override""" goodopt = good_opt_args(opt) - badopt = bad_opt_args(opt) refopt = ref_cfg_kvs(opt) # the config file, with option set @@ -180,8 +179,6 @@ def test_other_opts(self, tmp_path: pathlib.Path, opt: str) -> None: for cfg in [tmpconfigfile_wi_ref, tmpconfigfile_wo_ref]: # both versions error on bad values - with pytest.raises(click.exceptions.BadParameter): - parse_config_files(mockconfig, config_files=cfg, **badopt) # when supplied an override, both should have the override parse_config_files(mockconfig, config_files=cfg, **goodopt) for k, v in goodopt.items(): From 63f2759f29af37f897c065c21f1e798d2439faa0 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Mon, 28 Oct 2024 09:41:04 -0400 Subject: [PATCH 50/53] correct click version --- flepimop/gempyor_pkg/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flepimop/gempyor_pkg/setup.cfg b/flepimop/gempyor_pkg/setup.cfg index 48a2d3111..b79955d62 100644 --- a/flepimop/gempyor_pkg/setup.cfg +++ b/flepimop/gempyor_pkg/setup.cfg @@ -28,7 +28,7 @@ install_requires = matplotlib xarray emcee - click >= 8.7.1 + click >= 8.1.7 confuse pyarrow sympy From daebcdc9c3bdfd75feb15cadeb98b8bb57b884d0 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 29 Oct 2024 09:08:45 -0400 Subject: [PATCH 51/53] update documentation to remove reference to gempyor-simulate [skip ci] --- .../advanced-run-guides/quick-start-guide-conda.md | 4 ++-- .../advanced-run-guides/running-with-docker-locally.md | 6 +++--- documentation/gitbook/how-to-run/quick-start-guide.md | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md b/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md index 675dd5b94..eb632bf41 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md @@ -191,10 +191,10 @@ where: #### Non-inference run -Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor`. To do this, call `gempyor-simulate` providing the name of the configuration file you want to run (ex. `config.yml`). An example config is provided in `flepimop_sample/config_sample_2pop_interventions.yml.` +Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor`. To do this, call `flepimop simulate` providing the name of the configuration file you want to run (ex. `config.yml`). An example config is provided in `flepimop_sample/config_sample_2pop_interventions.yml.` ``` -gempyor-simulate -c config.yml +flepimop simulate config.yml ``` {% hint style="warning" %} diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md b/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md index c5f394ba3..b81eb8018 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md @@ -192,10 +192,10 @@ flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml ### Non-inference run -Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor,`call `gempyor-simulate` providing the name of the configuration file you want to run (ex. `config.yml` ; +Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor,`call `flepimop simulate` providing the name of the configuration file you want to run (ex. `config.yml`): ``` -gempyor-simulate -c config.yml +flepimop simulate config.yml ``` {% hint style="warning" %} @@ -216,7 +216,7 @@ Rscript build/local_install.R pip install --no-deps -e flepimop/gempyor_pkg/ cd $DATA_PATH rm -rf model_output -gempyor-simulate -c config.yml +flepimop simulate config.yml ## Finishing up diff --git a/documentation/gitbook/how-to-run/quick-start-guide.md b/documentation/gitbook/how-to-run/quick-start-guide.md index de213e086..05e03ffcd 100644 --- a/documentation/gitbook/how-to-run/quick-start-guide.md +++ b/documentation/gitbook/how-to-run/quick-start-guide.md @@ -170,10 +170,10 @@ To get started, let's start with just running a forward simulation (non-inferenc ### Non-inference run -Stay in the `PROJECT_PATH` folder, and run a simulation directly from forward-simulation Python package gempyor. Call `gempyor-simulate` providing the name of the configuration file you want to run. For example here, we use `config_sample_2pop.yml` from _flepimop\_sample_. +Stay in the `PROJECT_PATH` folder, and run a simulation directly from forward-simulation Python package gempyor. Call `flepimop simulate` providing the name of the configuration file you want to run. For example here, we use `config_sample_2pop.yml` from _flepimop\_sample_. ``` -gempyor-simulate -c config_sample_2pop.yml +flepimop simulate config_sample_2pop.yml ``` This will produce a `model_output` folder, which you can look using provided post-processing functions and scripts. @@ -189,14 +189,14 @@ cd $FLEPI_PATH pip install --no-deps -e flepimop/gempyor_pkg/ cd $PROJECT_PATH rm -rf model_output -gempyor-simulate -c config.yml +flepimop simulate config.yml ``` Note that you only have to re-run the installation steps once each time you update any of the files in the flepimop repository (either by pulling changes made by the developers and stored on Github, or by changing them yourself). If you're just running the same or different configuration file, just repeat the final steps ``` rm -rf model_output -gempyor-simulate -c new_config.yml +flepimop simulate new_config.yml ``` ### Inference run @@ -257,7 +257,7 @@ Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config_inference_n ## 📈 Examining model output -If your run is successful, you should see your output files in the model\_output folder. The structure of the files in this folder is described in the [Model Output](../gempyor/output-files.md) section. By default, all the output files are .parquet format (a compressed format which can be imported as dataframes using R's arrow package `arrow::read_parquet` or using the free desktop application [Tad ](https://www.tadviewer.com/)for quick viewing). However, you can add the option `--write-csv` to the end of the commands to run the code (e.g., `> gempyor-simulate -c config.yml --write-csv)` to have everything saved as .csv files instead ; +If your run is successful, you should see your output files in the model\_output folder. The structure of the files in this folder is described in the [Model Output](../gempyor/output-files.md) section. By default, all the output files are .parquet format (a compressed format which can be imported as dataframes using R's arrow package `arrow::read_parquet` or using the free desktop application [Tad ](https://www.tadviewer.com/) for quick viewing). However, you can add the option `--write-csv` to the end of the commands to run the code (e.g., `flepimop simulate --write-csv config.yml`) to have everything saved as .csv files instead ; ## 🪜 Next steps From cb91676ca849240ee84fcc14491cb2cb6bcb87ab Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 29 Oct 2024 09:55:18 -0400 Subject: [PATCH 52/53] add multi-config documentation --- documentation/gitbook/SUMMARY.md | 1 + .../gitbook/how-to-run/multi-configs.md | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 documentation/gitbook/how-to-run/multi-configs.md diff --git a/documentation/gitbook/SUMMARY.md b/documentation/gitbook/SUMMARY.md index db3000965..ec75af31a 100644 --- a/documentation/gitbook/SUMMARY.md +++ b/documentation/gitbook/SUMMARY.md @@ -48,6 +48,7 @@ * [Before any run](how-to-run/before-any-run.md) * [Quick Start Guide](how-to-run/quick-start-guide.md) +* [Multiple Configuration Files](multi-configs.md) * [Advanced run guides](how-to-run/advanced-run-guides/README.md) * [Running with Docker locally 🛳](how-to-run/advanced-run-guides/running-with-docker-locally.md) * [Running locally in a conda environment 🐍](how-to-run/advanced-run-guides/quick-start-guide-conda.md) diff --git a/documentation/gitbook/how-to-run/multi-configs.md b/documentation/gitbook/how-to-run/multi-configs.md new file mode 100644 index 000000000..928e1318a --- /dev/null +++ b/documentation/gitbook/how-to-run/multi-configs.md @@ -0,0 +1,45 @@ +--- +description: >- + How to use multiple configuration files. +--- + +# Using Multiple Configuration Files + +## 🧱 Set up + +Create a sample project by copying from the examples folder: + +```bash +mkdir myflepimopexample # or wherever +cd myflepimopexample +cp -r $FLEPI_PATH/examples/tutorials/* . +ls +``` + +You should see an assortment of yml files as a result of that `ls` command. + +## Usage + +If you run + +```bash +flepimop simulate config_sample_2pop.yml +``` + +you'll get a basic foward simulation of this example model. However, you might also note there are several `*_part.yml` files, corresponding to partial configs. You can `simulate` using the combination of multiple configs with, for example: + +```bash +flepimop simulate config_sample_2pop.yml config_sample_2pop_outcomes_part.yml +``` + +if want to see what the combined configuration is, you can use the `patch` command: + +```bash +flepimop patch config_sample_2pop.yml config_sample_2pop_outcomes_part.yml +``` + +You may provide an arbitrary number of separate configuration files to combine to create a complete configuration. + +At this time, only `simulate` supports multiple configuration files. Also, the patching operation is fairly crude: configuration options override previous ones completely, though with a warning. The files provided from left to right are from lowest priority (i.e. for the first file, only options specified in no other files are used) to highest priority (i.e. for the last file, its options override any other specification). + +We are expanding coverage of this capability to other flepimop actions, e.g. inference, and are exploring options for smarter patching. \ No newline at end of file From 00591eb2d1202cfeb50f9d95baa2f2aca8335a5b Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Tue, 29 Oct 2024 10:28:27 -0400 Subject: [PATCH 53/53] update multi-configs gitbook to give caveat example --- .../gitbook/how-to-run/multi-configs.md | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/documentation/gitbook/how-to-run/multi-configs.md b/documentation/gitbook/how-to-run/multi-configs.md index 928e1318a..738de3399 100644 --- a/documentation/gitbook/how-to-run/multi-configs.md +++ b/documentation/gitbook/how-to-run/multi-configs.md @@ -1,6 +1,6 @@ --- description: >- - How to use multiple configuration files. + Patching together multiple configuration files. --- # Using Multiple Configuration Files @@ -40,6 +40,38 @@ flepimop patch config_sample_2pop.yml config_sample_2pop_outcomes_part.yml You may provide an arbitrary number of separate configuration files to combine to create a complete configuration. +## Caveats + At this time, only `simulate` supports multiple configuration files. Also, the patching operation is fairly crude: configuration options override previous ones completely, though with a warning. The files provided from left to right are from lowest priority (i.e. for the first file, only options specified in no other files are used) to highest priority (i.e. for the last file, its options override any other specification). -We are expanding coverage of this capability to other flepimop actions, e.g. inference, and are exploring options for smarter patching. \ No newline at end of file +We are expanding coverage of this capability to other flepimop actions, e.g. inference, and are exploring options for smarter patching. + +However, currently there are pitfalls like + +```yaml +# config1 +seir_modifiers: + scenarios: ["one", "two"] + one: + # ... + two: + # ... +``` + +```yaml +# config2 +seir_modifiers: + scenarios: ["one", "three"] + one: + # ... + three: + # ... +``` + +Then you might expect + +```bash +flepimop simulate config1.yml config2.yml +``` + +...to override seir scenario one and add scenario three, but what actually happens is that the entire seir_modifiers from config1 is overriden by config2. Specifying the configuration files in the reverse order would lead to a different outcome (the config1 seir_modifiers overrides config2 settings). If you're doing complex combinations of configuration files, you should use `flepimop patch ...` to ensure you're getting what you expect. \ No newline at end of file