Skip to content

Commit

Permalink
bug fix: add a missing task dependency link
Browse files Browse the repository at this point in the history
**Fix:** Analysis tasks depend on the ValidationData task.

We could add all those direct links but it's lighter weight for Fireworks if we make just the VariantSimDataTasks depend on ValidationData.

**TODO:** Bump the `fw_validation_data` task up to priority 12 after testing that the added dependency links fix the bug.

Rename `fw_this_variant_this_seed_this_analysis` to `fw_this_variant_this_seed_multigen_analysis` for clarity.
  • Loading branch information
1fish2 committed Oct 1, 2021
1 parent 72e593c commit c759a73
Showing 1 changed file with 11 additions and 7 deletions.
18 changes: 11 additions & 7 deletions runscripts/fireworks/fw_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@
* CompressRawValidationData if COMPRESS_OUTPUT
* CompressRawData if COMPRESS_OUTPUT
* AnalysisParca if RUN_AGGREGATE_ANALYSIS
* VariantSimData [see the comment in the code]
FitSimData (fw_calculate_sim_data)
* CompressRawData if COMPRESS_OUTPUT
Expand Down Expand Up @@ -175,7 +176,7 @@
* CompressValidationData if COMPRESS_OUTPUT
* CompressSimulationOutput if COMPRESS_OUTPUT
AnalysisMultiGen (fw_this_variant_this_seed_this_analysis)
AnalysisMultiGen (fw_this_variant_this_seed_multigen_analysis)
* CompressVariantSimData if COMPRESS_OUTPUT
* CompressValidationData if COMPRESS_OUTPUT
* CompressSimulationOutput if COMPRESS_OUTPUT
Expand Down Expand Up @@ -612,6 +613,9 @@ def build_wcm_firetasks(self) -> Firework:
md_cohort = dict(self.metadata, variant_function = VARIANT, variant_index = i)

# Variant simData creation task
# Note: This task doesn't depend on fw_validation_data but such a
# link is lighter weight for Fireworks than making every analysis
# task directly depend on it.
fw_this_variant_sim_data = self.add_firework(
VariantSimDataTask(
variant_function=VARIANT,
Expand All @@ -622,7 +626,7 @@ def build_wcm_firetasks(self) -> Firework:
constants.SERIALIZED_SIM_DATA_MODIFIED),
variant_metadata_directory=VARIANT_METADATA_DIRECTORY),
name=f"VariantSimDataTask__{VARIANT}_{i:06d}",
parents=fw_calculate_sim_data,
parents=[fw_calculate_sim_data, fw_validation_data],
priority=12)

if COMPRESS_OUTPUT:
Expand Down Expand Up @@ -656,15 +660,15 @@ def build_wcm_firetasks(self) -> Firework:
cpus=analysis_cpus,
priority=4)

fw_this_variant_this_seed_this_analysis = None
fw_this_variant_this_seed_multigen_analysis = None

for j in range(SEED, SEED + N_INIT_SIMS):
SEED_DIRECTORY = os.path.join(VARIANT_DIRECTORY, "%06d" % j)
SEED_PLOT_DIRECTORY = os.path.join(SEED_DIRECTORY, constants.PLOTOUT_DIR)
md_multigen = dict(md_cohort, seed = j)

if RUN_AGGREGATE_ANALYSIS:
fw_this_variant_this_seed_this_analysis = self.add_firework(
fw_this_variant_this_seed_multigen_analysis = self.add_firework(
AnalysisMultiGenTask(
input_seed_directory=SEED_DIRECTORY,
input_sim_data=os.path.join(VARIANT_SIM_DATA_DIRECTORY,
Expand All @@ -681,7 +685,7 @@ def build_wcm_firetasks(self) -> Firework:
indent=1)

if COMPRESS_OUTPUT:
self.add_links(fw_this_variant_this_seed_this_analysis,
self.add_links(fw_this_variant_this_seed_multigen_analysis,
fw_this_variant_sim_data_compression)

sims_this_seed = collections.defaultdict(list)
Expand Down Expand Up @@ -758,7 +762,7 @@ def build_wcm_firetasks(self) -> Firework:
# Only add the last generation as dependencies for multiple sim analysis tasks
if RUN_AGGREGATE_ANALYSIS and k == N_GENS - 1:
self.add_links(fw_this_variant_this_gen_this_sim,
fw_this_variant_this_seed_this_analysis,
fw_this_variant_this_seed_multigen_analysis,
fw_this_variant_cohort_analysis,
fw_variant_analysis)

Expand Down Expand Up @@ -803,7 +807,7 @@ def build_wcm_firetasks(self) -> Firework:
# Don't compress any outputs or validation data until all analysis scripts (single gen, multigen, and cohort) have finished running
data_fws = [
fw_this_variant_this_gen_this_sim_analysis,
fw_this_variant_this_seed_this_analysis,
fw_this_variant_this_seed_multigen_analysis,
fw_this_variant_cohort_analysis,
fw_variant_analysis]
for data in data_fws:
Expand Down

0 comments on commit c759a73

Please sign in to comment.