From 40adc8abbf103d3b545a0ba8fb6564e269291c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 14 Mar 2024 20:20:00 +0100 Subject: [PATCH 1/2] add a few more rules from ORG, we should move this all to shared --- files/galaxy/tpv/tools.yml | 68 ++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/files/galaxy/tpv/tools.yml b/files/galaxy/tpv/tools.yml index 603bd3cfd..5c838175a 100644 --- a/files/galaxy/tpv/tools.yml +++ b/files/galaxy/tpv/tools.yml @@ -782,7 +782,8 @@ tools: singularity_enabled: true toolshed.g2.bx.psu.edu/repos/iuc/trinity/trinity/.*: - cores: 8 + cores: 24 + mem: 250 scheduling: prefer: - condor-tpv @@ -793,17 +794,21 @@ tools: # see usegalaxy-eu/issues#473: https://github.com/usegalaxy-eu/issues/issues/473 env: _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp + + - if: input_size < 0.1 + cores: 1 + mem: 4 - if: 0.1 <= input_size < 1 + cores: 12 + mem: 92 + - if: 1 <= input_size < 4 + cores: 16 + mem: 128 + - if: 4 <= input_size < 8 cores: 20 - mem: 100 - - if: 1 <= input_size < 2 - cores: 30 - mem: 200 - - if: 2 <= input_size < 30 - cores: 60 - mem: 950 - - if: input_size >= 30 - fail: + mem: 190 + - if: input_size >= 80 + fail: | Too much data, we cannot support such large Trinity assemblies with our backend. Please use another server for your job. @@ -1012,16 +1017,24 @@ tools: env: _JAVA_OPTIONS: -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp - ## bag things 40 is enough for most of the jobs, a better rule is needed toolshed.g2.bx.psu.edu/repos/iuc/unicycler/unicycler/.*: - cores: 16 - mem: 40 + cores: 24 + mem: 80 env: TERM: vt100 _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp scheduling: accept: - pulsar + rules: + - id: unicycler_small_input_rule + if: input_size < 0.05 + cores: 1 + mem: 3.8 + - id: unicycler_medium_input_rule + if: 0.05 <= input_size < 2 + cores: 8 + mem: 28 toolshed.g2.bx.psu.edu/repos/imgteam/unzip/unzip/.*: scheduling: @@ -1171,6 +1184,35 @@ tools: require: - singularity + toolshed.g2.bx.psu.edu/repos/iuc/bwa_mem2/bwa_mem2/.*: + cores: 32 + mem: 244 + rules: + - id: bwa_mem2_small_input_rule + if: input_size < 0.25 + cores: 2 + mem: 7.6 + - id: bwa_mem2_medium_input_rule + if: 0.25 <= input_size < 16 + cores: 8 + mem: 28 + - id: bwa_mem2_large_input_rule + if: 16 <= input_size < 32 + cores: 16 + mem: 58 + - id: bwa_mem2_xlarge_input_rule + if: 32 <= input_size < 64 + cores: 24 + mem: 120 + - id: bwa_mem2_history_reference_rule + if: | + helpers.job_args_match(job, app, {"reference_source": {"reference_source_selector": "history"}}) + # per https://github.com/bwa-mem2/bwa-mem2/issues/41 it's 28 * reference + mem: | + options = job.get_param_values(app) + size = options["reference_source"]["ref_file"].get_size() + min(max(float(size/1024**3) * 28, (input_size - float(size/1024**3)) * 2, 7.6), 120) + toolshed.g2.bx.psu.edu/repos/bgruening/flye/flye/.*: cores: 20 mem: min(max(input_size*1.2, 3.8), 256) From 2e450241739ad037dc901cfc38de1948dea1f426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 14 Mar 2024 20:23:41 +0100 Subject: [PATCH 2/2] Update tools.yml --- files/galaxy/tpv/tools.yml | 64 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/files/galaxy/tpv/tools.yml b/files/galaxy/tpv/tools.yml index 5c838175a..fcdafb255 100644 --- a/files/galaxy/tpv/tools.yml +++ b/files/galaxy/tpv/tools.yml @@ -1027,14 +1027,14 @@ tools: accept: - pulsar rules: - - id: unicycler_small_input_rule - if: input_size < 0.05 - cores: 1 - mem: 3.8 - - id: unicycler_medium_input_rule - if: 0.05 <= input_size < 2 - cores: 8 - mem: 28 + - id: unicycler_small_input_rule + if: input_size < 0.05 + cores: 1 + mem: 3.8 + - id: unicycler_medium_input_rule + if: 0.05 <= input_size < 2 + cores: 8 + mem: 28 toolshed.g2.bx.psu.edu/repos/imgteam/unzip/unzip/.*: scheduling: @@ -1188,30 +1188,30 @@ tools: cores: 32 mem: 244 rules: - - id: bwa_mem2_small_input_rule - if: input_size < 0.25 - cores: 2 - mem: 7.6 - - id: bwa_mem2_medium_input_rule - if: 0.25 <= input_size < 16 - cores: 8 - mem: 28 - - id: bwa_mem2_large_input_rule - if: 16 <= input_size < 32 - cores: 16 - mem: 58 - - id: bwa_mem2_xlarge_input_rule - if: 32 <= input_size < 64 - cores: 24 - mem: 120 - - id: bwa_mem2_history_reference_rule - if: | - helpers.job_args_match(job, app, {"reference_source": {"reference_source_selector": "history"}}) - # per https://github.com/bwa-mem2/bwa-mem2/issues/41 it's 28 * reference - mem: | - options = job.get_param_values(app) - size = options["reference_source"]["ref_file"].get_size() - min(max(float(size/1024**3) * 28, (input_size - float(size/1024**3)) * 2, 7.6), 120) + - id: bwa_mem2_small_input_rule + if: input_size < 0.25 + cores: 2 + mem: 7.6 + - id: bwa_mem2_medium_input_rule + if: 0.25 <= input_size < 16 + cores: 8 + mem: 28 + - id: bwa_mem2_large_input_rule + if: 16 <= input_size < 32 + cores: 16 + mem: 58 + - id: bwa_mem2_xlarge_input_rule + if: 32 <= input_size < 64 + cores: 24 + mem: 120 + - id: bwa_mem2_history_reference_rule + if: | + helpers.job_args_match(job, app, {"reference_source": {"reference_source_selector": "history"}}) + # per https://github.com/bwa-mem2/bwa-mem2/issues/41 it's 28 * reference + mem: | + options = job.get_param_values(app) + size = options["reference_source"]["ref_file"].get_size() + min(max(float(size/1024**3) * 28, (input_size - float(size/1024**3)) * 2, 7.6), 120) toolshed.g2.bx.psu.edu/repos/bgruening/flye/flye/.*: cores: 20