Skip to content

Commit

Permalink
Merge pull request galaxyproject#17016 from bernt-matthias/topic/stdi…
Browse files Browse the repository at this point in the history 
…o-regex-extend

[24.0] Extend regex groups in stdio regex matches
  • Loading branch information
@jdavcs
jdavcs authored Mar 5, 2024
2 parents fde1ec3 + 0739b13 commit 5e05e00
Show file tree
Hide file tree
Showing 11 changed files with 223 additions and 121 deletions.
2 changes: 1 addition & 1 deletion client/src/components/JobInformation/JobInformation.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ describe("JobInformation/JobInformation.vue", () => {
});

it("job messages", async () => {
const rendered_link = jobInfoTable.findAll(`#job-messages li`);
const rendered_link = jobInfoTable.findAll(`#job-messages .job-message`);
expect(rendered_link.length).toBe(jobResponse.job_messages.length);
for (let i = 0; i < rendered_link.length; i++) {
const msg = rendered_link.at(i).text();
Expand Down
179 changes: 100 additions & 79 deletions client/src/components/JobInformation/JobInformation.vue
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,75 @@
<script setup>
import CopyToClipboard from "components/CopyToClipboard";
import HelpText from "components/Help/HelpText";
import { JobDetailsProvider } from "components/providers/JobProvider";
import UtcDate from "components/UtcDate";
import { formatDuration, intervalToDuration } from "date-fns";
import JOB_STATES_MODEL from "utils/job-states-model";
import { computed, ref } from "vue";
import { invocationForJob } from "@/api/invocations";
import DecodedId from "../DecodedId.vue";
import CodeRow from "./CodeRow.vue";
const job = ref(null);
const invocationId = ref(null);
const props = defineProps({
job_id: {
type: String,
required: true,
},
includeTimes: {
type: Boolean,
default: false,
},
});
const runTime = computed(() =>
formatDuration(intervalToDuration({ start: new Date(job.value.create_time), end: new Date(job.value.update_time) }))
);
const jobIsTerminal = computed(() => job.value && !JOB_STATES_MODEL.NON_TERMINAL_STATES.includes(job.value.state));
const routeToInvocation = computed(() => `/workflows/invocations/${invocationId.value}`);
const metadataDetail = ref({
exit_code: `Tools may use exit codes to indicate specific execution errors. Many programs use 0 to indicate success and non-zero exit codes to indicate errors. Galaxy allows each tool to specify exit codes that indicate errors. https://docs.galaxyproject.org/en/master/dev/schema.html#tool-stdio-exit-code`,
error_level: `NO_ERROR = 0</br>LOG = 1</br>QC = 1.1</br>WARNING = 2</br>FATAL = 3</br>FATAL_OOM = 4</br>MAX = 4`,
});
function updateJob(newJob) {
job.value = newJob;
if (newJob) {
fetchInvocation(newJob.id);
}
}
function filterMetadata(jobMessages) {
return jobMessages.map((item) => {
return Object.entries(item).reduce((acc, [key, value]) => {
if (value) {
acc[key] = value;
}
return acc;
}, {});
});
}
async function fetchInvocation(jobId) {
if (jobId) {
const invocation = await invocationForJob({ jobId: jobId });
if (invocation) {
invocationId.value = invocation.id;
}
}
}
</script>

<template>
<div>
<JobDetailsProvider auto-refresh :job-id="job_id" @update:result="updateJob" />
<JobDetailsProvider auto-refresh :job-id="props.job_id" @update:result="updateJob" />
<h2 class="h-md">Job Information</h2>
<table id="job-information" class="tabletip info_data_table">
<tbody>
Expand All @@ -24,19 +93,19 @@
<td>Galaxy Tool Version</td>
<td id="galaxy-tool-version">{{ job.tool_version }}</td>
</tr>
<tr v-if="job && includeTimes">
<tr v-if="job && props.includeTimes">
<td>Created</td>
<td v-if="job.create_time" id="created">
<UtcDate :date="job.create_time" mode="pretty" />
</td>
</tr>
<tr v-if="job && includeTimes">
<tr v-if="job && props.includeTimes">
<td>Updated</td>
<td v-if="job.update_time" id="updated">
<UtcDate :date="job.update_time" mode="pretty" />
</td>
</tr>
<tr v-if="job && includeTimes && jobIsTerminal">
<tr v-if="job && props.includeTimes && jobIsTerminal">
<td>Time To Finish</td>
<td id="runtime">
{{ runTime }}
Expand Down Expand Up @@ -73,9 +142,28 @@
<tr v-if="job && job.job_messages && job.job_messages.length > 0" id="job-messages">
<td>Job Messages</td>
<td>
<ul style="padding-left: 15px; margin-bottom: 0px">
<li v-for="(message, index) in job.job_messages" :key="index">{{ message }}</li>
<ul v-if="Array.isArray(job.job_messages)" class="pl-2 mb-0">
<div v-for="(message, m) in filterMetadata(job.job_messages)" :key="m" class="job-message">
<div v-if="job.job_messages.length > 1">
<u>Job Message {{ m + 1 }}:</u>
</div>
<li v-for="(value, name, i) in message" :key="i">
<span
v-if="metadataDetail[name]"
v-b-tooltip.html
class="tooltipJobInfo"
:title="metadataDetail[name]"
><strong>{{ name }}:</strong></span
>
<strong v-else>{{ name }}:</strong>
{{ value }}
</li>
<hr v-if="m + 1 < job.job_messages.length" />
</div>
</ul>
<div v-else>
{{ job.job_messages }}
</div>
</td>
</tr>
<slot></slot>
Expand All @@ -100,76 +188,9 @@
</div>
</template>

<script>
import CopyToClipboard from "components/CopyToClipboard";
import HelpText from "components/Help/HelpText";
import { JobDetailsProvider } from "components/providers/JobProvider";
import UtcDate from "components/UtcDate";
import { formatDuration, intervalToDuration } from "date-fns";
import { getAppRoot } from "onload/loadConfig";
import JOB_STATES_MODEL from "utils/job-states-model";
import { invocationForJob } from "@/api/invocations";
import DecodedId from "../DecodedId.vue";
import CodeRow from "./CodeRow.vue";
export default {
components: {
CodeRow,
DecodedId,
JobDetailsProvider,
HelpText,
UtcDate,
CopyToClipboard,
},
props: {
job_id: {
type: String,
required: true,
},
includeTimes: {
type: Boolean,
default: false,
},
},
data() {
return {
job: null,
invocationId: null,
};
},
computed: {
runTime: function () {
return formatDuration(
intervalToDuration({ start: new Date(this.job.create_time), end: new Date(this.job.update_time) })
);
},
jobIsTerminal() {
return this.job && !JOB_STATES_MODEL.NON_TERMINAL_STATES.includes(this.job.state);
},
routeToInvocation() {
return `/workflows/invocations/${this.invocationId}`;
},
},
methods: {
getAppRoot() {
return getAppRoot();
},
updateJob(job) {
this.job = job;
if (job) {
this.fetchInvocation(job.id);
}
},
async fetchInvocation(jobId) {
if (jobId) {
const invocation = await invocationForJob({ jobId: jobId });
if (invocation) {
this.invocationId = invocation.id;
}
}
},
},
};
</script>
<style scoped>
.tooltipJobInfo {
text-decoration-line: underline;
text-decoration-style: dashed;
}
</style>
15 changes: 9 additions & 6 deletions lib/galaxy/jobs/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
check_output,
DETECTED_JOB_STATE,
)
from galaxy.tool_util.parser.stdio import StdioErrorLevel
from galaxy.tools.evaluation import (
PartialToolEvaluator,
ToolEvaluator,
Expand Down Expand Up @@ -1939,7 +1940,13 @@ def fail(message=job.info, exception=None):
self.discover_outputs(job, inp_data, out_data, out_collections, final_job_state=final_job_state)
except MaxDiscoveredFilesExceededError as e:
final_job_state = job.states.ERROR
job.job_messages = [str(e)]
job.job_messages = [
{
"type": "internal",
"desc": str(e),
"error_level": StdioErrorLevel.FATAL,
}
]

for dataset_assoc in output_dataset_associations:
if getattr(dataset_assoc.dataset, "discovered", False):
Expand Down Expand Up @@ -2088,12 +2095,8 @@ def discover_outputs(self, job, inp_data, out_data, out_collections, final_job_s
)

def check_tool_output(self, tool_stdout, tool_stderr, tool_exit_code, job, job_stdout=None, job_stderr=None):
job_id_tag = "<unknown job id>"
if job is not None:
job_id_tag = job.get_id_tag()

state, tool_stdout, tool_stderr, job_messages = check_output(
self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag
self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code
)

# Store the modified stdout and stderr in the job:
Expand Down
41 changes: 28 additions & 13 deletions lib/galaxy/metadata/set_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@
import traceback
from functools import partial
from pathlib import Path
from typing import Optional
from typing import (
Any,
Dict,
List,
Optional,
)

try:
from pulsar.client.staging import COMMAND_VERSION_FILENAME
Expand Down Expand Up @@ -61,13 +66,15 @@
DETECTED_JOB_STATE,
)
from galaxy.tool_util.parser.stdio import (
StdioErrorLevel,
ToolStdioExitCode,
ToolStdioRegex,
)
from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
from galaxy.util import (
safe_contains,
stringify_dictionary_keys,
unicodify,
)
from galaxy.util.expressions import ExpressionContext

Expand Down Expand Up @@ -216,7 +223,7 @@ def set_meta(new_dataset_instance, file_dict):

export_store = None
final_job_state = Job.states.OK
job_messages = []
job_messages: List[Dict[str, Any]] = []
if extended_metadata_collection:
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
Expand All @@ -237,25 +244,25 @@ def set_meta(new_dataset_instance, file_dict):
for directory, prefix in locations:
if directory and os.path.exists(os.path.join(directory, f"{prefix}stdout")):
with open(os.path.join(directory, f"{prefix}stdout"), "rb") as f:
tool_stdout = f.read(MAX_STDIO_READ_BYTES)
tool_stdout = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
with open(os.path.join(directory, f"{prefix}stderr"), "rb") as f:
tool_stderr = f.read(MAX_STDIO_READ_BYTES)
tool_stderr = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
break
else:
if os.path.exists(os.path.join(tool_job_working_directory, "task_0")):
# We have a task splitting job
tool_stdout = b""
tool_stderr = b""
tool_stdout = ""
tool_stderr = ""
paths = tool_job_working_directory.glob("task_*")
for path in paths:
with open(path / "outputs" / "tool_stdout", "rb") as f:
task_stdout = f.read(MAX_STDIO_READ_BYTES)
task_stdout = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
if task_stdout:
tool_stdout = b"%s[%s stdout]\n%s\n" % (tool_stdout, path.name.encode(), task_stdout)
tool_stdout = f"{tool_stdout}[{path.name} stdout]\n{task_stdout}\n"
with open(path / "outputs" / "tool_stderr", "rb") as f:
task_stderr = f.read(MAX_STDIO_READ_BYTES)
task_stderr = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
if task_stderr:
tool_stderr = b"%s[%s stdout]\n%s\n" % (tool_stderr, path.name.encode(), task_stderr)
tool_stderr = f"{tool_stderr}[{path.name} stderr]\n{task_stderr}\n"
else:
wdc = os.listdir(tool_job_working_directory)
odc = os.listdir(outputs_directory)
Expand All @@ -265,15 +272,15 @@ def set_meta(new_dataset_instance, file_dict):
log.warning(f"{error_desc}. {error_extra}")
raise Exception(error_desc)
else:
tool_stdout = tool_stderr = b""
tool_stdout = tool_stderr = ""

job_id_tag = metadata_params["job_id_tag"]

exit_code_file = default_exit_code_file(".", job_id_tag)
tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)

check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(
stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag
stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code
)
if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs():
final_job_state = Job.states.OK
Expand Down Expand Up @@ -340,7 +347,15 @@ def set_meta(new_dataset_instance, file_dict):
collect_dynamic_outputs(job_context, output_collections)
except MaxDiscoveredFilesExceededError as e:
final_job_state = Job.states.ERROR
job_messages.append(str(e))
job_messages.append(
{
"type": "max_discovered_files",
"desc": str(e),
"code_desc": None,
"error_level": StdioErrorLevel.FATAL,
}
)

if job:
job.set_streams(tool_stdout=tool_stdout, tool_stderr=tool_stderr, job_messages=job_messages)
job.state = final_job_state
Expand Down
Loading

0 comments on commit 5e05e00

Please sign in to comment.