From 16febbd4386e7f203ad8a59a31ef3b1881150d3b Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 2 Oct 2023 16:32:23 +0200 Subject: [PATCH 001/110] Remove redundant endpoints that have already been migrated --- lib/galaxy/webapps/galaxy/api/jobs.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 72a565b05ccd..c110110bdc69 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -32,7 +32,6 @@ ProvidesUserContext, ) from galaxy.managers.jobs import ( - JobLock, JobManager, JobSearch, summarize_destination_params, @@ -171,6 +170,7 @@ @router.cbv class FastAPIJobs: service: JobsService = depends(JobsService) + manager: JobManager = depends(JobManager) @router.get("/api/jobs/{id}") def show( @@ -547,23 +547,3 @@ def error(self, trans: ProvidesUserContext, id, payload, **kwd): ) return {"messages": messages} - - @require_admin - @expose_api - def show_job_lock(self, trans: ProvidesUserContext, **kwd): - """ - * GET /api/job_lock - return boolean indicating if job lock active. - """ - return self.job_manager.job_lock() - - @require_admin - @expose_api - def update_job_lock(self, trans: ProvidesUserContext, payload, **kwd): - """ - * PUT /api/job_lock - return boolean indicating if job lock active. - """ - active = payload.get("active") - job_lock = JobLock(active=active) - return self.job_manager.update_job_lock(job_lock) From 4b2293dadc862ccad689bf6927ad4f71753a7b92 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 3 Oct 2023 17:31:01 +0200 Subject: [PATCH 002/110] Move method get_job to JobsService --- lib/galaxy/webapps/galaxy/services/jobs.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index 5a92d629603a..892902574962 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -92,3 +92,21 @@ def _check_nonadmin_access( raise exceptions.AdminRequiredException("Only admins can index the jobs with user details enabled") if decoded_user_id is not None and decoded_user_id != trans_user_id: raise exceptions.AdminRequiredException("Only admins can index the jobs of others") + + def get_job( + self, + trans: ProvidesUserContext, + job_id: Optional[int] = None, + dataset_id: Optional[int] = None, + hda_ldda: str = "hda", + ): + if job_id is not None: + return self.job_manager.get_accessible_job(trans, decoded_job_id=job_id) + elif dataset_id is not None: + # Following checks dataset accessible + if hda_ldda == "hda": + dataset_instance = self.hda_manager.get_accessible(id=dataset_id, user=trans.user) + else: + dataset_instance = self.hda_manager.ldda_manager.get(trans, id=dataset_id) + # TODO Raise error if no ID passed? Never happens when called from Job API endpoints + return dataset_instance From 94fa1e0c67c69a43832b60a7b13b7550037ae84a Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 3 Oct 2023 17:32:50 +0200 Subject: [PATCH 003/110] Refactor common_problems operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index c110110bdc69..0d0d3750cc4f 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -19,8 +19,10 @@ from fastapi import ( Depends, + Path, Query, ) +from typing_extensions import Annotated from galaxy import ( exceptions, @@ -39,7 +41,10 @@ summarize_job_parameters, ) from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.schema import JobIndexSortByEnum +from galaxy.schema.schema import ( + JobIndexSortByEnum, + JobInputSummary, +) from galaxy.schema.types import OffsetNaiveDatetime from galaxy.web import ( expose_api, @@ -166,6 +171,8 @@ free_text_fields=["user", "tool", "handler", "runner"], ) +JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="TODO") + @router.cbv class FastAPIJobs: @@ -227,19 +234,17 @@ def index( ) return self.service.index(trans, payload) - -class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): - job_manager = depends(JobManager) - job_search = depends(JobSearch) - hda_manager = depends(hdas.HDAManager) - - @expose_api - def common_problems(self, trans: ProvidesUserContext, id, **kwd): - """ - * GET /api/jobs/{id}/common_problems - check inputs and job for common potential problems to aid in error reporting - """ - job = self.__get_job(trans, id) + @router.get( + "/api/jobs/{id}/common_problems", + name="check_common_problems", + summary="Check inputs and job for common potential problems to aid in error reporting", + ) + def common_problems( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + trans: ProvidesUserContext = DependsOnTrans, + ) -> JobInputSummary: + job = self.service.get_job(trans=trans, job_id=id) seen_ids = set() has_empty_inputs = False has_duplicate_inputs = False @@ -257,7 +262,13 @@ def common_problems(self, trans: ProvidesUserContext, id, **kwd): # TODO: check percent of failing jobs around a window on job.update_time for handler - report if high. # TODO: check percent of failing jobs around a window on job.update_time for destination_id - report if high. # TODO: sniff inputs (add flag to allow checking files?) - return {"has_empty_inputs": has_empty_inputs, "has_duplicate_inputs": has_duplicate_inputs} + return JobInputSummary(has_empty_inputs=has_empty_inputs, has_duplicate_inputs=has_duplicate_inputs) + + +class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): + job_manager = depends(JobManager) + job_search = depends(JobSearch) + hda_manager = depends(hdas.HDAManager) @expose_api def inputs(self, trans: ProvidesUserContext, id, **kwd) -> List[dict]: From a03984278e7d8950129a9a84d20931296c6b245d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 3 Oct 2023 17:33:51 +0200 Subject: [PATCH 004/110] Remove mapping to the legacy route of common_problems operation --- lib/galaxy/webapps/galaxy/buildapp.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 5048abcc36c8..5e55c04421ae 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -945,13 +945,13 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio webapp.mapper.connect( "job_error", "/api/jobs/{id}/error", controller="jobs", action="error", conditions=dict(method=["POST"]) ) - webapp.mapper.connect( - "common_problems", - "/api/jobs/{id}/common_problems", - controller="jobs", - action="common_problems", - conditions=dict(method=["GET"]), - ) + # webapp.mapper.connect( + # "common_problems", + # "/api/jobs/{id}/common_problems", + # controller="jobs", + # action="common_problems", + # conditions=dict(method=["GET"]), + # ) # Job metrics and parameters by job id or dataset id (for slightly different accessibility checking) webapp.mapper.connect( "destination_params", From e7e4fbe7c4280439831d3b09090cb0a48464211a Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 3 Oct 2023 17:34:24 +0200 Subject: [PATCH 005/110] Add pydantic model for common_problems operation --- lib/galaxy/schema/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 417c69f83cf0..28ab10e963a4 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -3631,3 +3631,8 @@ class DatasetSummary(Model): file_size: int total_size: int uuid: UUID4 = UuidField + + +class JobInputSummary(Model): + has_empty_inputs: bool + has_duplicate_inputs: bool From defec91551e51ce7165d56667a8d74367032870d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 3 Oct 2023 18:18:10 +0200 Subject: [PATCH 006/110] Update client schema --- client/src/api/schema/schema.ts | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 1eb290ef5ae3..7a9f29f5f354 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -904,6 +904,10 @@ export interface paths { */ get: operations["show_api_jobs__id__get"]; }; + "/api/jobs/{id}/common_problems": { + /** Check inputs and job for common potential problems to aid in error reporting */ + get: operations["check_common_problems_api_jobs__id__common_problems_get"]; + }; "/api/jobs/{job_id}/oidc-tokens": { /** * Get a fresh OIDC token @@ -6248,6 +6252,16 @@ export interface components { * @enum {string} */ JobIndexViewEnum: "collection" | "admin_job_list"; + /** + * JobInputSummary + * @description Base model definition with common configuration used by all derived models. + */ + JobInputSummary: { + /** Has Duplicate Inputs */ + has_duplicate_inputs: boolean; + /** Has Empty Inputs */ + has_empty_inputs: boolean; + }; /** JobLock */ JobLock: { /** @@ -14703,6 +14717,33 @@ export interface operations { }; }; }; + check_common_problems_api_jobs__id__common_problems_get: { + /** Check inputs and job for common potential problems to aid in error reporting */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description TODO */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobInputSummary"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_token_api_jobs__job_id__oidc_tokens_get: { /** * Get a fresh OIDC token From 5555d1f97223f6ce8582e796ed5f17845157753a Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:05:02 +0200 Subject: [PATCH 007/110] Create schema file for jobs API and move corresponding model to it --- lib/galaxy/schema/jobs.py | 6 ++++++ lib/galaxy/schema/schema.py | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 lib/galaxy/schema/jobs.py diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py new file mode 100644 index 000000000000..63c24071c34c --- /dev/null +++ b/lib/galaxy/schema/jobs.py @@ -0,0 +1,6 @@ +from galaxy.schema.schema import Model + + +class JobInputSummary(Model): + has_empty_inputs: bool + has_duplicate_inputs: bool diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 28ab10e963a4..417c69f83cf0 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -3631,8 +3631,3 @@ class DatasetSummary(Model): file_size: int total_size: int uuid: UUID4 = UuidField - - -class JobInputSummary(Model): - has_empty_inputs: bool - has_duplicate_inputs: bool From c6dab1c0fc887429f4b9f08e74986c87a234798b Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:06:49 +0200 Subject: [PATCH 008/110] Move dictify methods to JobsService --- lib/galaxy/webapps/galaxy/services/jobs.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index 892902574962..78e61304edf1 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -2,6 +2,7 @@ from typing import ( Any, Dict, + List, Optional, ) @@ -17,7 +18,8 @@ view_show_job, ) from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.schema import JobIndexQueryPayload +from galaxy.schema.schema import EncodedDatasetSourceId, JobIndexQueryPayload +from galaxy.schema.jobs import JobAssociation class JobIndexViewEnum(str, Enum): @@ -110,3 +112,19 @@ def get_job( dataset_instance = self.hda_manager.ldda_manager.get(trans, id=dataset_id) # TODO Raise error if no ID passed? Never happens when called from Job API endpoints return dataset_instance + + def dictify_associations(self, trans, *association_lists) -> List[JobAssociation]: + rval: List[dict] = [] + for association_list in association_lists: + rval.extend(self.__dictify_association(trans, a) for a in association_list) + return rval + + def __dictify_association(self, trans, job_dataset_association) -> JobAssociation: + dataset_dict = None + dataset = job_dataset_association.dataset + if dataset: + if isinstance(dataset, model.HistoryDatasetAssociation): + dataset_dict = EncodedDatasetSourceId(src="hda", id=dataset.id) + else: + dataset_dict = EncodedDatasetSourceId(src="ldda", id=dataset.id) + return JobAssociation(name=job_dataset_association.name, dataset=dataset_dict) From 2975601cb41d6bd7cd653abd5ea14487c0e8384f Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:10:03 +0200 Subject: [PATCH 009/110] Refactor resume operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 49 +++++++++++++-------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 0d0d3750cc4f..811ca4874809 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -41,10 +41,8 @@ summarize_job_parameters, ) from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.schema import ( - JobIndexSortByEnum, - JobInputSummary, -) +from galaxy.schema.jobs import JobAssociation, JobInputSummary +from galaxy.schema.schema import JobIndexSortByEnum from galaxy.schema.types import OffsetNaiveDatetime from galaxy.web import ( expose_api, @@ -171,7 +169,7 @@ free_text_fields=["user", "tool", "handler", "runner"], ) -JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="TODO") +JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="The ID of the job") @router.cbv @@ -264,10 +262,30 @@ def common_problems( # TODO: sniff inputs (add flag to allow checking files?) return JobInputSummary(has_empty_inputs=has_empty_inputs, has_duplicate_inputs=has_duplicate_inputs) + @router.put( + "/api/jobs/{id}/resume", + name="resume_paused_job", + summary="Resumes a paused job.", + ) + def resume( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + trans: ProvidesUserContext = DependsOnTrans, + ) -> List[JobAssociation]: + job = self.service.get_job(trans, job_id=id) + if not job: + raise exceptions.ObjectNotFound("Could not access job with the given id") + if job.state == job.states.PAUSED: + job.resume() + else: + exceptions.RequestParameterInvalidException(f"Job with id '{job.tool_id}' is not paused") + return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): job_manager = depends(JobManager) job_search = depends(JobSearch) + service = depends(JobsService) hda_manager = depends(hdas.HDAManager) @expose_api @@ -319,27 +337,6 @@ def delete(self, trans: ProvidesUserContext, id, **kwd): message = payload.get("message", None) return self.job_manager.stop(job, message=message) - @expose_api - def resume(self, trans: ProvidesUserContext, id, **kwd) -> List[dict]: - """ - * PUT /api/jobs/{id}/resume - Resumes a paused job - - :type id: string - :param id: Encoded job id - - :rtype: list of dicts - :returns: list of dictionaries containing output dataset associations - """ - job = self.__get_job(trans, id) - if not job: - raise exceptions.ObjectNotFound("Could not access job with the given id") - if job.state == job.states.PAUSED: - job.resume() - else: - exceptions.RequestParameterInvalidException(f"Job with id '{job.tool_id}' is not paused") - return self.__dictify_associations(trans, job.output_datasets, job.output_library_datasets) - @expose_api_anonymous def metrics(self, trans: ProvidesUserContext, **kwd): """ From ea520dfc2173ac7d67946ced4699fd43a38fdb57 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:10:48 +0200 Subject: [PATCH 010/110] Add pydantic model for dictify methods --- lib/galaxy/schema/jobs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 63c24071c34c..2a62c9315538 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,6 +1,11 @@ -from galaxy.schema.schema import Model +from galaxy.schema.schema import EncodedDatasetSourceId, Model class JobInputSummary(Model): has_empty_inputs: bool has_duplicate_inputs: bool + + +class JobAssociation(Model): + name: str + dataset: EncodedDatasetSourceId From ebdd5c95533b8fd0c59b024cd9e2cb35cd842d56 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:11:39 +0200 Subject: [PATCH 011/110] Remove mapping to the legacy route of resume operation --- lib/galaxy/webapps/galaxy/buildapp.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 5e55c04421ae..b0fe90c8697e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -939,20 +939,9 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio action="build_for_rerun", conditions=dict(method=["GET"]), ) - webapp.mapper.connect( - "resume", "/api/jobs/{id}/resume", controller="jobs", action="resume", conditions=dict(method=["PUT"]) - ) webapp.mapper.connect( "job_error", "/api/jobs/{id}/error", controller="jobs", action="error", conditions=dict(method=["POST"]) ) - # webapp.mapper.connect( - # "common_problems", - # "/api/jobs/{id}/common_problems", - # controller="jobs", - # action="common_problems", - # conditions=dict(method=["GET"]), - # ) - # Job metrics and parameters by job id or dataset id (for slightly different accessibility checking) webapp.mapper.connect( "destination_params", "/api/jobs/{job_id}/destination_params", From 347fdb5e5f2fd663225a772295e40e332d8c9676 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:18:14 +0200 Subject: [PATCH 012/110] Fix typing mistake --- lib/galaxy/webapps/galaxy/services/jobs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index 78e61304edf1..a69ca6d66862 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -18,8 +18,11 @@ view_show_job, ) from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.schema import EncodedDatasetSourceId, JobIndexQueryPayload from galaxy.schema.jobs import JobAssociation +from galaxy.schema.schema import ( + EncodedDatasetSourceId, + JobIndexQueryPayload, +) class JobIndexViewEnum(str, Enum): @@ -114,7 +117,7 @@ def get_job( return dataset_instance def dictify_associations(self, trans, *association_lists) -> List[JobAssociation]: - rval: List[dict] = [] + rval: List[JobAssociation] = [] for association_list in association_lists: rval.extend(self.__dictify_association(trans, a) for a in association_list) return rval From ab971e2c25ee0b54fc66cb5676fdf3f2df5455e5 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:18:40 +0200 Subject: [PATCH 013/110] Fix codestyle mistakes --- lib/galaxy/schema/jobs.py | 5 ++++- lib/galaxy/webapps/galaxy/api/jobs.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 2a62c9315538..7a1646118fcb 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,4 +1,7 @@ -from galaxy.schema.schema import EncodedDatasetSourceId, Model +from galaxy.schema.schema import ( + EncodedDatasetSourceId, + Model, +) class JobInputSummary(Model): diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 811ca4874809..110c4e353a01 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -41,7 +41,10 @@ summarize_job_parameters, ) from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.jobs import JobAssociation, JobInputSummary +from galaxy.schema.jobs import ( + JobAssociation, + JobInputSummary, +) from galaxy.schema.schema import JobIndexSortByEnum from galaxy.schema.types import OffsetNaiveDatetime from galaxy.web import ( From 141b7749d692e2251c48c785616ab27b42de7137 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 5 Oct 2023 09:20:14 +0200 Subject: [PATCH 014/110] Update client schema --- client/src/api/schema/schema.ts | 42 ++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 7a9f29f5f354..04172e75a173 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -908,6 +908,10 @@ export interface paths { /** Check inputs and job for common potential problems to aid in error reporting */ get: operations["check_common_problems_api_jobs__id__common_problems_get"]; }; + "/api/jobs/{id}/resume": { + /** Resumes a paused job. */ + put: operations["resume_paused_job_api_jobs__id__resume_put"]; + }; "/api/jobs/{job_id}/oidc-tokens": { /** * Get a fresh OIDC token @@ -6110,6 +6114,15 @@ export interface components { * @enum {string} */ ItemsFromSrc: "url" | "files" | "path" | "ftp_import" | "server_dir"; + /** + * JobAssociation + * @description Base model definition with common configuration used by all derived models. + */ + JobAssociation: { + dataset: components["schemas"]["EncodedDatasetSourceId"]; + /** Name */ + name: string; + }; /** * JobExportHistoryArchiveListResponse * @description Base model definition with common configuration used by all derived models. @@ -14724,7 +14737,7 @@ export interface operations { header?: { "run-as"?: string; }; - /** @description TODO */ + /** @description The ID of the job */ path: { id: string; }; @@ -14744,6 +14757,33 @@ export interface operations { }; }; }; + resume_paused_job_api_jobs__id__resume_put: { + /** Resumes a paused job. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobAssociation"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_token_api_jobs__job_id__oidc_tokens_get: { /** * Get a fresh OIDC token From 7f48d75e684558217f46db52e9e92a4fbd4eb119 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 9 Oct 2023 10:33:55 +0200 Subject: [PATCH 015/110] Reorder endpoint operations --- lib/galaxy/webapps/galaxy/api/jobs.py | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 110c4e353a01..a578582b5d3f 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -180,22 +180,6 @@ class FastAPIJobs: service: JobsService = depends(JobsService) manager: JobManager = depends(JobManager) - @router.get("/api/jobs/{id}") - def show( - self, - id: DecodedDatabaseIdField, - trans: ProvidesUserContext = DependsOnTrans, - full: Optional[bool] = False, - ) -> Dict[str, Any]: - """ - Return dictionary containing description of job data - - Parameters - - id: ID of job to return - - full: Return extra information ? - """ - return self.service.show(trans, id, bool(full)) - @router.get("/api/jobs") def index( self, @@ -284,6 +268,22 @@ def resume( exceptions.RequestParameterInvalidException(f"Job with id '{job.tool_id}' is not paused") return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + @router.get("/api/jobs/{id}") + def show( + self, + id: DecodedDatabaseIdField, + trans: ProvidesUserContext = DependsOnTrans, + full: Optional[bool] = False, + ) -> Dict[str, Any]: + """ + Return dictionary containing description of job data + + Parameters + - id: ID of job to return + - full: Return extra information ? + """ + return self.service.show(trans, id, bool(full)) + class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): job_manager = depends(JobManager) From c9cf40b868436766f69f62426502bba7a99f9709 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 9 Oct 2023 17:11:45 +0200 Subject: [PATCH 016/110] Refactor error operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 88 +++++++++++++++------------ 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index a578582b5d3f..10e719a270fe 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -18,10 +18,12 @@ ) from fastapi import ( + Body, Depends, Path, Query, ) +from pydantic import Required from typing_extensions import Annotated from galaxy import ( @@ -33,6 +35,7 @@ ProvidesHistoryContext, ProvidesUserContext, ) +from galaxy.managers.hdas import HDAManager from galaxy.managers.jobs import ( JobManager, JobSearch, @@ -43,7 +46,9 @@ from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.jobs import ( JobAssociation, + JobErrorSummary, JobInputSummary, + ReportJobErrorPayload, ) from galaxy.schema.schema import JobIndexSortByEnum from galaxy.schema.types import OffsetNaiveDatetime @@ -174,11 +179,14 @@ JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="The ID of the job") +ReportErrorBody = Body(default=Required, title="Report error", description="The values to report an Error") + @router.cbv class FastAPIJobs: service: JobsService = depends(JobsService) manager: JobManager = depends(JobManager) + hda_manager: HDAManager = depends(HDAManager) @router.get("/api/jobs") def index( @@ -268,6 +276,46 @@ def resume( exceptions.RequestParameterInvalidException(f"Job with id '{job.tool_id}' is not paused") return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + @router.post( + "/api/jobs/{id}/error", + name="report_error", + summary="Submits a bug report via the API.", + ) + def error( + self, + payload: Annotated[ReportJobErrorPayload, ReportErrorBody], + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + # payload: Any = {"dataset_id": 1, "email": None, "message": None}, + trans: ProvidesUserContext = DependsOnTrans, + ) -> JobErrorSummary: + """ + :rtype: dictionary + :returns: dictionary containing information regarding where the error report was sent. + """ + # Get dataset on which this error was triggered + dataset_id = payload.dataset_id + # if not dataset_id: + # raise exceptions.RequestParameterMissingException("No dataset_id") + dataset = self.hda_manager.get_accessible(id=dataset_id, user=trans.user) + # Get job + job = self.service.get_job(trans, id) + if dataset.creating_job.id != job.id: + raise exceptions.RequestParameterInvalidException("dataset_id was not created by job_id") + tool = trans.app.toolbox.get_tool(job.tool_id, tool_version=job.tool_version) or None + email = payload.email + if not email and not trans.anonymous: + email = trans.user.email + messages = trans.app.error_reports.default_error_plugin.submit_report( + dataset=dataset, + job=job, + tool=tool, + user_submission=True, + user=trans.user, + email=email, + message=payload.message, + ) + return JobErrorSummary(messages=messages) + @router.get("/api/jobs/{id}") def show( self, @@ -518,43 +566,3 @@ def search(self, trans: ProvidesHistoryContext, payload: dict, **kwd): if job: jobs.append(job) return [self.encode_all_ids(trans, single_job.to_dict("element"), True) for single_job in jobs] - - @expose_api_anonymous - def error(self, trans: ProvidesUserContext, id, payload, **kwd): - """ - error( trans, id ) - * POST /api/jobs/{id}/error - submits a bug report via the API. - - :type id: string - :param id: Encoded job id - - :rtype: dictionary - :returns: dictionary containing information regarding where the error report was sent. - """ - # Get dataset on which this error was triggered - dataset_id = payload.get("dataset_id") - if not dataset_id: - raise exceptions.RequestParameterMissingException("No dataset_id") - decoded_dataset_id = self.decode_id(dataset_id) - dataset = self.hda_manager.get_accessible(decoded_dataset_id, trans.user) - - # Get job - job = self.__get_job(trans, id) - if dataset.creating_job.id != job.id: - raise exceptions.RequestParameterInvalidException("dataset_id was not created by job_id") - tool = trans.app.toolbox.get_tool(job.tool_id, tool_version=job.tool_version) or None - email = payload.get("email") - if not email and not trans.anonymous: - email = trans.user.email - messages = trans.app.error_reports.default_error_plugin.submit_report( - dataset=dataset, - job=job, - tool=tool, - user_submission=True, - user=trans.user, - email=email, - message=payload.get("message"), - ) - - return {"messages": messages} From 5096cbe8eed5ed2b3baafa43b86830ac5d00310b Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 9 Oct 2023 17:14:24 +0200 Subject: [PATCH 017/110] Add pydantic models for error operation --- lib/galaxy/schema/jobs.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 7a1646118fcb..3220b4b9f036 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,3 +1,9 @@ +from typing import ( + List, + Optional, +) + +from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import ( EncodedDatasetSourceId, Model, @@ -9,6 +15,18 @@ class JobInputSummary(Model): has_duplicate_inputs: bool +# TODO: Use Tuple again when `make update-client-api-schema` supports them +class JobErrorSummary(Model): + # messages: List[Union[Tuple[str, str], List[str]]] + messages: List[List[str]] + + class JobAssociation(Model): name: str dataset: EncodedDatasetSourceId + + +class ReportJobErrorPayload(Model): + dataset_id: DecodedDatabaseIdField + email: Optional[str] = None + message: Optional[str] = None From e964c9eb390f8578d9797e3ef9d6e7e7014d289b Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 9 Oct 2023 17:15:05 +0200 Subject: [PATCH 018/110] Remove mapping to the legacy route of the error operation --- lib/galaxy/webapps/galaxy/buildapp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index b0fe90c8697e..8f5310f3a10f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -939,9 +939,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio action="build_for_rerun", conditions=dict(method=["GET"]), ) - webapp.mapper.connect( - "job_error", "/api/jobs/{id}/error", controller="jobs", action="error", conditions=dict(method=["POST"]) - ) webapp.mapper.connect( "destination_params", "/api/jobs/{job_id}/destination_params", From 14ba8b042c4f32b71b85444c80127855892347d9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 9 Oct 2023 17:15:26 +0200 Subject: [PATCH 019/110] Update client schema --- client/src/api/schema/schema.ts | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 04172e75a173..65e21d78d7ec 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -908,6 +908,14 @@ export interface paths { /** Check inputs and job for common potential problems to aid in error reporting */ get: operations["check_common_problems_api_jobs__id__common_problems_get"]; }; + "/api/jobs/{id}/error": { + /** + * Submits a bug report via the API. + * @description :rtype: dictionary + * :returns: dictionary containing information regarding where the error report was sent. + */ + post: operations["report_error_api_jobs__id__error_post"]; + }; "/api/jobs/{id}/resume": { /** Resumes a paused job. */ put: operations["resume_paused_job_api_jobs__id__resume_put"]; @@ -6123,6 +6131,14 @@ export interface components { /** Name */ name: string; }; + /** + * JobErrorSummary + * @description Base model definition with common configuration used by all derived models. + */ + JobErrorSummary: { + /** Messages */ + messages: string[][]; + }; /** * JobExportHistoryArchiveListResponse * @description Base model definition with common configuration used by all derived models. @@ -8142,6 +8158,21 @@ export interface components { */ remote_user_email: string; }; + /** + * ReportJobErrorPayload + * @description Base model definition with common configuration used by all derived models. + */ + ReportJobErrorPayload: { + /** + * Dataset Id + * @example 0123456789ABCDEF + */ + dataset_id: string; + /** Email */ + email?: string; + /** Message */ + message?: string; + }; /** * RequestDataType * @description Particular pieces of information that can be requested for a dataset. @@ -14757,6 +14788,42 @@ export interface operations { }; }; }; + report_error_api_jobs__id__error_post: { + /** + * Submits a bug report via the API. + * @description :rtype: dictionary + * :returns: dictionary containing information regarding where the error report was sent. + */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReportJobErrorPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobErrorSummary"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; resume_paused_job_api_jobs__id__resume_put: { /** Resumes a paused job. */ parameters: { From 7d78bf564748c5db61137691266d577e5b1aa561 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 9 Oct 2023 17:15:47 +0200 Subject: [PATCH 020/110] Refactor error test --- lib/galaxy_test/api/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index c2e289bd435f..01d186bfd027 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -488,7 +488,7 @@ def _run_error_report(self, history_id): job_id = run_response["jobs"][0]["id"] self.dataset_populator.wait_for_job(job_id) dataset_id = run_response["outputs"][0]["id"] - response = self._post(f"jobs/{job_id}/error", data={"dataset_id": dataset_id}) + response = self._post(f"jobs/{job_id}/error", data={"dataset_id": dataset_id}, json=True) assert response.status_code == 200, response.text @skip_without_tool("detect_errors_aggressive") From ee4731638dc48cad3891a9ece248dfbdfddd09f9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 16 Oct 2023 21:09:19 +0200 Subject: [PATCH 021/110] Refactor search operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 112 +++++++++++++------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 10e719a270fe..37cab964de11 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -4,6 +4,7 @@ .. seealso:: :class:`galaxy.model.Jobs` """ +import json import logging from datetime import ( date, @@ -49,6 +50,7 @@ JobErrorSummary, JobInputSummary, ReportJobErrorPayload, + SearchJobsPayload, ) from galaxy.schema.schema import JobIndexSortByEnum from galaxy.schema.types import OffsetNaiveDatetime @@ -180,6 +182,7 @@ JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="The ID of the job") ReportErrorBody = Body(default=Required, title="Report error", description="The values to report an Error") +SearchJobBody = Body(default=Required, title="Search job", description="The values to search an Job") @router.cbv @@ -187,6 +190,7 @@ class FastAPIJobs: service: JobsService = depends(JobsService) manager: JobManager = depends(JobManager) hda_manager: HDAManager = depends(HDAManager) + job_search: JobSearch = depends(JobSearch) @router.get("/api/jobs") def index( @@ -285,7 +289,6 @@ def error( self, payload: Annotated[ReportJobErrorPayload, ReportErrorBody], id: Annotated[DecodedDatabaseIdField, JobIdPathParam], - # payload: Any = {"dataset_id": 1, "email": None, "message": None}, trans: ProvidesUserContext = DependsOnTrans, ) -> JobErrorSummary: """ @@ -294,8 +297,6 @@ def error( """ # Get dataset on which this error was triggered dataset_id = payload.dataset_id - # if not dataset_id: - # raise exceptions.RequestParameterMissingException("No dataset_id") dataset = self.hda_manager.get_accessible(id=dataset_id, user=trans.user) # Get job job = self.service.get_job(trans, id) @@ -316,6 +317,59 @@ def error( ) return JobErrorSummary(messages=messages) + @router.post( + "/api/jobs/search", + name="search_jobs", + summary="Return jobs for current user", + ) + def search( + self, + payload: Annotated[SearchJobsPayload, SearchJobBody], + trans: ProvidesHistoryContext = DependsOnTrans, + ): + """ + :type payload: dict + :param payload: Dictionary containing description of requested job. This is in the same format as + a request to POST /apt/tools would take to initiate a job + + :rtype: list + :returns: list of dictionaries containing summary job information of the jobs that match the requested job run + + This method is designed to scan the list of previously run jobs and find records of jobs that had + the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply + recycle the old results. + """ + tool_id = payload.tool_id + + tool = trans.app.toolbox.get_tool(tool_id) + if tool is None: + raise exceptions.ObjectNotFound("Requested tool not found") + inputs = json.loads(payload.inputs) + # Find files coming in as multipart file data and add to inputs. + for k, v in payload.__annotations__.items(): + if k.startswith("files_") or k.startswith("__files_"): + inputs[k] = v + request_context = WorkRequestContext(app=trans.app, user=trans.user, history=trans.history) + all_params, all_errors, _, _ = tool.expand_incoming( + trans=trans, incoming=inputs, request_context=request_context + ) + if any(all_errors): + return [] + params_dump = [tool.params_to_strings(param, trans.app, nested=True) for param in all_params] + jobs = [] + for param_dump, param in zip(params_dump, all_params): + job = self.job_search.by_tool_input( + trans=trans, + tool_id=tool_id, + tool_version=tool.version, + param=param, + param_dump=param_dump, + job_state=payload.state, + ) + if job: + jobs.append(job) + return [self.service.encode_all_ids(single_job.to_dict("element"), True) for single_job in jobs] + @router.get("/api/jobs/{id}") def show( self, @@ -514,55 +568,3 @@ def __get_job(self, trans, job_id=None, dataset_id=None, **kwd): def create(self, trans: ProvidesUserContext, payload, **kwd): """See the create method in tools.py in order to submit a job.""" raise exceptions.NotImplemented("Please POST to /api/tools instead.") - - @expose_api - def search(self, trans: ProvidesHistoryContext, payload: dict, **kwd): - """ - search( trans, payload ) - * POST /api/jobs/search: - return jobs for current user - - :type payload: dict - :param payload: Dictionary containing description of requested job. This is in the same format as - a request to POST /apt/tools would take to initiate a job - - :rtype: list - :returns: list of dictionaries containing summary job information of the jobs that match the requested job run - - This method is designed to scan the list of previously run jobs and find records of jobs that had - the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply - recycle the old results. - """ - tool_id = payload.get("tool_id") - if tool_id is None: - raise exceptions.RequestParameterMissingException("No tool id") - tool = trans.app.toolbox.get_tool(tool_id) - if tool is None: - raise exceptions.ObjectNotFound("Requested tool not found") - if "inputs" not in payload: - raise exceptions.RequestParameterMissingException("No inputs defined") - inputs = payload.get("inputs", {}) - # Find files coming in as multipart file data and add to inputs. - for k, v in payload.items(): - if k.startswith("files_") or k.startswith("__files_"): - inputs[k] = v - request_context = WorkRequestContext(app=trans.app, user=trans.user, history=trans.history) - all_params, all_errors, _, _ = tool.expand_incoming( - trans=trans, incoming=inputs, request_context=request_context - ) - if any(all_errors): - return [] - params_dump = [tool.params_to_strings(param, self.app, nested=True) for param in all_params] - jobs = [] - for param_dump, param in zip(params_dump, all_params): - job = self.job_search.by_tool_input( - trans=trans, - tool_id=tool_id, - tool_version=tool.version, - param=param, - param_dump=param_dump, - job_state=payload.get("state"), - ) - if job: - jobs.append(job) - return [self.encode_all_ids(trans, single_job.to_dict("element"), True) for single_job in jobs] From a7d755c05391409247a5c5158647e9283bf6e1e1 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 16 Oct 2023 21:11:10 +0200 Subject: [PATCH 022/110] Add ServiceBase as base class to JobsService --- lib/galaxy/webapps/galaxy/services/jobs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index a69ca6d66862..c182a2ae56cd 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -23,6 +23,8 @@ EncodedDatasetSourceId, JobIndexQueryPayload, ) +from galaxy.security.idencoding import IdEncodingHelper +from galaxy.webapps.galaxy.services.base import ServiceBase class JobIndexViewEnum(str, Enum): @@ -34,17 +36,19 @@ class JobIndexPayload(JobIndexQueryPayload): view: JobIndexViewEnum = JobIndexViewEnum.collection -class JobsService: +class JobsService(ServiceBase): job_manager: JobManager job_search: JobSearch hda_manager: hdas.HDAManager def __init__( self, + security: IdEncodingHelper, job_manager: JobManager, job_search: JobSearch, hda_manager: hdas.HDAManager, ): + super().__init__(security=security) self.job_manager = job_manager self.job_search = job_search self.hda_manager = hda_manager From 220bd84436890389d411396db8ee2ffee7b67514 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 16 Oct 2023 21:13:43 +0200 Subject: [PATCH 023/110] Remove mapping to legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 8f5310f3a10f..1006f7e077e0 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -923,9 +923,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio ) webapp.mapper.resource("job", "jobs", path_prefix="/api") - webapp.mapper.connect( - "job_search", "/api/jobs/search", controller="jobs", action="search", conditions=dict(method=["POST"]) - ) webapp.mapper.connect( "job_inputs", "/api/jobs/{id}/inputs", controller="jobs", action="inputs", conditions=dict(method=["GET"]) ) From 2cf5dbffd8bdf623ea1d404049c9b0e7b4d9316e Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 16 Oct 2023 21:14:12 +0200 Subject: [PATCH 024/110] Add part of the pydantic models for search operation --- lib/galaxy/schema/jobs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 3220b4b9f036..8a6b39c99683 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -3,6 +3,8 @@ Optional, ) +from pydantic import Extra + from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import ( EncodedDatasetSourceId, @@ -30,3 +32,12 @@ class ReportJobErrorPayload(Model): dataset_id: DecodedDatabaseIdField email: Optional[str] = None message: Optional[str] = None + + +class SearchJobsPayload(Model): + tool_id: str + inputs: str + state: str + + class Config: + extra = Extra.allow # This is used for items named file_ and __file_ From 52cc418f8fd1d78988bb4fa9b8ff504b502067e1 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 16 Oct 2023 21:15:20 +0200 Subject: [PATCH 025/110] Refactor search tests to specify payload as json --- lib/galaxy_test/api/test_jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 01d186bfd027..ba79cf3e36f8 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -939,7 +939,7 @@ def test_job_build_for_rerun_list_list(self, history_id): def _job_search(self, tool_id, history_id, inputs): search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) - empty_search_response = self._post("jobs/search", data=search_payload) + empty_search_response = self._post("jobs/search", data=search_payload, json=True) self._assert_status_code_is(empty_search_response, 200) assert len(empty_search_response.json()) == 0 tool_response = self._post("tools", data=search_payload) @@ -966,7 +966,7 @@ def _search(self, payload, expected_search_count=1): return search_count def _search_count(self, search_payload): - search_response = self._post("jobs/search", data=search_payload) + search_response = self._post("jobs/search", data=search_payload, json=True) self._assert_status_code_is(search_response, 200) search_json = search_response.json() return len(search_json) From 9ec6b8cc65bdabcdd2286d7f167c97985bb741ab Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 16 Oct 2023 21:15:37 +0200 Subject: [PATCH 026/110] Regenerate client schema --- client/src/api/schema/schema.ts | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 65e21d78d7ec..54dc4bba893f 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -893,6 +893,22 @@ export interface paths { /** Index */ get: operations["index_api_jobs_get"]; }; + "/api/jobs/search": { + /** + * Return jobs for current user + * @description :type payload: dict + * :param payload: Dictionary containing description of requested job. This is in the same format as + * a request to POST /apt/tools would take to initiate a job + * + * :rtype: list + * :returns: list of dictionaries containing summary job information of the jobs that match the requested job run + * + * This method is designed to scan the list of previously run jobs and find records of jobs that had + * the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply + * recycle the old results. + */ + post: operations["search_jobs_api_jobs_search_post"]; + }; "/api/jobs/{id}": { /** * Show @@ -8264,6 +8280,18 @@ export interface components { */ url: string; }; + /** + * SearchJobsPayload + * @description Base model definition with common configuration used by all derived models. + */ + SearchJobsPayload: { + /** Inputs */ + inputs: string; + /** State */ + state: string; + /** Tool Id */ + tool_id: string; + }; /** * ServerDirElement * @description Base model definition with common configuration used by all derived models. @@ -14725,6 +14753,46 @@ export interface operations { }; }; }; + search_jobs_api_jobs_search_post: { + /** + * Return jobs for current user + * @description :type payload: dict + * :param payload: Dictionary containing description of requested job. This is in the same format as + * a request to POST /apt/tools would take to initiate a job + * + * :rtype: list + * :returns: list of dictionaries containing summary job information of the jobs that match the requested job run + * + * This method is designed to scan the list of previously run jobs and find records of jobs that had + * the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply + * recycle the old results. + */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SearchJobsPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; show_api_jobs__id__get: { /** * Show From 876a2f0615124fb667e9724af19fcc77aab7702a Mon Sep 17 00:00:00 2001 From: Tillman Date: Wed, 18 Oct 2023 09:06:18 +0200 Subject: [PATCH 027/110] Add descriptions to pydantic model --- lib/galaxy/schema/jobs.py | 71 ++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 8a6b39c99683..87132fefd8e0 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -3,7 +3,7 @@ Optional, ) -from pydantic import Extra +from pydantic import Extra, Field, Required from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import ( @@ -13,31 +13,78 @@ class JobInputSummary(Model): - has_empty_inputs: bool - has_duplicate_inputs: bool + has_empty_inputs: bool = Field( + default=Required, + title="Empty inputs", + description="Job has empty inputs.", + ) + has_duplicate_inputs: bool = Field( + default=Required, + title="Duplicate inputs", + description="Job has duplicate inputs.", + ) # TODO: Use Tuple again when `make update-client-api-schema` supports them class JobErrorSummary(Model): # messages: List[Union[Tuple[str, str], List[str]]] - messages: List[List[str]] + messages: List[List[str]] = Field( + default=Required, + title="Error messages", + description="The error messages for the specified job.", + ) class JobAssociation(Model): - name: str - dataset: EncodedDatasetSourceId + name: str = Field( + default=Required, + title="name", + description="The name of the associated dataset.", + ) + dataset: EncodedDatasetSourceId = Field( + default=Required, + title="dataset", + description="The associated dataset.", + ) class ReportJobErrorPayload(Model): - dataset_id: DecodedDatabaseIdField - email: Optional[str] = None - message: Optional[str] = None + dataset_id: DecodedDatabaseIdField = Field( + default=Required, + title="Dataset ID", + description="The dataset ID related to the error.", + ) + # TODO add description + email: Optional[str] = Field( + default=None, + title="Email", + description="", + ) + message: Optional[str] = Field( + default=None, + title="Message", + description="The optional message sent with the error report.", + ) class SearchJobsPayload(Model): - tool_id: str - inputs: str - state: str + tool_id: str = Field( + default=Required, + title="Tool ID", + description="The tool ID related to the job.", + ) + # TODO the inputs are actually a dict, but are passed as a JSON string + # maybe change it? + inputs: str = Field( + default=Required, + title="Inputs", + description="The inputs of the job as a JSON string.", + ) + state: str = Field( + default=Required, + title="State", + description="The state of the job.", + ) class Config: extra = Extra.allow # This is used for items named file_ and __file_ From ca28384ff29a1bc701455f37c9491190bbb5b7b4 Mon Sep 17 00:00:00 2001 From: Tillman Date: Wed, 18 Oct 2023 09:07:02 +0200 Subject: [PATCH 028/110] Add TODO/suggestion to the search operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 37cab964de11..bd241f301f08 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -291,10 +291,6 @@ def error( id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, ) -> JobErrorSummary: - """ - :rtype: dictionary - :returns: dictionary containing information regarding where the error report was sent. - """ # Get dataset on which this error was triggered dataset_id = payload.dataset_id dataset = self.hda_manager.get_accessible(id=dataset_id, user=trans.user) @@ -328,13 +324,6 @@ def search( trans: ProvidesHistoryContext = DependsOnTrans, ): """ - :type payload: dict - :param payload: Dictionary containing description of requested job. This is in the same format as - a request to POST /apt/tools would take to initiate a job - - :rtype: list - :returns: list of dictionaries containing summary job information of the jobs that match the requested job run - This method is designed to scan the list of previously run jobs and find records of jobs that had the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply recycle the old results. @@ -344,6 +333,8 @@ def search( tool = trans.app.toolbox.get_tool(tool_id) if tool is None: raise exceptions.ObjectNotFound("Requested tool not found") + # TODO the inputs are actually a dict, but are passed as a JSON string + # maybe change it? inputs = json.loads(payload.inputs) # Find files coming in as multipart file data and add to inputs. for k, v in payload.__annotations__.items(): From d70865f6808e9e696d3bbbe5cf1d9b249f449d07 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 20 Oct 2023 19:00:20 +0200 Subject: [PATCH 029/110] Refactor some JobModels to enable reuse --- lib/galaxy/schema/schema.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 417c69f83cf0..5ca9dd266655 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1820,19 +1820,22 @@ class JobIdResponse(Model): ) -class JobBaseModel(Model): +class JobIDs(Model): id: DecodedDatabaseIdField = EntityIdField + history_id: Optional[DecodedDatabaseIdField] = Field( + None, + title="History ID", + description="The encoded ID of the history associated with this item.", + ) + + +class JobBaseModel(Model): model_class: JOB_MODEL_CLASS = ModelClassField(JOB_MODEL_CLASS) tool_id: str = Field( ..., title="Tool ID", description="Identifier of the tool that generated this job.", ) - history_id: Optional[DecodedDatabaseIdField] = Field( - None, - title="History ID", - description="The encoded ID of the history associated with this item.", - ) state: JobState = Field( ..., title="State", @@ -1853,7 +1856,7 @@ class JobBaseModel(Model): ) -class JobImportHistoryResponse(JobBaseModel): +class JobImportHistoryResponse(JobBaseModel, JobIDs): message: str = Field( ..., title="Message", @@ -1929,7 +1932,7 @@ class DatasetJobInfo(DatasetSourceId): uuid: UUID4 = UuidField -class JobDetails(JobSummary): +class JobDetails(JobSummary, JobIDs): command_version: str = Field( ..., title="Command Version", From dd63f5afa21c5839b31b4c9d7aae8d73eea8929d Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 20 Oct 2023 19:01:44 +0200 Subject: [PATCH 030/110] Add pydantic models for search operation --- lib/galaxy/schema/jobs.py | 63 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 87132fefd8e0..7e632b4ecadc 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,14 +1,27 @@ from typing import ( + Any, + Dict, List, Optional, ) -from pydantic import Extra, Field, Required +from pydantic import ( + Extra, + Field, + Required, + UUID4, +) -from galaxy.schema.fields import DecodedDatabaseIdField +from galaxy.schema.fields import ( + DecodedDatabaseIdField, + EncodedDatabaseIdField, +) from galaxy.schema.schema import ( EncodedDatasetSourceId, + EntityIdField, + JobSummary, Model, + UuidField, ) @@ -73,12 +86,12 @@ class SearchJobsPayload(Model): title="Tool ID", description="The tool ID related to the job.", ) - # TODO the inputs are actually a dict, but are passed as a JSON string + # TODO the inputs are actually a dict, but are passed as a JSON dump # maybe change it? inputs: str = Field( default=Required, title="Inputs", - description="The inputs of the job as a JSON string.", + description="The inputs of the job as a JSON dump.", ) state: str = Field( default=Required, @@ -88,3 +101,45 @@ class SearchJobsPayload(Model): class Config: extra = Extra.allow # This is used for items named file_ and __file_ + + +class EncodedJobIDs(Model): + id: EncodedDatabaseIdField = EntityIdField + history_id: Optional[EncodedDatabaseIdField] = Field( + None, + title="History ID", + description="The encoded ID of the history associated with this item.", + ) + + +class EncodedDatasetJobInfo(EncodedDatasetSourceId): + uuid: UUID4 = UuidField + + +class EncodedJobDetails(JobSummary, EncodedDatasetSourceId): + command_version: str = Field( + ..., + title="Command Version", + description="Tool version indicated during job execution.", + ) + params: Any = Field( + ..., + title="Parameters", + description=( + "Object containing all the parameters of the tool associated with this job. " + "The specific parameters depend on the tool itself." + ), + ) + inputs: Dict[str, EncodedDatasetJobInfo] = Field( + {}, + title="Inputs", + description="Dictionary mapping all the tool inputs (by name) with the corresponding dataset information.", + ) + outputs: Dict[str, EncodedDatasetJobInfo] = Field( + {}, + title="Outputs", + description="Dictionary mapping all the tool outputs (by name) with the corresponding dataset information.", + ) + # TODO add description, check type and add proper default + copied_from_job_id: Any = Field(default=None, title="Copied from Job-ID", description="?") + output_collections: Any = Field(default={}, title="Output collections", description="?") From 58a59efbcf9999491e61b2d47a788d11405f2118 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 20 Oct 2023 19:02:14 +0200 Subject: [PATCH 031/110] Reformulate TODO --- lib/galaxy/webapps/galaxy/api/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index bd241f301f08..70a3d7abdecc 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -333,7 +333,7 @@ def search( tool = trans.app.toolbox.get_tool(tool_id) if tool is None: raise exceptions.ObjectNotFound("Requested tool not found") - # TODO the inputs are actually a dict, but are passed as a JSON string + # TODO the inputs are actually a dict, but are passed as a JSON dump # maybe change it? inputs = json.loads(payload.inputs) # Find files coming in as multipart file data and add to inputs. From 4b37dd237f8811b141809b4bb56acf1649b715f6 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 20 Oct 2023 19:02:44 +0200 Subject: [PATCH 032/110] Regenerate client schema --- client/src/api/schema/schema.ts | 77 ++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 54dc4bba893f..697cc2193fff 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -896,14 +896,7 @@ export interface paths { "/api/jobs/search": { /** * Return jobs for current user - * @description :type payload: dict - * :param payload: Dictionary containing description of requested job. This is in the same format as - * a request to POST /apt/tools would take to initiate a job - * - * :rtype: list - * :returns: list of dictionaries containing summary job information of the jobs that match the requested job run - * - * This method is designed to scan the list of previously run jobs and find records of jobs that had + * @description This method is designed to scan the list of previously run jobs and find records of jobs that had * the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply * recycle the old results. */ @@ -925,11 +918,7 @@ export interface paths { get: operations["check_common_problems_api_jobs__id__common_problems_get"]; }; "/api/jobs/{id}/error": { - /** - * Submits a bug report via the API. - * @description :rtype: dictionary - * :returns: dictionary containing information regarding where the error report was sent. - */ + /** Submits a bug report via the API. */ post: operations["report_error_api_jobs__id__error_post"]; }; "/api/jobs/{id}/resume": { @@ -6143,8 +6132,15 @@ export interface components { * @description Base model definition with common configuration used by all derived models. */ JobAssociation: { + /** + * dataset + * @description The associated dataset. + */ dataset: components["schemas"]["EncodedDatasetSourceId"]; - /** Name */ + /** + * name + * @description The name of the associated dataset. + */ name: string; }; /** @@ -6152,7 +6148,10 @@ export interface components { * @description Base model definition with common configuration used by all derived models. */ JobErrorSummary: { - /** Messages */ + /** + * Error messages + * @description The error messages for the specified job. + */ messages: string[][]; }; /** @@ -6302,9 +6301,15 @@ export interface components { * @description Base model definition with common configuration used by all derived models. */ JobInputSummary: { - /** Has Duplicate Inputs */ + /** + * Duplicate inputs + * @description Job has duplicate inputs. + */ has_duplicate_inputs: boolean; - /** Has Empty Inputs */ + /** + * Empty inputs + * @description Job has empty inputs. + */ has_empty_inputs: boolean; }; /** JobLock */ @@ -8180,13 +8185,17 @@ export interface components { */ ReportJobErrorPayload: { /** - * Dataset Id + * Dataset ID + * @description The dataset ID related to the error. * @example 0123456789ABCDEF */ dataset_id: string; /** Email */ email?: string; - /** Message */ + /** + * Message + * @description The optional message sent with the error report. + */ message?: string; }; /** @@ -8285,11 +8294,20 @@ export interface components { * @description Base model definition with common configuration used by all derived models. */ SearchJobsPayload: { - /** Inputs */ + /** + * Inputs + * @description The inputs of the job as a JSON dump. + */ inputs: string; - /** State */ + /** + * State + * @description The state of the job. + */ state: string; - /** Tool Id */ + /** + * Tool ID + * @description The tool ID related to the job. + */ tool_id: string; }; /** @@ -14756,14 +14774,7 @@ export interface operations { search_jobs_api_jobs_search_post: { /** * Return jobs for current user - * @description :type payload: dict - * :param payload: Dictionary containing description of requested job. This is in the same format as - * a request to POST /apt/tools would take to initiate a job - * - * :rtype: list - * :returns: list of dictionaries containing summary job information of the jobs that match the requested job run - * - * This method is designed to scan the list of previously run jobs and find records of jobs that had + * @description This method is designed to scan the list of previously run jobs and find records of jobs that had * the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply * recycle the old results. */ @@ -14857,11 +14868,7 @@ export interface operations { }; }; report_error_api_jobs__id__error_post: { - /** - * Submits a bug report via the API. - * @description :rtype: dictionary - * :returns: dictionary containing information regarding where the error report was sent. - */ + /** Submits a bug report via the API. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { From 2e10db9bec00315569f984b3b13161f4ff3de26f Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 14:48:32 +0200 Subject: [PATCH 033/110] Refactor inputs operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 29 ++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 70a3d7abdecc..c09faa602443 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -313,6 +313,19 @@ def error( ) return JobErrorSummary(messages=messages) + @router.get( + "/api/jobs/{id}/inputs", + name="get_inputs", + summary="Returns input datasets created by job.", + ) + def inputs( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + trans: ProvidesUserContext = DependsOnTrans, + ) -> List[JobAssociation]: + job = self.service.get_job(trans=trans, job_id=id) + return self.service.dictify_associations(trans, job.input_datasets, job.input_library_datasets) + @router.post( "/api/jobs/search", name="search_jobs", @@ -384,22 +397,6 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): service = depends(JobsService) hda_manager = depends(hdas.HDAManager) - @expose_api - def inputs(self, trans: ProvidesUserContext, id, **kwd) -> List[dict]: - """ - GET /api/jobs/{id}/inputs - - returns input datasets created by job - - :type id: string - :param id: Encoded job id - - :rtype: list of dicts - :returns: list of dictionaries containing input dataset associations - """ - job = self.__get_job(trans, id) - return self.__dictify_associations(trans, job.input_datasets, job.input_library_datasets) - @expose_api def outputs(self, trans: ProvidesUserContext, id, **kwd) -> List[dict]: """ From e3d3e443c0aa04de16cd607cc971fc771a3344df Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 14:48:47 +0200 Subject: [PATCH 034/110] Remove legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 1006f7e077e0..d6fd00a5bbe4 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -923,9 +923,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio ) webapp.mapper.resource("job", "jobs", path_prefix="/api") - webapp.mapper.connect( - "job_inputs", "/api/jobs/{id}/inputs", controller="jobs", action="inputs", conditions=dict(method=["GET"]) - ) webapp.mapper.connect( "job_outputs", "/api/jobs/{id}/outputs", controller="jobs", action="outputs", conditions=dict(method=["GET"]) ) From 0745696b038d66f6885ce6da9bba4bef41570069 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 14:49:08 +0200 Subject: [PATCH 035/110] Add test for inputs operation --- lib/galaxy_test/api/test_jobs.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index ba79cf3e36f8..f581ef6dd8ce 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -937,6 +937,27 @@ def test_job_build_for_rerun_list_list(self, history_id): ) assert rerun_content == run_content + @pytest.mark.require_new_history + def test_get_inputs(self, history_id): + dataset_id = self.__history_with_ok_dataset(history_id) + inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) + # create a job + search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) + tool_response = self._post("tools", data=search_payload) + self.dataset_populator.wait_for_tool_run(history_id, run_response=tool_response) + # search for the job and get the corresponding id + search_response = self._post("jobs/search", data=search_payload, json=True) + self._assert_status_code_is(search_response, 200) + job_id = search_response.json()[0]["id"] + # get the inputs of the job + job_response = self._get(f"jobs/{job_id}/inputs") + self._assert_status_code_is(job_response, 200) + job_inputs = job_response.json()[0] + # validate response + assert job_inputs["name"] == "input1" + assert job_inputs.get("name") == "input1" + assert job_inputs.get("dataset") == {"src": "hda", "id": dataset_id} + def _job_search(self, tool_id, history_id, inputs): search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) empty_search_response = self._post("jobs/search", data=search_payload, json=True) From e786bd1408ee4675d423485865354b1ca903aa2b Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 15:03:34 +0200 Subject: [PATCH 036/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 697cc2193fff..13fc71d9184d 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -921,6 +921,10 @@ export interface paths { /** Submits a bug report via the API. */ post: operations["report_error_api_jobs__id__error_post"]; }; + "/api/jobs/{id}/inputs": { + /** Returns input datasets created by job. */ + get: operations["get_inputs_api_jobs__id__inputs_get"]; + }; "/api/jobs/{id}/resume": { /** Resumes a paused job. */ put: operations["resume_paused_job_api_jobs__id__resume_put"]; @@ -14899,6 +14903,33 @@ export interface operations { }; }; }; + get_inputs_api_jobs__id__inputs_get: { + /** Returns input datasets created by job. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobAssociation"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; resume_paused_job_api_jobs__id__resume_put: { /** Resumes a paused job. */ parameters: { From 3b34f9bce8a9b34161afc0cb2ca9372378515585 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 16:05:13 +0200 Subject: [PATCH 037/110] Refactor outputs operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 31 ++++++++++++--------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index c09faa602443..86248635f718 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -316,7 +316,7 @@ def error( @router.get( "/api/jobs/{id}/inputs", name="get_inputs", - summary="Returns input datasets created by job.", + summary="Returns input datasets created by a job.", ) def inputs( self, @@ -326,6 +326,19 @@ def inputs( job = self.service.get_job(trans=trans, job_id=id) return self.service.dictify_associations(trans, job.input_datasets, job.input_library_datasets) + @router.get( + "/api/jobs/{id}/outputs", + name="get_outputs", + summary="Returns output datasets created by a job.", + ) + def outputs( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + trans: ProvidesUserContext = DependsOnTrans, + ) -> List[JobAssociation]: + job = self.service.get_job(trans=trans, job_id=id) + return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + @router.post( "/api/jobs/search", name="search_jobs", @@ -397,22 +410,6 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): service = depends(JobsService) hda_manager = depends(hdas.HDAManager) - @expose_api - def outputs(self, trans: ProvidesUserContext, id, **kwd) -> List[dict]: - """ - outputs( trans, id ) - * GET /api/jobs/{id}/outputs - returns output datasets created by job - - :type id: string - :param id: Encoded job id - - :rtype: list of dicts - :returns: list of dictionaries containing output dataset associations - """ - job = self.__get_job(trans, id) - return self.__dictify_associations(trans, job.output_datasets, job.output_library_datasets) - @expose_api def delete(self, trans: ProvidesUserContext, id, **kwd): """ From e4edab81b9a0a6fb8803a72543a74e117ededccd Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 16:05:34 +0200 Subject: [PATCH 038/110] Remove mapping to legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index d6fd00a5bbe4..2d7b53d7414e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -923,9 +923,7 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio ) webapp.mapper.resource("job", "jobs", path_prefix="/api") - webapp.mapper.connect( - "job_outputs", "/api/jobs/{id}/outputs", controller="jobs", action="outputs", conditions=dict(method=["GET"]) - ) + webapp.mapper.connect( "build_for_rerun", "/api/jobs/{id}/build_for_rerun", From 4df0aefede8253482ad0c2c144df8dd22db9b105 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 16:06:03 +0200 Subject: [PATCH 039/110] Refactor inputs test to include testing for outputs --- lib/galaxy_test/api/test_jobs.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index f581ef6dd8ce..23023277dab5 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -938,25 +938,33 @@ def test_job_build_for_rerun_list_list(self, history_id): assert rerun_content == run_content @pytest.mark.require_new_history - def test_get_inputs(self, history_id): + def test_get_inputs_and_outputs(self, history_id): dataset_id = self.__history_with_ok_dataset(history_id) inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) # create a job search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) tool_response = self._post("tools", data=search_payload) self.dataset_populator.wait_for_tool_run(history_id, run_response=tool_response) - # search for the job and get the corresponding id + # search for the job and get the corresponding values search_response = self._post("jobs/search", data=search_payload, json=True) self._assert_status_code_is(search_response, 200) job_id = search_response.json()[0]["id"] + job_first_output_name, job_first_output_values = list(search_response.json()[0]["outputs"].items())[0] # get the inputs of the job job_response = self._get(f"jobs/{job_id}/inputs") self._assert_status_code_is(job_response, 200) - job_inputs = job_response.json()[0] - # validate response - assert job_inputs["name"] == "input1" - assert job_inputs.get("name") == "input1" - assert job_inputs.get("dataset") == {"src": "hda", "id": dataset_id} + job_first_input = job_response.json()[0] + # validate input response + assert job_first_input.get("name") == "input1" + assert job_first_input.get("dataset") == {"src": "hda", "id": dataset_id} + # get the outputs of the job + job_response = self._get(f"jobs/{job_id}/outputs") + self._assert_status_code_is(job_response, 200) + job_first_output = job_response.json()[0] + # validate output response + assert job_first_output.get("name") == job_first_output_name + assert job_first_output.get("dataset").get("id") == job_first_output_values.get("id") + assert job_first_output.get("dataset").get("src") == job_first_output_values.get("src") def _job_search(self, tool_id, history_id, inputs): search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) From fb876e6dff3402afc45f4a2ffd7d5207b1bdf907 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 23 Oct 2023 16:06:25 +0200 Subject: [PATCH 040/110] Regenrate client schema --- client/src/api/schema/schema.ts | 35 +++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 13fc71d9184d..0a4cdf2c49aa 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -922,9 +922,13 @@ export interface paths { post: operations["report_error_api_jobs__id__error_post"]; }; "/api/jobs/{id}/inputs": { - /** Returns input datasets created by job. */ + /** Returns input datasets created by a job. */ get: operations["get_inputs_api_jobs__id__inputs_get"]; }; + "/api/jobs/{id}/outputs": { + /** Returns output datasets created by a job. */ + get: operations["get_outputs_api_jobs__id__outputs_get"]; + }; "/api/jobs/{id}/resume": { /** Resumes a paused job. */ put: operations["resume_paused_job_api_jobs__id__resume_put"]; @@ -14904,7 +14908,34 @@ export interface operations { }; }; get_inputs_api_jobs__id__inputs_get: { - /** Returns input datasets created by job. */ + /** Returns input datasets created by a job. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobAssociation"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_outputs_api_jobs__id__outputs_get: { + /** Returns output datasets created by a job. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { From 0fa51e7e336d71414bac52a593cfdd0c3aeebfcc Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 14:45:53 +0200 Subject: [PATCH 041/110] Refactor pydantic model for search operation --- lib/galaxy/schema/jobs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 7e632b4ecadc..bc8f2590f8a4 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -103,6 +103,10 @@ class Config: extra = Extra.allow # This is used for items named file_ and __file_ +class EncodedDatasetJobInfo(EncodedDatasetSourceId): + uuid: UUID4 = UuidField + + class EncodedJobIDs(Model): id: EncodedDatabaseIdField = EntityIdField history_id: Optional[EncodedDatabaseIdField] = Field( @@ -112,11 +116,7 @@ class EncodedJobIDs(Model): ) -class EncodedDatasetJobInfo(EncodedDatasetSourceId): - uuid: UUID4 = UuidField - - -class EncodedJobDetails(JobSummary, EncodedDatasetSourceId): +class EncodedJobDetails(JobSummary, EncodedJobIDs): command_version: str = Field( ..., title="Command Version", From 7a4350015329a00ffa3d7c0188543898454cb54a Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 14:46:37 +0200 Subject: [PATCH 042/110] Refactor search operation to return pydantic model --- lib/galaxy/webapps/galaxy/api/jobs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 86248635f718..872ca4c523a2 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -46,6 +46,7 @@ ) from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.jobs import ( + EncodedJobDetails, JobAssociation, JobErrorSummary, JobInputSummary, @@ -348,7 +349,7 @@ def search( self, payload: Annotated[SearchJobsPayload, SearchJobBody], trans: ProvidesHistoryContext = DependsOnTrans, - ): + ) -> List[EncodedJobDetails]: """ This method is designed to scan the list of previously run jobs and find records of jobs that had the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply @@ -385,7 +386,8 @@ def search( ) if job: jobs.append(job) - return [self.service.encode_all_ids(single_job.to_dict("element"), True) for single_job in jobs] + # return [self.service.job_to_encoded_details(job=single_job.to_dict("element")) for single_job in jobs] + return [EncodedJobDetails(**single_job.to_dict("element")) for single_job in jobs] @router.get("/api/jobs/{id}") def show( From d96673fcb402a6dd470810f7ef34df90c3890f80 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 15:23:36 +0200 Subject: [PATCH 043/110] Regenerate client schema --- client/src/api/schema/schema.ts | 135 +++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 1 deletion(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 0a4cdf2c49aa..74a7d09c8515 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4102,6 +4102,29 @@ export interface components { * @enum {string} */ ElementsFromType: "archive" | "bagit" | "bagit_archive" | "directory"; + /** + * EncodedDatasetJobInfo + * @description Base model definition with common configuration used by all derived models. + */ + EncodedDatasetJobInfo: { + /** + * ID + * @description The encoded ID of this entity. + * @example 0123456789ABCDEF + */ + id: string; + /** + * Source + * @description The source of this dataset, either `hda` or `ldda` depending of its origin. + */ + src: components["schemas"]["DatasetSourceType"]; + /** + * UUID + * Format: uuid4 + * @description Universal unique identifier for this dataset. + */ + uuid: string; + }; /** * EncodedDatasetSourceId * @description Base model definition with common configuration used by all derived models. @@ -4136,6 +4159,116 @@ export interface components { */ id: string; }; + /** + * EncodedJobDetails + * @description Basic information about a job. + */ + EncodedJobDetails: { + /** + * Command Line + * @description The command line produced by the job. Users can see this value if allowed in the configuration, administrator can always see this value. + */ + command_line?: string; + /** + * Command Version + * @description Tool version indicated during job execution. + */ + command_version: string; + /** + * Copied from Job-ID + * @description ? + */ + copied_from_job_id?: Record; + /** + * Create Time + * Format: date-time + * @description The time and date this item was created. + */ + create_time: string; + /** + * Exit Code + * @description The exit code returned by the tool. Can be unset if the job is not completed yet. + */ + exit_code?: number; + /** + * External ID + * @description The job id used by the external job runner (Condor, Pulsar, etc.)Only administrator can see this value. + */ + external_id?: string; + /** + * Galaxy Version + * @description The (major) version of Galaxy used to create this job. + * @example 21.05 + */ + galaxy_version: string; + /** + * History ID + * @description The encoded ID of the history associated with this item. + * @example 0123456789ABCDEF + */ + history_id?: string; + /** + * ID + * @description The encoded ID of this entity. + * @example 0123456789ABCDEF + */ + id: string; + /** + * Inputs + * @description Dictionary mapping all the tool inputs (by name) with the corresponding dataset information. + * @default {} + */ + inputs?: { + [key: string]: components["schemas"]["EncodedDatasetJobInfo"] | undefined; + }; + /** + * Model class + * @description The name of the database model class. + * @default Job + * @enum {string} + */ + model_class: "Job"; + /** + * Output collections + * @description ? + * @default {} + */ + output_collections?: Record; + /** + * Outputs + * @description Dictionary mapping all the tool outputs (by name) with the corresponding dataset information. + * @default {} + */ + outputs?: { + [key: string]: components["schemas"]["EncodedDatasetJobInfo"] | undefined; + }; + /** + * Parameters + * @description Object containing all the parameters of the tool associated with this job. The specific parameters depend on the tool itself. + */ + params: Record; + /** + * State + * @description Current state of the job. + */ + state: components["schemas"]["JobState"]; + /** + * Tool ID + * @description Identifier of the tool that generated this job. + */ + tool_id: string; + /** + * Update Time + * Format: date-time + * @description The last time and date this item was updated. + */ + update_time: string; + /** + * User Email + * @description The email of the user that owns this job. Only the owner of the job and administrators can see this value. + */ + user_email?: string; + }; /** * ExportHistoryArchivePayload * @description Base model definition with common configuration used by all derived models. @@ -14801,7 +14934,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["EncodedJobDetails"][]; }; }; /** @description Validation Error */ From a7edbc92bba6bb362fd882f4394bf58e696c4304 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 17:29:46 +0200 Subject: [PATCH 044/110] Refactor delete operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 41 +++++++++++++++------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 872ca4c523a2..f87296ad5401 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -46,6 +46,7 @@ ) from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.jobs import ( + DeleteJobPayload, EncodedJobDetails, JobAssociation, JobErrorSummary, @@ -184,6 +185,8 @@ ReportErrorBody = Body(default=Required, title="Report error", description="The values to report an Error") SearchJobBody = Body(default=Required, title="Search job", description="The values to search an Job") +# TODO The endpoint only stops/cancles a job, but is called delete settle for one name +DeleteJobBody = Body(default=None, title="Delete/cancel job", description="The values to delete/cancel a job") @router.cbv @@ -386,7 +389,6 @@ def search( ) if job: jobs.append(job) - # return [self.service.job_to_encoded_details(job=single_job.to_dict("element")) for single_job in jobs] return [EncodedJobDetails(**single_job.to_dict("element")) for single_job in jobs] @router.get("/api/jobs/{id}") @@ -405,6 +407,26 @@ def show( """ return self.service.show(trans, id, bool(full)) + # TODO find the mapping of the legacy route + # TODO rename? --> function cancels/stops job (no deletion involved?) + @router.delete( + "/api/jobs/{id}", + name="cancel_job", + summary="Cancels specified job", + ) + def delete( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + trans: ProvidesUserContext = DependsOnTrans, + payload: Optional[DeleteJobPayload] = DeleteJobBody, + ) -> bool: + job = self.service.get_job(trans=trans, job_id=id) + if payload: + message = payload.message + else: + message = None + return self.manager.stop(job, message=message) + class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): job_manager = depends(JobManager) @@ -412,23 +434,6 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): service = depends(JobsService) hda_manager = depends(hdas.HDAManager) - @expose_api - def delete(self, trans: ProvidesUserContext, id, **kwd): - """ - delete( trans, id ) - * Delete /api/jobs/{id} - cancels specified job - - :type id: string - :param id: Encoded job id - :type message: string - :param message: Stop message. - """ - payload = kwd.get("payload") or {} - job = self.__get_job(trans, id) - message = payload.get("message", None) - return self.job_manager.stop(job, message=message) - @expose_api_anonymous def metrics(self, trans: ProvidesUserContext, **kwd): """ From eec94304b7fd8062fc975ad5e43c904da2e29b4d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 17:30:04 +0200 Subject: [PATCH 045/110] Add pydantic model for delete operation --- lib/galaxy/schema/jobs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index bc8f2590f8a4..83d5edae97a5 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -103,6 +103,14 @@ class Config: extra = Extra.allow # This is used for items named file_ and __file_ +class DeleteJobPayload(Model): + message: Optional[str] = Field( + default=None, + title="Job message", + description="Stop message", + ) + + class EncodedDatasetJobInfo(EncodedDatasetSourceId): uuid: UUID4 = UuidField From 981489f71bc8d3937d1e817c12868df04e186347 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 17:31:08 +0200 Subject: [PATCH 046/110] Create internal method for Job testing and add test for delete operation --- lib/galaxy_test/api/test_jobs.py | 35 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 23023277dab5..7c91357ac524 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -941,13 +941,7 @@ def test_job_build_for_rerun_list_list(self, history_id): def test_get_inputs_and_outputs(self, history_id): dataset_id = self.__history_with_ok_dataset(history_id) inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) - # create a job - search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) - tool_response = self._post("tools", data=search_payload) - self.dataset_populator.wait_for_tool_run(history_id, run_response=tool_response) - # search for the job and get the corresponding values - search_response = self._post("jobs/search", data=search_payload, json=True) - self._assert_status_code_is(search_response, 200) + search_response = self._create_and_search_job(history_id, inputs, tool_id="cat1") job_id = search_response.json()[0]["id"] job_first_output_name, job_first_output_values = list(search_response.json()[0]["outputs"].items())[0] # get the inputs of the job @@ -966,6 +960,33 @@ def test_get_inputs_and_outputs(self, history_id): assert job_first_output.get("dataset").get("id") == job_first_output_values.get("id") assert job_first_output.get("dataset").get("src") == job_first_output_values.get("src") + # @pytest.mark.require_new_history + # def test_delete_job(self, history_id): + # dataset_id = self.__history_with_ok_dataset(history_id) + # inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) + # search_response = self._create_and_search_job(history_id, inputs, tool_id="cat1") + # job_id = search_response.json()[0]["id"] + # # delete the job without message + # delete_job_response = self._delete(f"jobs/{job_id}") + # self._assert_status_code_is(delete_job_response, 200) + # # TODO the job is finished as such it is not stopped Think of another way to test it + # assert delete_job_response.json() == True + # # now that we deleted the job we should not find it anymore + # search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) + # empty_search_response = self._post("jobs/search", data=search_payload, json=True) + # self._assert_status_code_is(empty_search_response, 200) + # assert len(empty_search_response.json()) == 0 + + def _create_and_search_job(self, history_id, inputs, tool_id): + # create a job + search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) + tool_response = self._post("tools", data=search_payload) + self.dataset_populator.wait_for_tool_run(history_id, run_response=tool_response) + # search for the job and get the corresponding values + search_response = self._post("jobs/search", data=search_payload, json=True) + self._assert_status_code_is(search_response, 200) + return search_response + def _job_search(self, tool_id, history_id, inputs): search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) empty_search_response = self._post("jobs/search", data=search_payload, json=True) From 2dc4cb2ea2b60a8f3e2d0f91cdb484d322189654 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 17:32:13 +0200 Subject: [PATCH 047/110] Regenerate client schema --- client/src/api/schema/schema.ts | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 74a7d09c8515..19df83b84b57 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -912,6 +912,8 @@ export interface paths { * - full: Return extra information ? */ get: operations["show_api_jobs__id__get"]; + /** Cancels specified job */ + delete: operations["cancel_job_api_jobs__id__delete"]; }; "/api/jobs/{id}/common_problems": { /** Check inputs and job for common potential problems to aid in error reporting */ @@ -3956,6 +3958,17 @@ export interface components { */ purge?: boolean; }; + /** + * DeleteJobPayload + * @description Base model definition with common configuration used by all derived models. + */ + DeleteJobPayload: { + /** + * Job message + * @description Stop message + */ + message?: string; + }; /** * DeleteLibraryPayload * @description Base model definition with common configuration used by all derived models. @@ -14981,6 +14994,38 @@ export interface operations { }; }; }; + cancel_job_api_jobs__id__delete: { + /** Cancels specified job */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + requestBody?: { + content: { + "application/json": components["schemas"]["DeleteJobPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": boolean; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; check_common_problems_api_jobs__id__common_problems_get: { /** Check inputs and job for common potential problems to aid in error reporting */ parameters: { From 5387bc3cc90e1ded1e3cbd004b32225d3759640d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 18:27:26 +0200 Subject: [PATCH 048/110] Add proper documentation to show operation and use Annotated on delete body --- lib/galaxy/webapps/galaxy/api/jobs.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index f87296ad5401..aed24301f3ce 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -181,12 +181,14 @@ free_text_fields=["user", "tool", "handler", "runner"], ) +FullShowQueryParam: Optional[bool] = Query(title="Full show", description="Show extra information.") + JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="The ID of the job") ReportErrorBody = Body(default=Required, title="Report error", description="The values to report an Error") SearchJobBody = Body(default=Required, title="Search job", description="The values to search an Job") # TODO The endpoint only stops/cancles a job, but is called delete settle for one name -DeleteJobBody = Body(default=None, title="Delete/cancel job", description="The values to delete/cancel a job") +DeleteJobBody = Body(title="Delete/cancel job", description="The values to delete/cancel a job") @router.cbv @@ -391,20 +393,17 @@ def search( jobs.append(job) return [EncodedJobDetails(**single_job.to_dict("element")) for single_job in jobs] - @router.get("/api/jobs/{id}") + @router.get( + "/api/jobs/{id}", + name="show_job", + summary="Return dictionary containing description of job data.", + ) def show( self, - id: DecodedDatabaseIdField, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + full: Annotated[Optional[bool], FullShowQueryParam] = False, trans: ProvidesUserContext = DependsOnTrans, - full: Optional[bool] = False, ) -> Dict[str, Any]: - """ - Return dictionary containing description of job data - - Parameters - - id: ID of job to return - - full: Return extra information ? - """ return self.service.show(trans, id, bool(full)) # TODO find the mapping of the legacy route @@ -418,7 +417,7 @@ def delete( self, id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, - payload: Optional[DeleteJobPayload] = DeleteJobBody, + payload: Annotated[Optional[DeleteJobPayload], DeleteJobBody] = None, ) -> bool: job = self.service.get_job(trans=trans, job_id=id) if payload: From e842c82d9a48d1e97fe0fe0d45769d8c89154ce4 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 20:03:21 +0200 Subject: [PATCH 049/110] Refactor destination_params operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 32 +++++++++++++-------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index aed24301f3ce..5162f8b2b104 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -49,6 +49,7 @@ DeleteJobPayload, EncodedJobDetails, JobAssociation, + JobDestinationParams, JobErrorSummary, JobInputSummary, ReportJobErrorPayload, @@ -59,7 +60,6 @@ from galaxy.web import ( expose_api, expose_api_anonymous, - require_admin, ) from galaxy.webapps.base.controller import UsesVisualizationMixin from galaxy.webapps.galaxy.api import ( @@ -345,6 +345,20 @@ def outputs( job = self.service.get_job(trans=trans, job_id=id) return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + @router.get( + "/api/jobs/{job_id}/destination_params", + name="destination_params_job", + summary="Return destination parameters for specified job.", + require_admin=True, + ) + def destination_params( + self, + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + trans: ProvidesUserContext = DependsOnTrans, + ) -> JobDestinationParams: + job = self.service.get_job(trans, job_id=job_id) + return JobDestinationParams(**summarize_destination_params(trans, job)) + @router.post( "/api/jobs/search", name="search_jobs", @@ -457,22 +471,6 @@ def metrics(self, trans: ProvidesUserContext, **kwd): job = self.__get_job(trans, **kwd) return summarize_job_metrics(trans, job) - @require_admin - @expose_api - def destination_params(self, trans: ProvidesUserContext, **kwd): - """ - * GET /api/jobs/{job_id}/destination_params - Return destination parameters for specified job. - - :type job_id: string - :param job_id: Encoded job id - - :rtype: list - :returns: list containing job destination parameters - """ - job = self.__get_job(trans, **kwd) - return summarize_destination_params(trans, job) - @expose_api_anonymous def parameters_display(self, trans: ProvidesUserContext, **kwd): """ From 9e48fa035c89e4be139950b645c3959f8dea58dc Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 20:04:43 +0200 Subject: [PATCH 050/110] Add pydantic model for destination_params operation --- lib/galaxy/schema/jobs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 83d5edae97a5..de002634c4c1 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -151,3 +151,9 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): # TODO add description, check type and add proper default copied_from_job_id: Any = Field(default=None, title="Copied from Job-ID", description="?") output_collections: Any = Field(default={}, title="Output collections", description="?") + + +class JobDestinationParams(Model): + runner: Any = Field(default=Required, title="Runner", description="?", alias="Runner") + runner_job_id: Any = Field(default=Required, title="Runner Job ID", description="?", alias="Runner Job ID") + handler: Any = Field(default=Required, title="Handler", description="?", alias="Handler") From 479e673d3e038c16caca8bf23d5c7d3fa397307c Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 20:04:55 +0200 Subject: [PATCH 051/110] Add test for destination_params operation --- lib/galaxy_test/api/test_jobs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 7c91357ac524..d0fa0786545e 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -977,6 +977,16 @@ def test_get_inputs_and_outputs(self, history_id): # self._assert_status_code_is(empty_search_response, 200) # assert len(empty_search_response.json()) == 0 + @pytest.mark.require_new_history + def test_destination_params(self, history_id): + dataset_id = self.__history_with_ok_dataset(history_id) + inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) + search_response = self._create_and_search_job(history_id, inputs, tool_id="cat1") + job_id = search_response.json()[0]["id"] + destination_params_response = self._get(f"/api/jobs/{job_id}/destination_params", admin=True) + self._assert_status_code_is(destination_params_response, 200) + # TODO maybe extend test? + def _create_and_search_job(self, history_id, inputs, tool_id): # create a job search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) From 9806bd08b812b42b5ede34ed04aa564a986b4969 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 20:05:14 +0200 Subject: [PATCH 052/110] Remove mapping to the legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 2d7b53d7414e..457d8682ad09 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -931,13 +931,7 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio action="build_for_rerun", conditions=dict(method=["GET"]), ) - webapp.mapper.connect( - "destination_params", - "/api/jobs/{job_id}/destination_params", - controller="jobs", - action="destination_params", - conditions=dict(method=["GET"]), - ) + webapp.mapper.connect( "metrics", "/api/jobs/{job_id}/metrics", From a7c3dce1323ab57870a9ebb988c46a27ccfd80d0 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 24 Oct 2023 20:06:09 +0200 Subject: [PATCH 053/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 76 +++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 19df83b84b57..f0552119785d 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -903,15 +903,8 @@ export interface paths { post: operations["search_jobs_api_jobs_search_post"]; }; "/api/jobs/{id}": { - /** - * Show - * @description Return dictionary containing description of job data - * - * Parameters - * - id: ID of job to return - * - full: Return extra information ? - */ - get: operations["show_api_jobs__id__get"]; + /** Return dictionary containing description of job data. */ + get: operations["show_job_api_jobs__id__get"]; /** Cancels specified job */ delete: operations["cancel_job_api_jobs__id__delete"]; }; @@ -935,6 +928,10 @@ export interface paths { /** Resumes a paused job. */ put: operations["resume_paused_job_api_jobs__id__resume_put"]; }; + "/api/jobs/{job_id}/destination_params": { + /** Return destination parameters for specified job. */ + get: operations["destination_params_job_api_jobs__job_id__destination_params_get"]; + }; "/api/jobs/{job_id}/oidc-tokens": { /** * Get a fresh OIDC token @@ -6297,6 +6294,27 @@ export interface components { */ name: string; }; + /** + * JobDestinationParams + * @description Base model definition with common configuration used by all derived models. + */ + JobDestinationParams: { + /** + * Handler + * @description ? + */ + Handler: Record; + /** + * Runner + * @description ? + */ + Runner: Record; + /** + * Runner Job ID + * @description ? + */ + "Runner Job ID": Record; + }; /** * JobErrorSummary * @description Base model definition with common configuration used by all derived models. @@ -14958,16 +14976,10 @@ export interface operations { }; }; }; - show_api_jobs__id__get: { - /** - * Show - * @description Return dictionary containing description of job data - * - * Parameters - * - id: ID of job to return - * - full: Return extra information ? - */ + show_job_api_jobs__id__get: { + /** Return dictionary containing description of job data. */ parameters: { + /** @description Show extra information. */ query?: { full?: boolean; }; @@ -14975,6 +14987,7 @@ export interface operations { header?: { "run-as"?: string; }; + /** @description The ID of the job */ path: { id: string; }; @@ -15166,6 +15179,33 @@ export interface operations { }; }; }; + destination_params_job_api_jobs__job_id__destination_params_get: { + /** Return destination parameters for specified job. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + job_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobDestinationParams"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_token_api_jobs__job_id__oidc_tokens_get: { /** * Get a fresh OIDC token From f5690e9d62d094fbab359f52a1437ff84e3d6d42 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 10:00:27 +0100 Subject: [PATCH 054/110] Refactor part of the metrics operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 54 +++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 5162f8b2b104..140c715a1457 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -55,7 +55,10 @@ ReportJobErrorPayload, SearchJobsPayload, ) -from galaxy.schema.schema import JobIndexSortByEnum +from galaxy.schema.schema import ( + DatasetSourceType, + JobIndexSortByEnum, +) from galaxy.schema.types import OffsetNaiveDatetime from galaxy.web import ( expose_api, @@ -182,12 +185,17 @@ ) FullShowQueryParam: Optional[bool] = Query(title="Full show", description="Show extra information.") +HdaLddaQueryParam: DatasetSourceType = Query( + title="HDA or LDDA", description="Whether this dataset belongs to a history (HDA) or a library (LDDA)." +) + JobIdPathParam: DecodedDatabaseIdField = Path(title="Job ID", description="The ID of the job") +DatasetIdPathParam: DecodedDatabaseIdField = Path(title="Dataset ID", description="The ID of the dataset") ReportErrorBody = Body(default=Required, title="Report error", description="The values to report an Error") SearchJobBody = Body(default=Required, title="Search job", description="The values to search an Job") -# TODO The endpoint only stops/cancles a job, but is called delete settle for one name +# TODO The endpoint only stops/cancels a job, but is called delete settle for one name DeleteJobBody = Body(title="Delete/cancel job", description="The values to delete/cancel a job") @@ -345,6 +353,45 @@ def outputs( job = self.service.get_job(trans=trans, job_id=id) return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + # TODO add pydantic model for output + @router.get( + "/api/jobs/{id}/metrics", + name="get_metrics", + summary="Return job metrics for specified job.", + ) + def metrics_by_job( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam], + trans: ProvidesUserContext = DependsOnTrans, + ): + """ + :rtype: list + :returns: list containing job metrics + """ + job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) + return summarize_job_metrics(trans, job) + + # TODO get this running + # @router.get( + # "/api/datasets/{id}/metrics", + # name="get_metrics", + # summary="Return job metrics for specified job.", + # ) + # def metrics_by_dataset( + # self, + # id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], + # hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = "hda", + # trans: ProvidesUserContext = DependsOnTrans, + # ): + # """ + # # TODO add pydantic model for return value + # :rtype: list + # :returns: list containing job metrics + # """ + # job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) + # return summarize_job_metrics(trans, job) + @router.get( "/api/jobs/{job_id}/destination_params", name="destination_params_job", @@ -450,10 +497,9 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): @expose_api_anonymous def metrics(self, trans: ProvidesUserContext, **kwd): """ - * GET /api/jobs/{job_id}/metrics * GET /api/datasets/{dataset_id}/metrics Return job metrics for specified job. Job accessibility checks are slightly - different than dataset checks, so both methods are available. + different than dataset checks, so both methods are available. Job metrics is already migrated to FastAPI :type job_id: string :param job_id: Encoded job id From 813d846972e6787eec3eb6bd7b7c1f172c9a4f19 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 10:00:55 +0100 Subject: [PATCH 055/110] Remove mapping to the legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 457d8682ad09..1e6ecab77279 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -932,13 +932,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio conditions=dict(method=["GET"]), ) - webapp.mapper.connect( - "metrics", - "/api/jobs/{job_id}/metrics", - controller="jobs", - action="metrics", - conditions=dict(method=["GET"]), - ) webapp.mapper.connect( "dataset_metrics", "/api/datasets/{dataset_id}/metrics", From 2fb687510c362f2074f45305cbdc8810177e8ae2 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 10:01:07 +0100 Subject: [PATCH 056/110] Add test for part of the metrics operation --- lib/galaxy_test/api/test_jobs.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index d0fa0786545e..72ed13b865d8 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -987,6 +987,19 @@ def test_destination_params(self, history_id): self._assert_status_code_is(destination_params_response, 200) # TODO maybe extend test? + @pytest.mark.require_new_history + def test_job_metrics(self, history_id): + dataset_id = self.__history_with_ok_dataset(history_id) + inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) + search_response = self._create_and_search_job(history_id, inputs, tool_id="cat1") + job_id = search_response.json()[0]["id"] + metrics_by_job_response = self._get(f"/api/jobs/{job_id}/metrics", data={"hda_ldda": "hda"}) + self._assert_status_code_is(metrics_by_job_response, 200) + # TODO enable this once metrics_by_dataset_works + metrics_by_dataset_response = self._get(f"/api/datasets/{dataset_id}/metrics", data={"hda_ldda": "hda"}) + self._assert_status_code_is(metrics_by_dataset_response, 200) + # TODO maybe extend test? + def _create_and_search_job(self, history_id, inputs, tool_id): # create a job search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) From 14154a6145c4bc4efe27c2f6177450db45b263b8 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 10:02:05 +0100 Subject: [PATCH 057/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index f0552119785d..85c7c2f3c851 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -920,6 +920,14 @@ export interface paths { /** Returns input datasets created by a job. */ get: operations["get_inputs_api_jobs__id__inputs_get"]; }; + "/api/jobs/{id}/metrics": { + /** + * Return job metrics for specified job. + * @description :rtype: list + * :returns: list containing job metrics + */ + get: operations["get_metrics_api_jobs__id__metrics_get"]; + }; "/api/jobs/{id}/outputs": { /** Returns output datasets created by a job. */ get: operations["get_outputs_api_jobs__id__outputs_get"]; @@ -15125,6 +15133,41 @@ export interface operations { }; }; }; + get_metrics_api_jobs__id__metrics_get: { + /** + * Return job metrics for specified job. + * @description :rtype: list + * :returns: list containing job metrics + */ + parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query: { + hda_ldda: components["schemas"]["DatasetSourceType"]; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_outputs_api_jobs__id__outputs_get: { /** Returns output datasets created by a job. */ parameters: { From 24b7a68f866242c933c3e92135e2c9dcca167f80 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 11:57:58 +0100 Subject: [PATCH 058/110] Refactor part of the parameters_display operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 59 ++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 140c715a1457..9a104283cba8 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -353,6 +353,64 @@ def outputs( job = self.service.get_job(trans=trans, job_id=id) return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + # TODO add pydantic model for output + @router.get( + "/api/jobs/{id}/parameters_display", + name="resolve_parameters_display", + summary="Resolve parameters as a list for nested display.", + ) + def parameters_display_by_job( + self, + id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam], + trans: ProvidesUserContext = DependsOnTrans, + ): + """ + # TODO is this still true? + Resolve parameters as a list for nested display. More client logic + here than is ideal but it is hard to reason about tool parameter + types on the client relative to the server. Job accessibility checks + are slightly different than dataset checks, so both methods are + available. + + This API endpoint is unstable and tied heavily to Galaxy's JS client code, + this endpoint will change frequently. + + :rtype: list + :returns: job parameters for for display + """ + job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) + return summarize_job_parameters(trans, job) + + # TODO add pydantic model for output and get this running + # @router.get( + # "/api/datasets/{id}/parameters_display", + # name="resolve_parameters_display", + # summary="Resolve parameters as a list for nested display.", + # ) + # def parameters_display_by_dataset( + # self, + # id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], + # hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam], + # trans: ProvidesUserContext = DependsOnTrans, + # ): + # """ + # # TODO is this still true? + # Resolve parameters as a list for nested display. More client logic + # here than is ideal but it is hard to reason about tool parameter + # types on the client relative to the server. Job accessibility checks + # are slightly different than dataset checks, so both methods are + # available. + + # This API endpoint is unstable and tied heavily to Galaxy's JS client code, + # this endpoint will change frequently. + + # :rtype: list + # :returns: job parameters for for display + # """ + # job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) + # return summarize_job_parameters(trans, job) + # TODO add pydantic model for output @router.get( "/api/jobs/{id}/metrics", @@ -521,7 +579,6 @@ def metrics(self, trans: ProvidesUserContext, **kwd): def parameters_display(self, trans: ProvidesUserContext, **kwd): """ * GET /api/jobs/{job_id}/parameters_display - * GET /api/datasets/{dataset_id}/parameters_display Resolve parameters as a list for nested display. More client logic here than is ideal but it is hard to reason about tool parameter From c4d9cb72d933a941c19b7320a119126b9153af03 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 11:58:16 +0100 Subject: [PATCH 059/110] Remove mapping to legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 1e6ecab77279..ab2b7abe1fa8 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -939,13 +939,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio action="metrics", conditions=dict(method=["GET"]), ) - webapp.mapper.connect( - "parameters_display", - "/api/jobs/{job_id}/parameters_display", - controller="jobs", - action="parameters_display", - conditions=dict(method=["GET"]), - ) webapp.mapper.connect( "dataset_parameters_display", "/api/datasets/{dataset_id}/parameters_display", From dd7aa025b431284d90e8c3f32e39cfe4efd5849b Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 11:58:46 +0100 Subject: [PATCH 060/110] Add tests for part of parameters_display operation --- lib/galaxy_test/api/test_jobs.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 72ed13b865d8..b2982e65aa82 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -995,11 +995,28 @@ def test_job_metrics(self, history_id): job_id = search_response.json()[0]["id"] metrics_by_job_response = self._get(f"/api/jobs/{job_id}/metrics", data={"hda_ldda": "hda"}) self._assert_status_code_is(metrics_by_job_response, 200) - # TODO enable this once metrics_by_dataset_works + # TODO enable this once metrics_by_dataset works metrics_by_dataset_response = self._get(f"/api/datasets/{dataset_id}/metrics", data={"hda_ldda": "hda"}) self._assert_status_code_is(metrics_by_dataset_response, 200) # TODO maybe extend test? + @pytest.mark.require_new_history + def test_parameters_display(self, history_id): + dataset_id = self.__history_with_ok_dataset(history_id) + inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) + search_response = self._create_and_search_job(history_id, inputs, tool_id="cat1") + job_id = search_response.json()[0]["id"] + display_parameters_by_job_response = self._get( + f"/api/jobs/{job_id}/parameters_display", data={"hda_ldda": "hda"} + ) + self._assert_status_code_is(display_parameters_by_job_response, 200) + # TODO enable this once parameters_display_by_dataset works + display_parameters_by_dataset_response = self._get( + f"/api/datasets/{dataset_id}/parameters_display", data={"hda_ldda": "hda"} + ) + self._assert_status_code_is(display_parameters_by_dataset_response, 200) + # TODO maybe extend test + def _create_and_search_job(self, history_id, inputs, tool_id): # create a job search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) From e14f66c6da541604d7042475777e2a8132b39291 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 11:59:33 +0100 Subject: [PATCH 061/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 85c7c2f3c851..124de9f9f0ba 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -932,6 +932,24 @@ export interface paths { /** Returns output datasets created by a job. */ get: operations["get_outputs_api_jobs__id__outputs_get"]; }; + "/api/jobs/{id}/parameters_display": { + /** + * Resolve parameters as a list for nested display. + * @description # TODO is this still true? + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + * + * :rtype: list + * :returns: job parameters for for display + */ + get: operations["resolve_parameters_display_api_jobs__id__parameters_display_get"]; + }; "/api/jobs/{id}/resume": { /** Resumes a paused job. */ put: operations["resume_paused_job_api_jobs__id__resume_put"]; @@ -15195,6 +15213,51 @@ export interface operations { }; }; }; + resolve_parameters_display_api_jobs__id__parameters_display_get: { + /** + * Resolve parameters as a list for nested display. + * @description # TODO is this still true? + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + * + * :rtype: list + * :returns: job parameters for for display + */ + parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query: { + hda_ldda: components["schemas"]["DatasetSourceType"]; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the job */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; resume_paused_job_api_jobs__id__resume_put: { /** Resumes a paused job. */ parameters: { From 324db18f005fb9b7d77d19cabd676e6371e36539 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:28:24 +0100 Subject: [PATCH 062/110] Refactor second part of the metrics operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 70 +++++++++------------------ 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 9a104283cba8..97a4a6e244b9 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -174,7 +174,7 @@ query_tags = [ IndexQueryTag("user", "The user email of the user that executed the Job.", "u"), IndexQueryTag("tool_id", "The tool ID corresponding to the job.", "t"), - IndexQueryTag("runner", "The job runner name used to execte the job.", "r", admin_only=True), + IndexQueryTag("runner", "The job runner name used to execute the job.", "r", admin_only=True), IndexQueryTag("handler", "The job handler name used to execute the job.", "h", admin_only=True), ] @@ -353,7 +353,7 @@ def outputs( job = self.service.get_job(trans=trans, job_id=id) return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) - # TODO add pydantic model for output + # TODO add pydantic model for return value @router.get( "/api/jobs/{id}/parameters_display", name="resolve_parameters_display", @@ -362,7 +362,7 @@ def outputs( def parameters_display_by_job( self, id: Annotated[DecodedDatabaseIdField, JobIdPathParam], - hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam], + hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ): """ @@ -391,7 +391,7 @@ def parameters_display_by_job( # def parameters_display_by_dataset( # self, # id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], - # hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam], + # hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, # trans: ProvidesUserContext = DependsOnTrans, # ): # """ @@ -420,7 +420,7 @@ def parameters_display_by_job( def metrics_by_job( self, id: Annotated[DecodedDatabaseIdField, JobIdPathParam], - hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam], + hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ): """ @@ -430,25 +430,24 @@ def metrics_by_job( job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) return summarize_job_metrics(trans, job) - # TODO get this running - # @router.get( - # "/api/datasets/{id}/metrics", - # name="get_metrics", - # summary="Return job metrics for specified job.", - # ) - # def metrics_by_dataset( - # self, - # id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], - # hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = "hda", - # trans: ProvidesUserContext = DependsOnTrans, - # ): - # """ - # # TODO add pydantic model for return value - # :rtype: list - # :returns: list containing job metrics - # """ - # job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) - # return summarize_job_metrics(trans, job) + # TODO add pydantic model for return value + @router.get( + "/api/datasets/{id}/metrics", + name="get_metrics", + summary="Return job metrics for specified job.", + ) + def metrics_by_dataset( + self, + id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], + hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, + trans: ProvidesUserContext = DependsOnTrans, + ): + """ + :rtype: list + :returns: list containing job metrics + """ + job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) + return summarize_job_metrics(trans, job) @router.get( "/api/jobs/{job_id}/destination_params", @@ -552,29 +551,6 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): service = depends(JobsService) hda_manager = depends(hdas.HDAManager) - @expose_api_anonymous - def metrics(self, trans: ProvidesUserContext, **kwd): - """ - * GET /api/datasets/{dataset_id}/metrics - Return job metrics for specified job. Job accessibility checks are slightly - different than dataset checks, so both methods are available. Job metrics is already migrated to FastAPI - - :type job_id: string - :param job_id: Encoded job id - - :type dataset_id: string - :param dataset_id: Encoded HDA or LDDA id - - :type hda_ldda: string - :param hda_ldda: hda if dataset_id is an HDA id (default), ldda if - it is an ldda id. - - :rtype: list - :returns: list containing job metrics - """ - job = self.__get_job(trans, **kwd) - return summarize_job_metrics(trans, job) - @expose_api_anonymous def parameters_display(self, trans: ProvidesUserContext, **kwd): """ From 063e2211d6469352d06174cf2b3f1d67eab22b30 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:28:53 +0100 Subject: [PATCH 063/110] Remove mapping to the legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index ab2b7abe1fa8..fac94e57119f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -931,14 +931,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio action="build_for_rerun", conditions=dict(method=["GET"]), ) - - webapp.mapper.connect( - "dataset_metrics", - "/api/datasets/{dataset_id}/metrics", - controller="jobs", - action="metrics", - conditions=dict(method=["GET"]), - ) webapp.mapper.connect( "dataset_parameters_display", "/api/datasets/{dataset_id}/parameters_display", From 4688dc5e7203a6663759a6358e48d795a04e0ae5 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:29:31 +0100 Subject: [PATCH 064/110] Regenerate client schema --- client/src/api/schema/schema.ts | 53 +++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 124de9f9f0ba..d74f631d4fe2 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -196,6 +196,14 @@ export interface paths { /** Check if metadata file can be downloaded. */ head: operations["get_metadata_file_datasets_api_datasets__history_content_id__metadata_file_head"]; }; + "/api/datasets/{id}/metrics": { + /** + * Return job metrics for specified job. + * @description :rtype: list + * :returns: list containing job metrics + */ + get: operations["get_metrics_api_datasets__id__metrics_get"]; + }; "/api/datatypes": { /** * Lists all available data types @@ -11053,6 +11061,41 @@ export interface operations { }; }; }; + get_metrics_api_datasets__id__metrics_get: { + /** + * Return job metrics for specified job. + * @description :rtype: list + * :returns: list containing job metrics + */ + parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the dataset */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_datatypes_get: { /** * Lists all available data types @@ -14917,7 +14960,7 @@ export interface operations { * : The tool ID corresponding to the job. (The tag `t` can be used a short hand alias for this tag to filter on this attribute.) * * `runner` - * : The job runner name used to execte the job. (The tag `r` can be used a short hand alias for this tag to filter on this attribute.) This tag is only available for requests using admin keys and/or sessions. + * : The job runner name used to execute the job. (The tag `r` can be used a short hand alias for this tag to filter on this attribute.) This tag is only available for requests using admin keys and/or sessions. * * `handler` * : The job handler name used to execute the job. (The tag `h` can be used a short hand alias for this tag to filter on this attribute.) This tag is only available for requests using admin keys and/or sessions. @@ -15159,8 +15202,8 @@ export interface operations { */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ - query: { - hda_ldda: components["schemas"]["DatasetSourceType"]; + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15231,8 +15274,8 @@ export interface operations { */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ - query: { - hda_ldda: components["schemas"]["DatasetSourceType"]; + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { From efa74731e94c2bf3865bbcd4bc3cbf0437f4fdf5 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:29:56 +0100 Subject: [PATCH 065/110] Refactor get_job method --- lib/galaxy/webapps/galaxy/services/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index c182a2ae56cd..c0fea605da2e 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -118,7 +118,7 @@ def get_job( else: dataset_instance = self.hda_manager.ldda_manager.get(trans, id=dataset_id) # TODO Raise error if no ID passed? Never happens when called from Job API endpoints - return dataset_instance + return dataset_instance.creating_job def dictify_associations(self, trans, *association_lists) -> List[JobAssociation]: rval: List[JobAssociation] = [] From 3077a067319bceeb0a3887c8026a2c6188d135b4 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:34:58 +0100 Subject: [PATCH 066/110] Remove TODO from tests --- lib/galaxy_test/api/test_jobs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index b2982e65aa82..a4e58aec84f2 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -995,7 +995,6 @@ def test_job_metrics(self, history_id): job_id = search_response.json()[0]["id"] metrics_by_job_response = self._get(f"/api/jobs/{job_id}/metrics", data={"hda_ldda": "hda"}) self._assert_status_code_is(metrics_by_job_response, 200) - # TODO enable this once metrics_by_dataset works metrics_by_dataset_response = self._get(f"/api/datasets/{dataset_id}/metrics", data={"hda_ldda": "hda"}) self._assert_status_code_is(metrics_by_dataset_response, 200) # TODO maybe extend test? @@ -1010,7 +1009,6 @@ def test_parameters_display(self, history_id): f"/api/jobs/{job_id}/parameters_display", data={"hda_ldda": "hda"} ) self._assert_status_code_is(display_parameters_by_job_response, 200) - # TODO enable this once parameters_display_by_dataset works display_parameters_by_dataset_response = self._get( f"/api/datasets/{dataset_id}/parameters_display", data={"hda_ldda": "hda"} ) From 5b8a031f98c33e9a77b0181f14f7959529f50431 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:35:44 +0100 Subject: [PATCH 067/110] Refactor second part of parameters_display operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 86 +++++++++------------------ 1 file changed, 28 insertions(+), 58 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 97a4a6e244b9..bfee481e1c05 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -382,34 +382,34 @@ def parameters_display_by_job( job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) - # TODO add pydantic model for output and get this running - # @router.get( - # "/api/datasets/{id}/parameters_display", - # name="resolve_parameters_display", - # summary="Resolve parameters as a list for nested display.", - # ) - # def parameters_display_by_dataset( - # self, - # id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], - # hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, - # trans: ProvidesUserContext = DependsOnTrans, - # ): - # """ - # # TODO is this still true? - # Resolve parameters as a list for nested display. More client logic - # here than is ideal but it is hard to reason about tool parameter - # types on the client relative to the server. Job accessibility checks - # are slightly different than dataset checks, so both methods are - # available. - - # This API endpoint is unstable and tied heavily to Galaxy's JS client code, - # this endpoint will change frequently. - - # :rtype: list - # :returns: job parameters for for display - # """ - # job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) - # return summarize_job_parameters(trans, job) + # TODO add pydantic model for return + @router.get( + "/api/datasets/{id}/parameters_display", + name="resolve_parameters_display", + summary="Resolve parameters as a list for nested display.", + ) + def parameters_display_by_dataset( + self, + id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], + hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, + trans: ProvidesUserContext = DependsOnTrans, + ): + """ + # TODO is this still true? + Resolve parameters as a list for nested display. More client logic + here than is ideal but it is hard to reason about tool parameter + types on the client relative to the server. Job accessibility checks + are slightly different than dataset checks, so both methods are + available. + + This API endpoint is unstable and tied heavily to Galaxy's JS client code, + this endpoint will change frequently. + + :rtype: list + :returns: job parameters for for display + """ + job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) + return summarize_job_parameters(trans, job) # TODO add pydantic model for output @router.get( @@ -551,36 +551,6 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): service = depends(JobsService) hda_manager = depends(hdas.HDAManager) - @expose_api_anonymous - def parameters_display(self, trans: ProvidesUserContext, **kwd): - """ - * GET /api/jobs/{job_id}/parameters_display - - Resolve parameters as a list for nested display. More client logic - here than is ideal but it is hard to reason about tool parameter - types on the client relative to the server. Job accessibility checks - are slightly different than dataset checks, so both methods are - available. - - This API endpoint is unstable and tied heavily to Galaxy's JS client code, - this endpoint will change frequently. - - :type job_id: string - :param job_id: Encoded job id - - :type dataset_id: string - :param dataset_id: Encoded HDA or LDDA id - - :type hda_ldda: string - :param hda_ldda: hda if dataset_id is an HDA id (default), ldda if - it is an ldda id. - - :rtype: list - :returns: job parameters for for display - """ - job = self.__get_job(trans, **kwd) - return summarize_job_parameters(trans, job) - @expose_api_anonymous def build_for_rerun(self, trans: ProvidesHistoryContext, id, **kwd): """ From 5ad4141c1526c53a4708efe28713b3dda9096408 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:35:57 +0100 Subject: [PATCH 068/110] Remove mapping to legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index fac94e57119f..ce72aed4c70f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -931,13 +931,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio action="build_for_rerun", conditions=dict(method=["GET"]), ) - webapp.mapper.connect( - "dataset_parameters_display", - "/api/datasets/{dataset_id}/parameters_display", - controller="jobs", - action="parameters_display", - conditions=dict(method=["GET"]), - ) # Job files controllers. Only for consumption by remote job runners. webapp.mapper.resource( From f44798889d246b50450c42009bdca5f5841e5e3f Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 12:36:40 +0100 Subject: [PATCH 069/110] Regenerate client schema --- client/src/api/schema/schema.ts | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index d74f631d4fe2..753651349221 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -204,6 +204,24 @@ export interface paths { */ get: operations["get_metrics_api_datasets__id__metrics_get"]; }; + "/api/datasets/{id}/parameters_display": { + /** + * Resolve parameters as a list for nested display. + * @description # TODO is this still true? + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + * + * :rtype: list + * :returns: job parameters for for display + */ + get: operations["resolve_parameters_display_api_datasets__id__parameters_display_get"]; + }; "/api/datatypes": { /** * Lists all available data types @@ -11096,6 +11114,51 @@ export interface operations { }; }; }; + resolve_parameters_display_api_datasets__id__parameters_display_get: { + /** + * Resolve parameters as a list for nested display. + * @description # TODO is this still true? + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + * + * :rtype: list + * :returns: job parameters for for display + */ + parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the dataset */ + path: { + id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_datatypes_get: { /** * Lists all available data types From a4ba468f37977c476fdd5952fdefe667ae8d3a61 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 14:36:05 +0100 Subject: [PATCH 070/110] Remove unused methods --- lib/galaxy/webapps/galaxy/api/jobs.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index bfee481e1c05..2e71e2ca62fb 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -576,22 +576,6 @@ def build_for_rerun(self, trans: ProvidesHistoryContext, id, **kwd): raise exceptions.ConfigDoesNotAllowException(f"Tool '{job.tool_id}' cannot be rerun.") return tool.to_json(trans, {}, job=job) - def __dictify_associations(self, trans, *association_lists) -> List[dict]: - rval: List[dict] = [] - for association_list in association_lists: - rval.extend(self.__dictify_association(trans, a) for a in association_list) - return rval - - def __dictify_association(self, trans, job_dataset_association) -> dict: - dataset_dict = None - dataset = job_dataset_association.dataset - if dataset: - if isinstance(dataset, model.HistoryDatasetAssociation): - dataset_dict = dict(src="hda", id=trans.security.encode_id(dataset.id)) - else: - dataset_dict = dict(src="ldda", id=trans.security.encode_id(dataset.id)) - return dict(name=job_dataset_association.name, dataset=dataset_dict) - def __get_job(self, trans, job_id=None, dataset_id=None, **kwd): if job_id is not None: decoded_job_id = self.decode_id(job_id) From 08010df14568955d3012db62cfc88bf589ff6d78 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 14:39:36 +0100 Subject: [PATCH 071/110] Remove unused dependencies of JobController --- lib/galaxy/webapps/galaxy/api/jobs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 2e71e2ca62fb..cea2f5fbbf8b 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -547,9 +547,6 @@ def delete( class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): job_manager = depends(JobManager) - job_search = depends(JobSearch) - service = depends(JobsService) - hda_manager = depends(hdas.HDAManager) @expose_api_anonymous def build_for_rerun(self, trans: ProvidesHistoryContext, id, **kwd): From 0258a742729a47c973b22fa7fe4734b22689a7b8 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 15:12:02 +0100 Subject: [PATCH 072/110] Add TODO --- lib/galaxy/schema/jobs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index de002634c4c1..a5bbb7d2b5a0 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -154,6 +154,7 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): class JobDestinationParams(Model): + # TODO add description, check type and add proper default runner: Any = Field(default=Required, title="Runner", description="?", alias="Runner") runner_job_id: Any = Field(default=Required, title="Runner Job ID", description="?", alias="Runner Job ID") handler: Any = Field(default=Required, title="Handler", description="?", alias="Handler") From 053ef2b192a2a9fb4756f581f0e9cdcd4a1fdb50 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 15:14:58 +0100 Subject: [PATCH 073/110] Add pydantic model in metrics operations return --- lib/galaxy/webapps/galaxy/api/jobs.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index cea2f5fbbf8b..408387b6fd45 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -27,11 +27,7 @@ from pydantic import Required from typing_extensions import Annotated -from galaxy import ( - exceptions, - model, -) -from galaxy.managers import hdas +from galaxy import exceptions from galaxy.managers.context import ( ProvidesHistoryContext, ProvidesUserContext, @@ -58,6 +54,7 @@ from galaxy.schema.schema import ( DatasetSourceType, JobIndexSortByEnum, + JobMetric, ) from galaxy.schema.types import OffsetNaiveDatetime from galaxy.web import ( @@ -422,13 +419,13 @@ def metrics_by_job( id: Annotated[DecodedDatabaseIdField, JobIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, - ): + ) -> List[Optional[JobMetric]]: """ :rtype: list :returns: list containing job metrics """ job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) - return summarize_job_metrics(trans, job) + return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] # TODO add pydantic model for return value @router.get( @@ -441,13 +438,13 @@ def metrics_by_dataset( id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, - ): + ) -> List[Optional[JobMetric]]: """ :rtype: list :returns: list containing job metrics """ job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) - return summarize_job_metrics(trans, job) + return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] @router.get( "/api/jobs/{job_id}/destination_params", From 53055d66b8244757cd387bf8ab5df63f56ca52dc Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 15:18:38 +0100 Subject: [PATCH 074/110] Remove outdated TODOs --- lib/galaxy/webapps/galaxy/api/jobs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 408387b6fd45..a72ab6fc3cbd 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -408,7 +408,6 @@ def parameters_display_by_dataset( job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) - # TODO add pydantic model for output @router.get( "/api/jobs/{id}/metrics", name="get_metrics", @@ -427,7 +426,6 @@ def metrics_by_job( job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] - # TODO add pydantic model for return value @router.get( "/api/datasets/{id}/metrics", name="get_metrics", From 1d38b374fb966dbd9f6e247a20d0770a1c53b9f8 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 30 Oct 2023 15:19:31 +0100 Subject: [PATCH 075/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 42 +++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 753651349221..797554bd8171 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -6544,6 +6544,44 @@ export interface components { */ active: boolean; }; + /** + * JobMetric + * @description Base model definition with common configuration used by all derived models. + * @example { + * "name": "start_epoch", + * "plugin": "core", + * "raw_value": "1614261340.0000000", + * "title": "Job Start Time", + * "value": "2021-02-25 14:55:40" + * } + */ + JobMetric: { + /** + * Name + * @description The name of the metric variable. + */ + name: string; + /** + * Plugin + * @description The instrumenter plugin that generated this metric. + */ + plugin: string; + /** + * Raw Value + * @description The raw value of the metric as a string. + */ + raw_value: string; + /** + * Title + * @description A descriptive title for this metric. + */ + title: string; + /** + * Value + * @description The textual representation of the metric value. + */ + value: string; + }; /** * JobSourceType * @description Available types of job sources (model classes) that produce dataset collections. @@ -11103,7 +11141,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["JobMetric"][]; }; }; /** @description Validation Error */ @@ -15281,7 +15319,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["JobMetric"][]; }; }; /** @description Validation Error */ From baf5cdf3916338b5322d18ecb15ab68ec777e32c Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 15:50:34 +0100 Subject: [PATCH 076/110] Access managers via service, remove uneccessary comments/TODOs and add pydantic model to parameters_display_by_job_id return --- lib/galaxy/webapps/galaxy/api/jobs.py | 44 +++++++++------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index a72ab6fc3cbd..47d87ce1e9d6 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -32,10 +32,8 @@ ProvidesHistoryContext, ProvidesUserContext, ) -from galaxy.managers.hdas import HDAManager from galaxy.managers.jobs import ( JobManager, - JobSearch, summarize_destination_params, summarize_job_metrics, summarize_job_parameters, @@ -46,6 +44,7 @@ EncodedJobDetails, JobAssociation, JobDestinationParams, + JobDisplayParametersSummary, JobErrorSummary, JobInputSummary, ReportJobErrorPayload, @@ -199,9 +198,6 @@ @router.cbv class FastAPIJobs: service: JobsService = depends(JobsService) - manager: JobManager = depends(JobManager) - hda_manager: HDAManager = depends(HDAManager) - job_search: JobSearch = depends(JobSearch) @router.get("/api/jobs") def index( @@ -304,7 +300,7 @@ def error( ) -> JobErrorSummary: # Get dataset on which this error was triggered dataset_id = payload.dataset_id - dataset = self.hda_manager.get_accessible(id=dataset_id, user=trans.user) + dataset = self.service.hda_manager.get_accessible(id=dataset_id, user=trans.user) # Get job job = self.service.get_job(trans, id) if dataset.creating_job.id != job.id: @@ -350,7 +346,6 @@ def outputs( job = self.service.get_job(trans=trans, job_id=id) return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) - # TODO add pydantic model for return value @router.get( "/api/jobs/{id}/parameters_display", name="resolve_parameters_display", @@ -361,20 +356,17 @@ def parameters_display_by_job( id: Annotated[DecodedDatabaseIdField, JobIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, - ): + ) -> JobDisplayParametersSummary: """ - # TODO is this still true? - Resolve parameters as a list for nested display. More client logic - here than is ideal but it is hard to reason about tool parameter - types on the client relative to the server. Job accessibility checks - are slightly different than dataset checks, so both methods are - available. - - This API endpoint is unstable and tied heavily to Galaxy's JS client code, - this endpoint will change frequently. - - :rtype: list - :returns: job parameters for for display + # TODO is this still true? + Resolve parameters as a list for nested display. More client logic + here than is ideal but it is hard to reason about tool parameter + types on the client relative to the server. Job accessibility checks + are slightly different than dataset checks, so both methods are + available. + + This API endpoint is unstable and tied heavily to Galaxy's JS client code, + this endpoint will change frequently. """ job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) @@ -419,10 +411,6 @@ def metrics_by_job( hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> List[Optional[JobMetric]]: - """ - :rtype: list - :returns: list containing job metrics - """ job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] @@ -437,10 +425,6 @@ def metrics_by_dataset( hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> List[Optional[JobMetric]]: - """ - :rtype: list - :returns: list containing job metrics - """ job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] @@ -494,7 +478,7 @@ def search( params_dump = [tool.params_to_strings(param, trans.app, nested=True) for param in all_params] jobs = [] for param_dump, param in zip(params_dump, all_params): - job = self.job_search.by_tool_input( + job = self.service.job_search.by_tool_input( trans=trans, tool_id=tool_id, tool_version=tool.version, @@ -537,7 +521,7 @@ def delete( message = payload.message else: message = None - return self.manager.stop(job, message=message) + return self.service.job_manager.stop(job, message=message) class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): From 47aa990615d11134a9611befd8c50911257881b8 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 15:51:09 +0100 Subject: [PATCH 077/110] Add pydantic model for parameters_display_by_job_id operation --- lib/galaxy/schema/jobs.py | 87 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index a5bbb7d2b5a0..c595e9b3df17 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -17,6 +17,7 @@ EncodedDatabaseIdField, ) from galaxy.schema.schema import ( + DatasetSourceType, EncodedDatasetSourceId, EntityIdField, JobSummary, @@ -149,12 +150,90 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): description="Dictionary mapping all the tool outputs (by name) with the corresponding dataset information.", ) # TODO add description, check type and add proper default - copied_from_job_id: Any = Field(default=None, title="Copied from Job-ID", description="?") + copied_from_job_id: Optional[EncodedDatabaseIdField] = Field( + default=None, title="Copied from Job-ID", description="?" + ) output_collections: Any = Field(default={}, title="Output collections", description="?") class JobDestinationParams(Model): # TODO add description, check type and add proper default - runner: Any = Field(default=Required, title="Runner", description="?", alias="Runner") - runner_job_id: Any = Field(default=Required, title="Runner Job ID", description="?", alias="Runner Job ID") - handler: Any = Field(default=Required, title="Handler", description="?", alias="Handler") + runner: str = Field(default=Required, title="Runner", description="?", alias="Runner") + runner_job_id: str = Field(default=Required, title="Runner Job ID", description="?", alias="Runner Job ID") + handler: str = Field(default=Required, title="Handler", description="?", alias="Handler") + + +class JobOutput(Model): + label: str = Field(default=Required, title="Output label", description="The output label") + value: EncodedDatasetSourceId = Field(default=Required, title="dataset", description="The associated dataset.") + + +class JobParameterValues(Model): + src: DatasetSourceType = Field( + default=Required, + title="Source", + description="The source of this dataset, either `hda` or `ldda` depending of its origin.", + ) + id: EncodedDatabaseIdField = EntityIdField + name: str = Field( + default=Required, + title="Name", + description="The name of the item.", + ) + hid: int = Field( + default=Required, + title="HID", + description="The index position of this item in the History.", + ) + + +class JobParameter(Model): + text: str = Field( + default=Required, + title="Text", + description="Text associated with the job parameter.", + ) + depth: int = Field( + default=Required, + title="Depth", + description="The depth of the job parameter.", + ) + value: List[JobParameterValues] + + +class JobDisplayParametersSummary(Model): + parameters: List[JobParameter] + has_parameter_errors: bool = Field( + default=Required, title="Has parameter errors", description="The job has parameter errors" + ) + outputs: Dict[str, List[JobOutput]] = Field( + default=Required, + title="Outputs", + description="Dictionary mapping all the tool outputs (by name) with the corresponding dataset information in a nested format.", + ) + + +# class ParameterValueModel(Model): +# src: str +# id: str +# hid: int +# name: str + + +# class ParametersModel(Model): +# text: str +# depth: int +# value: List[Optional[ParameterValueModel]] + + +# class OutputsModel(Model): +# label: Optional[str] +# value: EncodedDatasetSourceId + + +# class MyPydanticModel(Model): +# parameters: List[ParametersModel] +# has_parameter_errors: bool +# outputs: dict[str, List[OutputsModel]] + +# = Field(default="",title="",description="") From b85994a612cd0347015201fc59c8b943d0f16baf Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 15:51:29 +0100 Subject: [PATCH 078/110] Regenerate client schema --- client/src/api/schema/schema.ts | 150 ++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 45 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 797554bd8171..d94ca445ae2b 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -197,11 +197,7 @@ export interface paths { head: operations["get_metadata_file_datasets_api_datasets__history_content_id__metadata_file_head"]; }; "/api/datasets/{id}/metrics": { - /** - * Return job metrics for specified job. - * @description :rtype: list - * :returns: list containing job metrics - */ + /** Return job metrics for specified job. */ get: operations["get_metrics_api_datasets__id__metrics_get"]; }; "/api/datasets/{id}/parameters_display": { @@ -947,11 +943,7 @@ export interface paths { get: operations["get_inputs_api_jobs__id__inputs_get"]; }; "/api/jobs/{id}/metrics": { - /** - * Return job metrics for specified job. - * @description :rtype: list - * :returns: list containing job metrics - */ + /** Return job metrics for specified job. */ get: operations["get_metrics_api_jobs__id__metrics_get"]; }; "/api/jobs/{id}/outputs": { @@ -962,17 +954,14 @@ export interface paths { /** * Resolve parameters as a list for nested display. * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. * - * :rtype: list - * :returns: job parameters for for display + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. */ get: operations["resolve_parameters_display_api_jobs__id__parameters_display_get"]; }; @@ -4239,8 +4228,9 @@ export interface components { /** * Copied from Job-ID * @description ? + * @example 0123456789ABCDEF */ - copied_from_job_id?: Record; + copied_from_job_id?: string; /** * Create Time * Format: date-time @@ -6355,17 +6345,37 @@ export interface components { * Handler * @description ? */ - Handler: Record; + Handler: string; /** * Runner * @description ? */ - Runner: Record; + Runner: string; /** * Runner Job ID * @description ? */ - "Runner Job ID": Record; + "Runner Job ID": string; + }; + /** + * JobDisplayParametersSummary + * @description Base model definition with common configuration used by all derived models. + */ + JobDisplayParametersSummary: { + /** + * Has parameter errors + * @description The job has parameter errors + */ + has_parameter_errors: boolean; + /** + * Outputs + * @description Dictionary mapping all the tool outputs (by name) with the corresponding dataset information in a nested format. + */ + outputs: { + [key: string]: components["schemas"]["JobOutput"][] | undefined; + }; + /** Parameters */ + parameters: components["schemas"]["JobParameter"][]; }; /** * JobErrorSummary @@ -6582,6 +6592,67 @@ export interface components { */ value: string; }; + /** + * JobOutput + * @description Base model definition with common configuration used by all derived models. + */ + JobOutput: { + /** + * Output label + * @description The output label + */ + label: string; + /** + * dataset + * @description The associated dataset. + */ + value: components["schemas"]["EncodedDatasetSourceId"]; + }; + /** + * JobParameter + * @description Base model definition with common configuration used by all derived models. + */ + JobParameter: { + /** + * Depth + * @description The depth of the job parameter. + */ + depth: number; + /** + * Text + * @description Text associated with the job parameter. + */ + text: string; + /** Value */ + value: components["schemas"]["JobParameterValues"][]; + }; + /** + * JobParameterValues + * @description Base model definition with common configuration used by all derived models. + */ + JobParameterValues: { + /** + * HID + * @description The index position of this item in the History. + */ + hid: number; + /** + * ID + * @description The encoded ID of this entity. + * @example 0123456789ABCDEF + */ + id: string; + /** + * Name + * @description The name of the item. + */ + name: string; + /** + * Source + * @description The source of this dataset, either `hda` or `ldda` depending of its origin. + */ + src: components["schemas"]["DatasetSourceType"]; + }; /** * JobSourceType * @description Available types of job sources (model classes) that produce dataset collections. @@ -11118,11 +11189,7 @@ export interface operations { }; }; get_metrics_api_datasets__id__metrics_get: { - /** - * Return job metrics for specified job. - * @description :rtype: list - * :returns: list containing job metrics - */ + /** Return job metrics for specified job. */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ query?: { @@ -15296,11 +15363,7 @@ export interface operations { }; }; get_metrics_api_jobs__id__metrics_get: { - /** - * Return job metrics for specified job. - * @description :rtype: list - * :returns: list containing job metrics - */ + /** Return job metrics for specified job. */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ query?: { @@ -15361,17 +15424,14 @@ export interface operations { /** * Resolve parameters as a list for nested display. * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. * - * :rtype: list - * :returns: job parameters for for display + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ @@ -15391,7 +15451,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["JobDisplayParametersSummary"]; }; }; /** @description Validation Error */ From 23c8f17512f4adce891590e3cba093de3d1a2200 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 18:10:58 +0100 Subject: [PATCH 079/110] Remove uneccessary comments and add pydantic model to prarameters_display_by_dataset return --- lib/galaxy/webapps/galaxy/api/jobs.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 47d87ce1e9d6..8a7b9c9420ed 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -371,7 +371,6 @@ def parameters_display_by_job( job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) - # TODO add pydantic model for return @router.get( "/api/datasets/{id}/parameters_display", name="resolve_parameters_display", @@ -382,20 +381,17 @@ def parameters_display_by_dataset( id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, - ): + ) -> JobDisplayParametersSummary: """ - # TODO is this still true? - Resolve parameters as a list for nested display. More client logic - here than is ideal but it is hard to reason about tool parameter - types on the client relative to the server. Job accessibility checks - are slightly different than dataset checks, so both methods are - available. - - This API endpoint is unstable and tied heavily to Galaxy's JS client code, - this endpoint will change frequently. - - :rtype: list - :returns: job parameters for for display + # TODO is this still true? + Resolve parameters as a list for nested display. More client logic + here than is ideal but it is hard to reason about tool parameter + types on the client relative to the server. Job accessibility checks + are slightly different than dataset checks, so both methods are + available. + + This API endpoint is unstable and tied heavily to Galaxy's JS client code, + this endpoint will change frequently. """ job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) From 0ddc883288d454cdfd494ca74c0a817c465ccf86 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 18:12:17 +0100 Subject: [PATCH 080/110] Extend JobDisplayParametersSummary to use it in parameters_display_by_dataset return --- lib/galaxy/schema/jobs.py | 89 ++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index c595e9b3df17..8db34fb94015 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -3,6 +3,7 @@ Dict, List, Optional, + Union, ) from pydantic import ( @@ -164,11 +165,11 @@ class JobDestinationParams(Model): class JobOutput(Model): - label: str = Field(default=Required, title="Output label", description="The output label") + label: Any = Field(default=Required, title="Output label", description="The output label") # check if this is true value: EncodedDatasetSourceId = Field(default=Required, title="dataset", description="The associated dataset.") -class JobParameterValues(Model): +class JobParameterValuesSimple(Model): src: DatasetSourceType = Field( default=Required, title="Source", @@ -187,6 +188,55 @@ class JobParameterValues(Model): ) +class JobTargetElement(Model): # TODO rename? + name: str = Field(default=Required, title="Name", description="Name of the element") # TODO check correctness + dbkey: str = Field(default=Required, title="Database Key", description="Database key") # TODO check correctness + ext: str = Field(default=Required, title="Extension", description="Extension") # TODO check correctness + to_posix_lines: bool = Field( + default=Required, title="To POSIX Lines", description="Flag indicating if to convert to POSIX lines" + ) # TODO check correctness + auto_decompress: bool = Field( + default=Required, title="Auto Decompress", description="Flag indicating if to auto decompress" + ) # TODO check correctness + src: str = Field(default=Required, title="Source", description="Source") # TODO check correctness + paste_content: str = Field( + default=Required, title="Paste Content", description="Content to paste" + ) # TODO check correctness + hashes: List[str] = Field(default=Required, title="Hashes", description="List of hashes") # TODO check correctness + purge_source: bool = Field( + default=Required, title="Purge Source", description="Flag indicating if to purge the source" + ) # TODO check correctness + object_id: EncodedDatabaseIdField = Field( + default=Required, title="Object ID", description="Object ID" + ) # TODO check correctness + + +class JobTargetDestination(Model): + type: str = Field( + default=Required, + title="Type", + description="Type of destination, either `hda` or `ldda` depending on the destination", + ) # TODO check correctness + + +class JobTarget(Model): # TODO rename? + destination: JobTargetDestination = Field( + default=Required, title="Destination", description="Destination details" + ) # TODO check correctness + elements: List[JobTargetElement] = Field( + default=Required, title="Elements", description="List of job target elements" + ) # TODO check correctness + + +class JobParameterValuesExtensive(Model): # TODO rename? + targets: List[JobTarget] = Field( + default=Required, title="Targets", description="List of job value targets" + ) # TODO check correctness + check_content: bool = Field( + default=Required, title="Check Content", description="Flag to check content" + ) # TODO check correctness + + class JobParameter(Model): text: str = Field( default=Required, @@ -198,11 +248,16 @@ class JobParameter(Model): title="Depth", description="The depth of the job parameter.", ) - value: List[JobParameterValues] + value: Union[List[JobParameterValuesSimple], JobParameterValuesExtensive, str] = Field( + default=Required, title="Value", description="The values of the job parameter" + ) + notes: Optional[str] = Field(default=None, title="notes", description="Notes associated with the job parameter.") class JobDisplayParametersSummary(Model): - parameters: List[JobParameter] + parameters: List[JobParameter] = Field( + default=Required, title="Parameters", description="The parameters of the job in a nested format." + ) has_parameter_errors: bool = Field( default=Required, title="Has parameter errors", description="The job has parameter errors" ) @@ -211,29 +266,3 @@ class JobDisplayParametersSummary(Model): title="Outputs", description="Dictionary mapping all the tool outputs (by name) with the corresponding dataset information in a nested format.", ) - - -# class ParameterValueModel(Model): -# src: str -# id: str -# hid: int -# name: str - - -# class ParametersModel(Model): -# text: str -# depth: int -# value: List[Optional[ParameterValueModel]] - - -# class OutputsModel(Model): -# label: Optional[str] -# value: EncodedDatasetSourceId - - -# class MyPydanticModel(Model): -# parameters: List[ParametersModel] -# has_parameter_errors: bool -# outputs: dict[str, List[OutputsModel]] - -# = Field(default="",title="",description="") From 604d360baefb34b63745ade2b8c08658df2125f7 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 18:12:35 +0100 Subject: [PATCH 081/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 128 ++++++++++++++++++++++++++++++-- 1 file changed, 121 insertions(+), 7 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index d94ca445ae2b..390d642e5dc8 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -6374,7 +6374,10 @@ export interface components { outputs: { [key: string]: components["schemas"]["JobOutput"][] | undefined; }; - /** Parameters */ + /** + * Parameters + * @description The parameters of the job in a nested format. + */ parameters: components["schemas"]["JobParameter"][]; }; /** @@ -6601,7 +6604,7 @@ export interface components { * Output label * @description The output label */ - label: string; + label: Record; /** * dataset * @description The associated dataset. @@ -6618,19 +6621,46 @@ export interface components { * @description The depth of the job parameter. */ depth: number; + /** + * notes + * @description Notes associated with the job parameter. + */ + notes?: string; /** * Text * @description Text associated with the job parameter. */ text: string; - /** Value */ - value: components["schemas"]["JobParameterValues"][]; + /** + * Value + * @description The values of the job parameter + */ + value: + | components["schemas"]["JobParameterValuesSimple"][] + | components["schemas"]["JobParameterValuesExtensive"] + | string; }; /** - * JobParameterValues + * JobParameterValuesExtensive * @description Base model definition with common configuration used by all derived models. */ - JobParameterValues: { + JobParameterValuesExtensive: { + /** + * Check Content + * @description Flag to check content + */ + check_content: boolean; + /** + * Targets + * @description List of job value targets + */ + targets: components["schemas"]["JobTarget"][]; + }; + /** + * JobParameterValuesSimple + * @description Base model definition with common configuration used by all derived models. + */ + JobParameterValuesSimple: { /** * HID * @description The index position of this item in the History. @@ -6712,6 +6742,90 @@ export interface components { [key: string]: number | undefined; }; }; + /** + * JobTarget + * @description Base model definition with common configuration used by all derived models. + */ + JobTarget: { + /** + * Destination + * @description Destination details + */ + destination: components["schemas"]["JobTargetDestination"]; + /** + * Elements + * @description List of job target elements + */ + elements: components["schemas"]["JobTargetElement"][]; + }; + /** + * JobTargetDestination + * @description Base model definition with common configuration used by all derived models. + */ + JobTargetDestination: { + /** + * Type + * @description Type of destination, either `hda` or `ldda` depending on the destination + */ + type: string; + }; + /** + * JobTargetElement + * @description Base model definition with common configuration used by all derived models. + */ + JobTargetElement: { + /** + * Auto Decompress + * @description Flag indicating if to auto decompress + */ + auto_decompress: boolean; + /** + * Database Key + * @description Database key + */ + dbkey: string; + /** + * Extension + * @description Extension + */ + ext: string; + /** + * Hashes + * @description List of hashes + */ + hashes: string[]; + /** + * Name + * @description Name of the element + */ + name: string; + /** + * Object ID + * @description Object ID + * @example 0123456789ABCDEF + */ + object_id: string; + /** + * Paste Content + * @description Content to paste + */ + paste_content: string; + /** + * Purge Source + * @description Flag indicating if to purge the source + */ + purge_source: boolean; + /** + * Source + * @description Source + */ + src: string; + /** + * To POSIX Lines + * @description Flag indicating if to convert to POSIX lines + */ + to_posix_lines: boolean; + }; /** * LabelValuePair * @description Generic Label/Value pair model. @@ -11253,7 +11367,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["JobDisplayParametersSummary"]; }; }; /** @description Validation Error */ From e1675815269693822f09c16f7c977c99e2e266f0 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 4 Nov 2023 18:17:00 +0100 Subject: [PATCH 082/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 34 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 390d642e5dc8..95a8845c2ffe 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -204,17 +204,14 @@ export interface paths { /** * Resolve parameters as a list for nested display. * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. * - * :rtype: list - * :returns: job parameters for for display + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. */ get: operations["resolve_parameters_display_api_datasets__id__parameters_display_get"]; }; @@ -11337,17 +11334,14 @@ export interface operations { /** * Resolve parameters as a list for nested display. * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. + * Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. * - * :rtype: list - * :returns: job parameters for for display + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ From fb0526da9f4b72efb8399dee37b0f98c8caa749a Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 10 Nov 2023 14:20:57 +0100 Subject: [PATCH 083/110] Apply some suggestions --- lib/galaxy/schema/jobs.py | 26 ++++++++++-------- lib/galaxy/webapps/galaxy/api/jobs.py | 38 +++++++++++++-------------- lib/galaxy_test/api/test_jobs.py | 4 +-- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 8db34fb94015..3e97c2d22604 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -54,26 +54,25 @@ class JobAssociation(Model): name: str = Field( default=Required, title="name", - description="The name of the associated dataset.", + description="Name of the job parameter.", ) dataset: EncodedDatasetSourceId = Field( default=Required, title="dataset", - description="The associated dataset.", + description="Reference to the associated item.", ) class ReportJobErrorPayload(Model): dataset_id: DecodedDatabaseIdField = Field( default=Required, - title="Dataset ID", - description="The dataset ID related to the error.", + title="History Dataset Association ID", + description="The History Dataset Association ID related to the error.", ) - # TODO add description email: Optional[str] = Field( default=None, title="Email", - description="", + description="Email address for communication with the user. Only required for anonymous users.", ) message: Optional[str] = Field( default=None, @@ -143,24 +142,29 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): inputs: Dict[str, EncodedDatasetJobInfo] = Field( {}, title="Inputs", - description="Dictionary mapping all the tool inputs (by name) with the corresponding dataset information.", + description="Dictionary mapping all the tool inputs (by name) to the corresponding data references.", ) outputs: Dict[str, EncodedDatasetJobInfo] = Field( {}, title="Outputs", - description="Dictionary mapping all the tool outputs (by name) with the corresponding dataset information.", + description="Dictionary mapping all the tool outputs (by name) to the corresponding data references.", ) # TODO add description, check type and add proper default copied_from_job_id: Optional[EncodedDatabaseIdField] = Field( - default=None, title="Copied from Job-ID", description="?" + default=None, title="Copied from Job-ID", description="Reference to cached job if job execution was cached." ) output_collections: Any = Field(default={}, title="Output collections", description="?") class JobDestinationParams(Model): # TODO add description, check type and add proper default - runner: str = Field(default=Required, title="Runner", description="?", alias="Runner") - runner_job_id: str = Field(default=Required, title="Runner Job ID", description="?", alias="Runner Job ID") + runner: str = Field(default=Required, title="Runner", description="Job runner class", alias="Runner") + runner_job_id: str = Field( + default=Required, + title="Runner Job ID", + description="ID assigned to submitted job by external job running system", + alias="Runner Job ID", + ) handler: str = Field(default=Required, title="Handler", description="?", alias="Handler") diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 8a7b9c9420ed..f2f4c4816dc0 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -347,18 +347,17 @@ def outputs( return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) @router.get( - "/api/jobs/{id}/parameters_display", + "/api/jobs/{job_id}/parameters_display", name="resolve_parameters_display", summary="Resolve parameters as a list for nested display.", ) def parameters_display_by_job( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> JobDisplayParametersSummary: """ - # TODO is this still true? Resolve parameters as a list for nested display. More client logic here than is ideal but it is hard to reason about tool parameter types on the client relative to the server. Job accessibility checks @@ -368,22 +367,21 @@ def parameters_display_by_job( This API endpoint is unstable and tied heavily to Galaxy's JS client code, this endpoint will change frequently. """ - job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) + job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) @router.get( - "/api/datasets/{id}/parameters_display", + "/api/datasets/{dataset_id}/parameters_display", name="resolve_parameters_display", summary="Resolve parameters as a list for nested display.", ) def parameters_display_by_dataset( self, - id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], + dataset_id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> JobDisplayParametersSummary: """ - # TODO is this still true? Resolve parameters as a list for nested display. More client logic here than is ideal but it is hard to reason about tool parameter types on the client relative to the server. Job accessibility checks @@ -393,35 +391,35 @@ def parameters_display_by_dataset( This API endpoint is unstable and tied heavily to Galaxy's JS client code, this endpoint will change frequently. """ - job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) + job = self.service.get_job(trans, dataset_id=dataset_id, hda_ldda=hda_ldda) return summarize_job_parameters(trans, job) @router.get( - "/api/jobs/{id}/metrics", + "/api/jobs/{job_id}/metrics", name="get_metrics", summary="Return job metrics for specified job.", ) def metrics_by_job( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> List[Optional[JobMetric]]: - job = self.service.get_job(trans, job_id=id, hda_ldda=hda_ldda) + job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda) return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] @router.get( - "/api/datasets/{id}/metrics", + "/api/datasets/{dataset_id}/metrics", name="get_metrics", summary="Return job metrics for specified job.", ) def metrics_by_dataset( self, - id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], + dataset_id: Annotated[DecodedDatabaseIdField, DatasetIdPathParam], hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> List[Optional[JobMetric]]: - job = self.service.get_job(trans, dataset_id=id, hda_ldda=hda_ldda) + job = self.service.get_job(trans, dataset_id=dataset_id, hda_ldda=hda_ldda) return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] @router.get( @@ -487,32 +485,32 @@ def search( return [EncodedJobDetails(**single_job.to_dict("element")) for single_job in jobs] @router.get( - "/api/jobs/{id}", + "/api/jobs/{job_id}", name="show_job", summary="Return dictionary containing description of job data.", ) def show( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], full: Annotated[Optional[bool], FullShowQueryParam] = False, trans: ProvidesUserContext = DependsOnTrans, ) -> Dict[str, Any]: - return self.service.show(trans, id, bool(full)) + return self.service.show(trans, job_id, bool(full)) # TODO find the mapping of the legacy route # TODO rename? --> function cancels/stops job (no deletion involved?) @router.delete( - "/api/jobs/{id}", + "/api/jobs/{job_id}", name="cancel_job", summary="Cancels specified job", ) def delete( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, payload: Annotated[Optional[DeleteJobPayload], DeleteJobBody] = None, ) -> bool: - job = self.service.get_job(trans=trans, job_id=id) + job = self.service.get_job(trans=trans, job_id=job_id) if payload: message = payload.message else: diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index a4e58aec84f2..3fe9dd89dfbe 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -985,7 +985,6 @@ def test_destination_params(self, history_id): job_id = search_response.json()[0]["id"] destination_params_response = self._get(f"/api/jobs/{job_id}/destination_params", admin=True) self._assert_status_code_is(destination_params_response, 200) - # TODO maybe extend test? @pytest.mark.require_new_history def test_job_metrics(self, history_id): @@ -997,7 +996,6 @@ def test_job_metrics(self, history_id): self._assert_status_code_is(metrics_by_job_response, 200) metrics_by_dataset_response = self._get(f"/api/datasets/{dataset_id}/metrics", data={"hda_ldda": "hda"}) self._assert_status_code_is(metrics_by_dataset_response, 200) - # TODO maybe extend test? @pytest.mark.require_new_history def test_parameters_display(self, history_id): @@ -1013,7 +1011,6 @@ def test_parameters_display(self, history_id): f"/api/datasets/{dataset_id}/parameters_display", data={"hda_ldda": "hda"} ) self._assert_status_code_is(display_parameters_by_dataset_response, 200) - # TODO maybe extend test def _create_and_search_job(self, history_id, inputs, tool_id): # create a job @@ -1096,3 +1093,4 @@ def __jobs_index(self, **kwds): jobs = jobs_response.json() assert isinstance(jobs, list) return jobs + return jobs From dd410f4e91709e11d67d616285748af2a3277922 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 10 Nov 2023 16:55:09 +0100 Subject: [PATCH 084/110] Apply some suggestions on pydantic models --- lib/galaxy/schema/jobs.py | 97 ++++++------------------------------- lib/galaxy/schema/schema.py | 17 +++++++ 2 files changed, 32 insertions(+), 82 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 3e97c2d22604..6d5b6365724f 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -3,7 +3,6 @@ Dict, List, Optional, - Union, ) from pydantic import ( @@ -18,12 +17,11 @@ EncodedDatabaseIdField, ) from galaxy.schema.schema import ( - DatasetSourceType, - EncodedDatasetSourceId, + EncodedDataItemSourceId, EntityIdField, + JobState, JobSummary, Model, - UuidField, ) @@ -56,7 +54,7 @@ class JobAssociation(Model): title="name", description="Name of the job parameter.", ) - dataset: EncodedDatasetSourceId = Field( + dataset: EncodedDataItemSourceId = Field( default=Required, title="dataset", description="Reference to the associated item.", @@ -94,10 +92,10 @@ class SearchJobsPayload(Model): title="Inputs", description="The inputs of the job as a JSON dump.", ) - state: str = Field( + state: JobState = Field( default=Required, title="State", - description="The state of the job.", + description="Current state of the job.", ) class Config: @@ -112,8 +110,13 @@ class DeleteJobPayload(Model): ) -class EncodedDatasetJobInfo(EncodedDatasetSourceId): - uuid: UUID4 = UuidField +class EncodedDatasetJobInfo(EncodedDataItemSourceId): + uuid: Optional[UUID4] = Field( + default=None, + deprecated=True, + title="UUID", + description="Universal unique identifier for this dataset. In this context the uuid is optional and marked as deprecated.", + ) class EncodedJobIDs(Model): @@ -153,7 +156,7 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): copied_from_job_id: Optional[EncodedDatabaseIdField] = Field( default=None, title="Copied from Job-ID", description="Reference to cached job if job execution was cached." ) - output_collections: Any = Field(default={}, title="Output collections", description="?") + output_collections: Any = Field(default=None, title="Output collections", description="?") class JobDestinationParams(Model): @@ -170,75 +173,7 @@ class JobDestinationParams(Model): class JobOutput(Model): label: Any = Field(default=Required, title="Output label", description="The output label") # check if this is true - value: EncodedDatasetSourceId = Field(default=Required, title="dataset", description="The associated dataset.") - - -class JobParameterValuesSimple(Model): - src: DatasetSourceType = Field( - default=Required, - title="Source", - description="The source of this dataset, either `hda` or `ldda` depending of its origin.", - ) - id: EncodedDatabaseIdField = EntityIdField - name: str = Field( - default=Required, - title="Name", - description="The name of the item.", - ) - hid: int = Field( - default=Required, - title="HID", - description="The index position of this item in the History.", - ) - - -class JobTargetElement(Model): # TODO rename? - name: str = Field(default=Required, title="Name", description="Name of the element") # TODO check correctness - dbkey: str = Field(default=Required, title="Database Key", description="Database key") # TODO check correctness - ext: str = Field(default=Required, title="Extension", description="Extension") # TODO check correctness - to_posix_lines: bool = Field( - default=Required, title="To POSIX Lines", description="Flag indicating if to convert to POSIX lines" - ) # TODO check correctness - auto_decompress: bool = Field( - default=Required, title="Auto Decompress", description="Flag indicating if to auto decompress" - ) # TODO check correctness - src: str = Field(default=Required, title="Source", description="Source") # TODO check correctness - paste_content: str = Field( - default=Required, title="Paste Content", description="Content to paste" - ) # TODO check correctness - hashes: List[str] = Field(default=Required, title="Hashes", description="List of hashes") # TODO check correctness - purge_source: bool = Field( - default=Required, title="Purge Source", description="Flag indicating if to purge the source" - ) # TODO check correctness - object_id: EncodedDatabaseIdField = Field( - default=Required, title="Object ID", description="Object ID" - ) # TODO check correctness - - -class JobTargetDestination(Model): - type: str = Field( - default=Required, - title="Type", - description="Type of destination, either `hda` or `ldda` depending on the destination", - ) # TODO check correctness - - -class JobTarget(Model): # TODO rename? - destination: JobTargetDestination = Field( - default=Required, title="Destination", description="Destination details" - ) # TODO check correctness - elements: List[JobTargetElement] = Field( - default=Required, title="Elements", description="List of job target elements" - ) # TODO check correctness - - -class JobParameterValuesExtensive(Model): # TODO rename? - targets: List[JobTarget] = Field( - default=Required, title="Targets", description="List of job value targets" - ) # TODO check correctness - check_content: bool = Field( - default=Required, title="Check Content", description="Flag to check content" - ) # TODO check correctness + value: EncodedDataItemSourceId = Field(default=Required, title="dataset", description="The associated dataset.") class JobParameter(Model): @@ -252,9 +187,7 @@ class JobParameter(Model): title="Depth", description="The depth of the job parameter.", ) - value: Union[List[JobParameterValuesSimple], JobParameterValuesExtensive, str] = Field( - default=Required, title="Value", description="The values of the job parameter" - ) + value: Any = Field(default=Required, title="Value", description="The values of the job parameter") notes: Optional[str] = Field(default=None, title="notes", description="Notes associated with the job parameter.") diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 5ca9dd266655..4396e270f2f2 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -475,6 +475,14 @@ class DatasetSourceType(str, Enum): ldda = "ldda" +class DataItemSourceType(str, Enum): + hda = "hda" + ldda = "ldda" + hdca = "hdca" + dce = "dce" + dc = "dc" + + class ColletionSourceType(str, Enum): hda = "hda" ldda = "ldda" @@ -1928,6 +1936,15 @@ class EncodedDatasetSourceId(DatasetSourceIdBase): id: EncodedDatabaseIdField = EntityIdField +class EncodedDataItemSourceId(Model): + id: EncodedDatabaseIdField = EntityIdField + src: DataItemSourceType = Field( + ..., + title="Source", + description="The source of this dataset, either `hda` or `ldda` depending of its origin.", + ) + + class DatasetJobInfo(DatasetSourceId): uuid: UUID4 = UuidField From b0852aec38bbcf1d358360b36c150fdd526ef0f7 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 10 Nov 2023 19:36:35 +0100 Subject: [PATCH 085/110] Refined EncodedJobDetails model --- lib/galaxy/schema/jobs.py | 21 +++++++++++++++++---- lib/galaxy/schema/schema.py | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 6d5b6365724f..d0b71413b0ee 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import ( Any, Dict, @@ -110,12 +111,24 @@ class DeleteJobPayload(Model): ) +class EncodedHdcaSourceId(Model): + class Hdca(str, Enum): + hdca = "hdca" + + id: EncodedDatabaseIdField = EntityIdField + src: Hdca = Field( + default=Required, + title="Source", + description="The source of this dataset, which in the case of the model can only be `hdca`.", + ) + + class EncodedDatasetJobInfo(EncodedDataItemSourceId): uuid: Optional[UUID4] = Field( default=None, deprecated=True, title="UUID", - description="Universal unique identifier for this dataset. In this context the uuid is optional and marked as deprecated.", + description="Universal unique identifier for this dataset.", ) @@ -156,7 +169,7 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): copied_from_job_id: Optional[EncodedDatabaseIdField] = Field( default=None, title="Copied from Job-ID", description="Reference to cached job if job execution was cached." ) - output_collections: Any = Field(default=None, title="Output collections", description="?") + output_collections: Dict[str, EncodedHdcaSourceId] = Field(default={}, title="Output collections", description="?") class JobDestinationParams(Model): @@ -173,7 +186,7 @@ class JobDestinationParams(Model): class JobOutput(Model): label: Any = Field(default=Required, title="Output label", description="The output label") # check if this is true - value: EncodedDataItemSourceId = Field(default=Required, title="dataset", description="The associated dataset.") + value: EncodedDataItemSourceId = Field(default=Required, title="Dataset", description="The associated dataset.") class JobParameter(Model): @@ -188,7 +201,7 @@ class JobParameter(Model): description="The depth of the job parameter.", ) value: Any = Field(default=Required, title="Value", description="The values of the job parameter") - notes: Optional[str] = Field(default=None, title="notes", description="Notes associated with the job parameter.") + notes: Optional[str] = Field(default=None, title="Notes", description="Notes associated with the job parameter.") class JobDisplayParametersSummary(Model): diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 4396e270f2f2..c726ba43829e 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1941,7 +1941,7 @@ class EncodedDataItemSourceId(Model): src: DataItemSourceType = Field( ..., title="Source", - description="The source of this dataset, either `hda` or `ldda` depending of its origin.", + description="The source of this dataset, either `hda`, `ldda`, `hdca`, `dce` or `dc` depending of its origin.", ) From 8c6a64e3ab89c6ae3d97663b3194fe7fed6967a1 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 10 Nov 2023 19:42:43 +0100 Subject: [PATCH 086/110] Add description to JobDestinationParams model and remove TODOs --- lib/galaxy/schema/jobs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index d0b71413b0ee..f8fc76c0631e 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -165,15 +165,13 @@ class EncodedJobDetails(JobSummary, EncodedJobIDs): title="Outputs", description="Dictionary mapping all the tool outputs (by name) to the corresponding data references.", ) - # TODO add description, check type and add proper default copied_from_job_id: Optional[EncodedDatabaseIdField] = Field( default=None, title="Copied from Job-ID", description="Reference to cached job if job execution was cached." ) - output_collections: Dict[str, EncodedHdcaSourceId] = Field(default={}, title="Output collections", description="?") + output_collections: Dict[str, EncodedHdcaSourceId] = Field(default={}, title="Output collections", description="") class JobDestinationParams(Model): - # TODO add description, check type and add proper default runner: str = Field(default=Required, title="Runner", description="Job runner class", alias="Runner") runner_job_id: str = Field( default=Required, @@ -181,7 +179,9 @@ class JobDestinationParams(Model): description="ID assigned to submitted job by external job running system", alias="Runner Job ID", ) - handler: str = Field(default=Required, title="Handler", description="?", alias="Handler") + handler: str = Field( + default=Required, title="Handler", description="Name of the process that handled the job.", alias="Handler" + ) class JobOutput(Model): From 3dd5cb70b338a8b27dd089ccefaaa8725f81d9d8 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 10 Nov 2023 19:43:37 +0100 Subject: [PATCH 087/110] Regenerate client schema --- client/src/api/schema/schema.ts | 599 ++++++++++++++------------------ 1 file changed, 258 insertions(+), 341 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 95a8845c2ffe..cb460c170919 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -167,6 +167,24 @@ export interface paths { /** For internal use, this endpoint may change without warning. */ get: operations["show_inheritance_chain_api_datasets__dataset_id__inheritance_chain_get"]; }; + "/api/datasets/{dataset_id}/metrics": { + /** Return job metrics for specified job. */ + get: operations["get_metrics_api_datasets__dataset_id__metrics_get"]; + }; + "/api/datasets/{dataset_id}/parameters_display": { + /** + * Resolve parameters as a list for nested display. + * @description Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + */ + get: operations["resolve_parameters_display_api_datasets__dataset_id__parameters_display_get"]; + }; "/api/datasets/{dataset_id}/permissions": { /** * Set permissions of the given history dataset to the given role ids. @@ -196,25 +214,6 @@ export interface paths { /** Check if metadata file can be downloaded. */ head: operations["get_metadata_file_datasets_api_datasets__history_content_id__metadata_file_head"]; }; - "/api/datasets/{id}/metrics": { - /** Return job metrics for specified job. */ - get: operations["get_metrics_api_datasets__id__metrics_get"]; - }; - "/api/datasets/{id}/parameters_display": { - /** - * Resolve parameters as a list for nested display. - * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. - */ - get: operations["resolve_parameters_display_api_datasets__id__parameters_display_get"]; - }; "/api/datatypes": { /** * Lists all available data types @@ -921,12 +920,6 @@ export interface paths { */ post: operations["search_jobs_api_jobs_search_post"]; }; - "/api/jobs/{id}": { - /** Return dictionary containing description of job data. */ - get: operations["show_job_api_jobs__id__get"]; - /** Cancels specified job */ - delete: operations["cancel_job_api_jobs__id__delete"]; - }; "/api/jobs/{id}/common_problems": { /** Check inputs and job for common potential problems to aid in error reporting */ get: operations["check_common_problems_api_jobs__id__common_problems_get"]; @@ -939,37 +932,28 @@ export interface paths { /** Returns input datasets created by a job. */ get: operations["get_inputs_api_jobs__id__inputs_get"]; }; - "/api/jobs/{id}/metrics": { - /** Return job metrics for specified job. */ - get: operations["get_metrics_api_jobs__id__metrics_get"]; - }; "/api/jobs/{id}/outputs": { /** Returns output datasets created by a job. */ get: operations["get_outputs_api_jobs__id__outputs_get"]; }; - "/api/jobs/{id}/parameters_display": { - /** - * Resolve parameters as a list for nested display. - * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. - */ - get: operations["resolve_parameters_display_api_jobs__id__parameters_display_get"]; - }; "/api/jobs/{id}/resume": { /** Resumes a paused job. */ put: operations["resume_paused_job_api_jobs__id__resume_put"]; }; + "/api/jobs/{job_id}": { + /** Return dictionary containing description of job data. */ + get: operations["show_job_api_jobs__job_id__get"]; + /** Cancels specified job */ + delete: operations["cancel_job_api_jobs__job_id__delete"]; + }; "/api/jobs/{job_id}/destination_params": { /** Return destination parameters for specified job. */ get: operations["destination_params_job_api_jobs__job_id__destination_params_get"]; }; + "/api/jobs/{job_id}/metrics": { + /** Return job metrics for specified job. */ + get: operations["get_metrics_api_jobs__job_id__metrics_get"]; + }; "/api/jobs/{job_id}/oidc-tokens": { /** * Get a fresh OIDC token @@ -977,6 +961,20 @@ export interface paths { */ get: operations["get_token_api_jobs__job_id__oidc_tokens_get"]; }; + "/api/jobs/{job_id}/parameters_display": { + /** + * Resolve parameters as a list for nested display. + * @description Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + */ + get: operations["resolve_parameters_display_api_jobs__job_id__parameters_display_get"]; + }; "/api/libraries": { /** * Returns a list of summary data for all libraries. @@ -3446,6 +3444,12 @@ export interface components { | components["schemas"]["NestedElement"] )[]; }; + /** + * DataItemSourceType + * @description An enumeration. + * @enum {string} + */ + DataItemSourceType: "hda" | "ldda" | "hdca" | "dce" | "dc"; /** * DatasetAssociationRoles * @description Base model definition with common configuration used by all derived models. @@ -4150,6 +4154,23 @@ export interface components { * @enum {string} */ ElementsFromType: "archive" | "bagit" | "bagit_archive" | "directory"; + /** + * EncodedDataItemSourceId + * @description Base model definition with common configuration used by all derived models. + */ + EncodedDataItemSourceId: { + /** + * ID + * @description The encoded ID of this entity. + * @example 0123456789ABCDEF + */ + id: string; + /** + * Source + * @description The source of this dataset, either `hda`, `ldda`, `hdca`, `dce` or `dc` depending of its origin. + */ + src: components["schemas"]["DataItemSourceType"]; + }; /** * EncodedDatasetJobInfo * @description Base model definition with common configuration used by all derived models. @@ -4163,15 +4184,16 @@ export interface components { id: string; /** * Source - * @description The source of this dataset, either `hda` or `ldda` depending of its origin. + * @description The source of this dataset, either `hda`, `ldda`, `hdca`, `dce` or `dc` depending of its origin. */ - src: components["schemas"]["DatasetSourceType"]; + src: components["schemas"]["DataItemSourceType"]; /** * UUID * Format: uuid4 + * @deprecated * @description Universal unique identifier for this dataset. */ - uuid: string; + uuid?: string; }; /** * EncodedDatasetSourceId @@ -4190,6 +4212,23 @@ export interface components { */ src: components["schemas"]["DatasetSourceType"]; }; + /** + * EncodedHdcaSourceId + * @description Base model definition with common configuration used by all derived models. + */ + EncodedHdcaSourceId: { + /** + * ID + * @description The encoded ID of this entity. + * @example 0123456789ABCDEF + */ + id: string; + /** + * Source + * @description The source of this dataset, which in the case of the model can only be `hdca`. + */ + src: components["schemas"]["Hdca"]; + }; /** * EncodedHistoryContentItem * @description Identifies a dataset or collection contained in a History. @@ -4224,7 +4263,7 @@ export interface components { command_version: string; /** * Copied from Job-ID - * @description ? + * @description Reference to cached job if job execution was cached. * @example 0123456789ABCDEF */ copied_from_job_id?: string; @@ -4264,7 +4303,7 @@ export interface components { id: string; /** * Inputs - * @description Dictionary mapping all the tool inputs (by name) with the corresponding dataset information. + * @description Dictionary mapping all the tool inputs (by name) to the corresponding data references. * @default {} */ inputs?: { @@ -4279,13 +4318,14 @@ export interface components { model_class: "Job"; /** * Output collections - * @description ? * @default {} */ - output_collections?: Record; + output_collections?: { + [key: string]: components["schemas"]["EncodedHdcaSourceId"] | undefined; + }; /** * Outputs - * @description Dictionary mapping all the tool outputs (by name) with the corresponding dataset information. + * @description Dictionary mapping all the tool outputs (by name) to the corresponding data references. * @default {} */ outputs?: { @@ -5687,6 +5727,12 @@ export interface components { */ type: "hdas"; }; + /** + * Hdca + * @description An enumeration. + * @enum {string} + */ + Hdca: "hdca"; /** * HdcaDataItemsFromTarget * @description Base model definition with common configuration used by all derived models. @@ -6324,12 +6370,12 @@ export interface components { JobAssociation: { /** * dataset - * @description The associated dataset. + * @description Reference to the associated item. */ - dataset: components["schemas"]["EncodedDatasetSourceId"]; + dataset: components["schemas"]["EncodedDataItemSourceId"]; /** * name - * @description The name of the associated dataset. + * @description Name of the job parameter. */ name: string; }; @@ -6340,17 +6386,17 @@ export interface components { JobDestinationParams: { /** * Handler - * @description ? + * @description Name of the process that handled the job. */ Handler: string; /** * Runner - * @description ? + * @description Job runner class */ Runner: string; /** * Runner Job ID - * @description ? + * @description ID assigned to submitted job by external job running system */ "Runner Job ID": string; }; @@ -6603,10 +6649,10 @@ export interface components { */ label: Record; /** - * dataset + * Dataset * @description The associated dataset. */ - value: components["schemas"]["EncodedDatasetSourceId"]; + value: components["schemas"]["EncodedDataItemSourceId"]; }; /** * JobParameter @@ -6619,7 +6665,7 @@ export interface components { */ depth: number; /** - * notes + * Notes * @description Notes associated with the job parameter. */ notes?: string; @@ -6632,53 +6678,7 @@ export interface components { * Value * @description The values of the job parameter */ - value: - | components["schemas"]["JobParameterValuesSimple"][] - | components["schemas"]["JobParameterValuesExtensive"] - | string; - }; - /** - * JobParameterValuesExtensive - * @description Base model definition with common configuration used by all derived models. - */ - JobParameterValuesExtensive: { - /** - * Check Content - * @description Flag to check content - */ - check_content: boolean; - /** - * Targets - * @description List of job value targets - */ - targets: components["schemas"]["JobTarget"][]; - }; - /** - * JobParameterValuesSimple - * @description Base model definition with common configuration used by all derived models. - */ - JobParameterValuesSimple: { - /** - * HID - * @description The index position of this item in the History. - */ - hid: number; - /** - * ID - * @description The encoded ID of this entity. - * @example 0123456789ABCDEF - */ - id: string; - /** - * Name - * @description The name of the item. - */ - name: string; - /** - * Source - * @description The source of this dataset, either `hda` or `ldda` depending of its origin. - */ - src: components["schemas"]["DatasetSourceType"]; + value: Record; }; /** * JobSourceType @@ -6739,90 +6739,6 @@ export interface components { [key: string]: number | undefined; }; }; - /** - * JobTarget - * @description Base model definition with common configuration used by all derived models. - */ - JobTarget: { - /** - * Destination - * @description Destination details - */ - destination: components["schemas"]["JobTargetDestination"]; - /** - * Elements - * @description List of job target elements - */ - elements: components["schemas"]["JobTargetElement"][]; - }; - /** - * JobTargetDestination - * @description Base model definition with common configuration used by all derived models. - */ - JobTargetDestination: { - /** - * Type - * @description Type of destination, either `hda` or `ldda` depending on the destination - */ - type: string; - }; - /** - * JobTargetElement - * @description Base model definition with common configuration used by all derived models. - */ - JobTargetElement: { - /** - * Auto Decompress - * @description Flag indicating if to auto decompress - */ - auto_decompress: boolean; - /** - * Database Key - * @description Database key - */ - dbkey: string; - /** - * Extension - * @description Extension - */ - ext: string; - /** - * Hashes - * @description List of hashes - */ - hashes: string[]; - /** - * Name - * @description Name of the element - */ - name: string; - /** - * Object ID - * @description Object ID - * @example 0123456789ABCDEF - */ - object_id: string; - /** - * Paste Content - * @description Content to paste - */ - paste_content: string; - /** - * Purge Source - * @description Flag indicating if to purge the source - */ - purge_source: boolean; - /** - * Source - * @description Source - */ - src: string; - /** - * To POSIX Lines - * @description Flag indicating if to convert to POSIX lines - */ - to_posix_lines: boolean; - }; /** * LabelValuePair * @description Generic Label/Value pair model. @@ -8629,12 +8545,15 @@ export interface components { */ ReportJobErrorPayload: { /** - * Dataset ID - * @description The dataset ID related to the error. + * History Dataset Association ID + * @description The History Dataset Association ID related to the error. * @example 0123456789ABCDEF */ dataset_id: string; - /** Email */ + /** + * Email + * @description Email address for communication with the user. Only required for anonymous users. + */ email?: string; /** * Message @@ -8745,9 +8664,9 @@ export interface components { inputs: string; /** * State - * @description The state of the job. + * @description Current state of the job. */ - state: string; + state: components["schemas"]["JobState"]; /** * Tool ID * @description The tool ID related to the job. @@ -11083,6 +11002,78 @@ export interface operations { }; }; }; + get_metrics_api_datasets__dataset_id__metrics_get: { + /** Return job metrics for specified job. */ + parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the dataset */ + path: { + dataset_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobMetric"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + resolve_parameters_display_api_datasets__dataset_id__parameters_display_get: { + /** + * Resolve parameters as a list for nested display. + * @description Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. + */ + parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The ID of the dataset */ + path: { + dataset_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["JobDisplayParametersSummary"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; update_permissions_api_datasets__dataset_id__permissions_put: { /** * Set permissions of the given history dataset to the given role ids. @@ -11299,79 +11290,6 @@ export interface operations { }; }; }; - get_metrics_api_datasets__id__metrics_get: { - /** Return job metrics for specified job. */ - parameters: { - /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ - query?: { - hda_ldda?: components["schemas"]["DatasetSourceType"]; - }; - /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ - header?: { - "run-as"?: string; - }; - /** @description The ID of the dataset */ - path: { - id: string; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - content: { - "application/json": components["schemas"]["JobMetric"][]; - }; - }; - /** @description Validation Error */ - 422: { - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - resolve_parameters_display_api_datasets__id__parameters_display_get: { - /** - * Resolve parameters as a list for nested display. - * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. - */ - parameters: { - /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ - query?: { - hda_ldda?: components["schemas"]["DatasetSourceType"]; - }; - /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ - header?: { - "run-as"?: string; - }; - /** @description The ID of the dataset */ - path: { - id: string; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - content: { - "application/json": components["schemas"]["JobDisplayParametersSummary"]; - }; - }; - /** @description Validation Error */ - 422: { - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; index_api_datatypes_get: { /** * Lists all available data types @@ -15321,13 +15239,9 @@ export interface operations { }; }; }; - show_job_api_jobs__id__get: { - /** Return dictionary containing description of job data. */ + check_common_problems_api_jobs__id__common_problems_get: { + /** Check inputs and job for common potential problems to aid in error reporting */ parameters: { - /** @description Show extra information. */ - query?: { - full?: boolean; - }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; @@ -15341,7 +15255,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["JobInputSummary"]; }; }; /** @description Validation Error */ @@ -15352,8 +15266,8 @@ export interface operations { }; }; }; - cancel_job_api_jobs__id__delete: { - /** Cancels specified job */ + report_error_api_jobs__id__error_post: { + /** Submits a bug report via the API. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15364,16 +15278,16 @@ export interface operations { id: string; }; }; - requestBody?: { + requestBody: { content: { - "application/json": components["schemas"]["DeleteJobPayload"]; + "application/json": components["schemas"]["ReportJobErrorPayload"]; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": boolean; + "application/json": components["schemas"]["JobErrorSummary"]; }; }; /** @description Validation Error */ @@ -15384,8 +15298,8 @@ export interface operations { }; }; }; - check_common_problems_api_jobs__id__common_problems_get: { - /** Check inputs and job for common potential problems to aid in error reporting */ + get_inputs_api_jobs__id__inputs_get: { + /** Returns input datasets created by a job. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15400,7 +15314,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobInputSummary"]; + "application/json": components["schemas"]["JobAssociation"][]; }; }; /** @description Validation Error */ @@ -15411,8 +15325,8 @@ export interface operations { }; }; }; - report_error_api_jobs__id__error_post: { - /** Submits a bug report via the API. */ + get_outputs_api_jobs__id__outputs_get: { + /** Returns output datasets created by a job. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15423,16 +15337,11 @@ export interface operations { id: string; }; }; - requestBody: { - content: { - "application/json": components["schemas"]["ReportJobErrorPayload"]; - }; - }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobErrorSummary"]; + "application/json": components["schemas"]["JobAssociation"][]; }; }; /** @description Validation Error */ @@ -15443,8 +15352,8 @@ export interface operations { }; }; }; - get_inputs_api_jobs__id__inputs_get: { - /** Returns input datasets created by a job. */ + resume_paused_job_api_jobs__id__resume_put: { + /** Resumes a paused job. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15470,12 +15379,12 @@ export interface operations { }; }; }; - get_metrics_api_jobs__id__metrics_get: { - /** Return job metrics for specified job. */ + show_job_api_jobs__job_id__get: { + /** Return dictionary containing description of job data. */ parameters: { - /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + /** @description Show extra information. */ query?: { - hda_ldda?: components["schemas"]["DatasetSourceType"]; + full?: boolean; }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15483,14 +15392,14 @@ export interface operations { }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobMetric"][]; + "application/json": Record; }; }; /** @description Validation Error */ @@ -15501,8 +15410,8 @@ export interface operations { }; }; }; - get_outputs_api_jobs__id__outputs_get: { - /** Returns output datasets created by a job. */ + cancel_job_api_jobs__job_id__delete: { + /** Cancels specified job */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15510,14 +15419,19 @@ export interface operations { }; /** @description The ID of the job */ path: { - id: string; + job_id: string; + }; + }; + requestBody?: { + content: { + "application/json": components["schemas"]["DeleteJobPayload"]; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobAssociation"][]; + "application/json": boolean; }; }; /** @description Validation Error */ @@ -15528,38 +15442,23 @@ export interface operations { }; }; }; - resolve_parameters_display_api_jobs__id__parameters_display_get: { - /** - * Resolve parameters as a list for nested display. - * @description # TODO is this still true? - * Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. - */ + destination_params_job_api_jobs__job_id__destination_params_get: { + /** Return destination parameters for specified job. */ parameters: { - /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ - query?: { - hda_ldda?: components["schemas"]["DatasetSourceType"]; - }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobDisplayParametersSummary"]; + "application/json": components["schemas"]["JobDestinationParams"]; }; }; /** @description Validation Error */ @@ -15570,23 +15469,27 @@ export interface operations { }; }; }; - resume_paused_job_api_jobs__id__resume_put: { - /** Resumes a paused job. */ + get_metrics_api_jobs__job_id__metrics_get: { + /** Return job metrics for specified job. */ parameters: { + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; + }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobAssociation"][]; + "application/json": components["schemas"]["JobMetric"][]; }; }; /** @description Validation Error */ @@ -15597,14 +15500,22 @@ export interface operations { }; }; }; - destination_params_job_api_jobs__job_id__destination_params_get: { - /** Return destination parameters for specified job. */ + get_token_api_jobs__job_id__oidc_tokens_get: { + /** + * Get a fresh OIDC token + * @description Allows remote job running mechanisms to get a fresh OIDC token that can be used on remote side to authorize user. It is not meant to represent part of Galaxy's stable, user facing API + */ parameters: { + /** @description A key used to authenticate this request as acting onbehalf or a job runner for the specified job */ + /** @description OIDC provider name */ + query: { + job_key: string; + provider: string; + }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; - /** @description The ID of the job */ path: { job_id: string; }; @@ -15613,7 +15524,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobDestinationParams"]; + "text/plain": string; }; }; /** @description Validation Error */ @@ -15624,22 +15535,28 @@ export interface operations { }; }; }; - get_token_api_jobs__job_id__oidc_tokens_get: { + resolve_parameters_display_api_jobs__job_id__parameters_display_get: { /** - * Get a fresh OIDC token - * @description Allows remote job running mechanisms to get a fresh OIDC token that can be used on remote side to authorize user. It is not meant to represent part of Galaxy's stable, user facing API + * Resolve parameters as a list for nested display. + * @description Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. */ parameters: { - /** @description A key used to authenticate this request as acting onbehalf or a job runner for the specified job */ - /** @description OIDC provider name */ - query: { - job_key: string; - provider: string; + /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; + /** @description The ID of the job */ path: { job_id: string; }; @@ -15648,7 +15565,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "text/plain": string; + "application/json": components["schemas"]["JobDisplayParametersSummary"]; }; }; /** @description Validation Error */ From 9b107d7c85ab1b8bd0659c13d4d71103d6a3d9d9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Fri, 10 Nov 2023 20:55:18 +0100 Subject: [PATCH 088/110] Mark *_by_dataset operations as deprecated --- lib/galaxy/webapps/galaxy/api/jobs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index f2f4c4816dc0..e7d956b1fe97 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -374,6 +374,7 @@ def parameters_display_by_job( "/api/datasets/{dataset_id}/parameters_display", name="resolve_parameters_display", summary="Resolve parameters as a list for nested display.", + deprecated=True, ) def parameters_display_by_dataset( self, @@ -412,6 +413,7 @@ def metrics_by_job( "/api/datasets/{dataset_id}/metrics", name="get_metrics", summary="Return job metrics for specified job.", + deprecated=True, ) def metrics_by_dataset( self, From ecddea3a4f7ff52cd6cb9c55c2d975ea36969e65 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 11 Nov 2023 18:00:08 +0100 Subject: [PATCH 089/110] Mark some HdaLddaQueryParam's as deprecated --- lib/galaxy/webapps/galaxy/api/jobs.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index e7d956b1fe97..e1056fca854b 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -181,8 +181,14 @@ ) FullShowQueryParam: Optional[bool] = Query(title="Full show", description="Show extra information.") +DeprecatedHdaLddaQueryParam: DatasetSourceType = Query( + deprecated=True, + title="HDA or LDDA", + description="Whether this dataset belongs to a history (HDA) or a library (LDDA).", +) HdaLddaQueryParam: DatasetSourceType = Query( - title="HDA or LDDA", description="Whether this dataset belongs to a history (HDA) or a library (LDDA)." + title="HDA or LDDA", + description="Whether this dataset belongs to a history (HDA) or a library (LDDA).", ) @@ -354,7 +360,7 @@ def outputs( def parameters_display_by_job( self, job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], - hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, + hda_ldda: Annotated[Optional[DatasetSourceType], DeprecatedHdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> JobDisplayParametersSummary: """ @@ -403,7 +409,7 @@ def parameters_display_by_dataset( def metrics_by_job( self, job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], - hda_ldda: Annotated[DatasetSourceType, HdaLddaQueryParam] = DatasetSourceType.hda, + hda_ldda: Annotated[Optional[DatasetSourceType], DeprecatedHdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> List[Optional[JobMetric]]: job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda) @@ -413,7 +419,7 @@ def metrics_by_job( "/api/datasets/{dataset_id}/metrics", name="get_metrics", summary="Return job metrics for specified job.", - deprecated=True, + deprecated=True, ) def metrics_by_dataset( self, From a48ebf1c7d4e601497a93f328fdc13bb6ce78435 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 11 Nov 2023 18:35:21 +0100 Subject: [PATCH 090/110] Follow up on previous deprecation and refactor create operation --- lib/galaxy/webapps/galaxy/api/jobs.py | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index e1056fca854b..bb959c8b5c2f 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -21,6 +21,7 @@ from fastapi import ( Body, Depends, + HTTPException, Path, Query, ) @@ -56,10 +57,7 @@ JobMetric, ) from galaxy.schema.types import OffsetNaiveDatetime -from galaxy.web import ( - expose_api, - expose_api_anonymous, -) +from galaxy.web import expose_api_anonymous from galaxy.webapps.base.controller import UsesVisualizationMixin from galaxy.webapps.galaxy.api import ( BaseGalaxyAPIController, @@ -181,7 +179,7 @@ ) FullShowQueryParam: Optional[bool] = Query(title="Full show", description="Show extra information.") -DeprecatedHdaLddaQueryParam: DatasetSourceType = Query( +DeprecatedHdaLddaQueryParam: Optional[DatasetSourceType] = Query( deprecated=True, title="HDA or LDDA", description="Whether this dataset belongs to a history (HDA) or a library (LDDA).", @@ -244,6 +242,19 @@ def index( ) return self.service.index(trans, payload) + @router.post( + "/api/jobs", + name="create_job", + summary="Not implemented.", + ) + def create( + self, + trans: ProvidesUserContext = DependsOnTrans, + **kwd, + ): + """See the create method in tools.py in order to submit a job.""" + raise HTTPException(status_code=501, detail="Please POST to /api/tools instead.") + @router.get( "/api/jobs/{id}/common_problems", name="check_common_problems", @@ -373,7 +384,8 @@ def parameters_display_by_job( This API endpoint is unstable and tied heavily to Galaxy's JS client code, this endpoint will change frequently. """ - job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda) + hda_ldda_str = hda_ldda or "hda" + job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda_str) return summarize_job_parameters(trans, job) @router.get( @@ -412,7 +424,8 @@ def metrics_by_job( hda_ldda: Annotated[Optional[DatasetSourceType], DeprecatedHdaLddaQueryParam] = DatasetSourceType.hda, trans: ProvidesUserContext = DependsOnTrans, ) -> List[Optional[JobMetric]]: - job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda) + hda_ldda_str = hda_ldda or "hda" + job = self.service.get_job(trans, job_id=job_id, hda_ldda=hda_ldda_str) return [JobMetric(**metric) for metric in summarize_job_metrics(trans, job)] @router.get( @@ -505,8 +518,6 @@ def show( ) -> Dict[str, Any]: return self.service.show(trans, job_id, bool(full)) - # TODO find the mapping of the legacy route - # TODO rename? --> function cancels/stops job (no deletion involved?) @router.delete( "/api/jobs/{job_id}", name="cancel_job", @@ -563,8 +574,3 @@ def __get_job(self, trans, job_id=None, dataset_id=None, **kwd): # Following checks dataset accessible dataset_instance = self.get_hda_or_ldda(trans, hda_ldda=hda_ldda, dataset_id=dataset_id) return dataset_instance.creating_job - - @expose_api - def create(self, trans: ProvidesUserContext, payload, **kwd): - """See the create method in tools.py in order to submit a job.""" - raise exceptions.NotImplemented("Please POST to /api/tools instead.") From 75def3549ceecf8d016d2c2ddd12237ac61f41fe Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 11 Nov 2023 18:35:49 +0100 Subject: [PATCH 091/110] Remove mapping to some legacy routes --- lib/galaxy/webapps/galaxy/buildapp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index ce72aed4c70f..ed2b363e868f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -922,8 +922,6 @@ def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditio webapp, name_prefix="library_dataset_", path_prefix="/api/libraries/{library_id}/contents/{library_content_id}" ) - webapp.mapper.resource("job", "jobs", path_prefix="/api") - webapp.mapper.connect( "build_for_rerun", "/api/jobs/{id}/build_for_rerun", From 92559ef84acb2cf4c97ce85f6457740837b6d0eb Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 11 Nov 2023 18:46:11 +0100 Subject: [PATCH 092/110] Regenerate client schema --- client/src/api/schema/schema.ts | 56 ++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index cb460c170919..009c55c586c0 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -168,12 +168,16 @@ export interface paths { get: operations["show_inheritance_chain_api_datasets__dataset_id__inheritance_chain_get"]; }; "/api/datasets/{dataset_id}/metrics": { - /** Return job metrics for specified job. */ + /** + * Return job metrics for specified job. + * @deprecated + */ get: operations["get_metrics_api_datasets__dataset_id__metrics_get"]; }; "/api/datasets/{dataset_id}/parameters_display": { /** * Resolve parameters as a list for nested display. + * @deprecated * @description Resolve parameters as a list for nested display. More client logic * here than is ideal but it is hard to reason about tool parameter * types on the client relative to the server. Job accessibility checks @@ -910,6 +914,11 @@ export interface paths { "/api/jobs": { /** Index */ get: operations["index_api_jobs_get"]; + /** + * Not implemented. + * @description See the create method in tools.py in order to submit a job. + */ + post: operations["create_job_api_jobs_post"]; }; "/api/jobs/search": { /** @@ -11003,7 +11012,10 @@ export interface operations { }; }; get_metrics_api_datasets__dataset_id__metrics_get: { - /** Return job metrics for specified job. */ + /** + * Return job metrics for specified job. + * @deprecated + */ parameters: { /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ query?: { @@ -11036,6 +11048,7 @@ export interface operations { resolve_parameters_display_api_datasets__dataset_id__parameters_display_get: { /** * Resolve parameters as a list for nested display. + * @deprecated * @description Resolve parameters as a list for nested display. More client logic * here than is ideal but it is hard to reason about tool parameter * types on the client relative to the server. Job accessibility checks @@ -15206,6 +15219,35 @@ export interface operations { }; }; }; + create_job_api_jobs_post: { + /** + * Not implemented. + * @description See the create method in tools.py in order to submit a job. + */ + parameters: { + query: { + kwd: Record; + }; + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; search_jobs_api_jobs_search_post: { /** * Return jobs for current user @@ -15472,7 +15514,10 @@ export interface operations { get_metrics_api_jobs__job_id__metrics_get: { /** Return job metrics for specified job. */ parameters: { - /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + /** + * @deprecated + * @description Whether this dataset belongs to a history (HDA) or a library (LDDA). + */ query?: { hda_ldda?: components["schemas"]["DatasetSourceType"]; }; @@ -15548,7 +15593,10 @@ export interface operations { * this endpoint will change frequently. */ parameters: { - /** @description Whether this dataset belongs to a history (HDA) or a library (LDDA). */ + /** + * @deprecated + * @description Whether this dataset belongs to a history (HDA) or a library (LDDA). + */ query?: { hda_ldda?: components["schemas"]["DatasetSourceType"]; }; From 6571f087cadecf4e9d8b237bd6a08619227fe265 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sun, 12 Nov 2023 11:12:30 +0100 Subject: [PATCH 093/110] Divide JobAssociation into Input and Output model --- lib/galaxy/schema/jobs.py | 16 ++++++++++++++++ lib/galaxy/webapps/galaxy/api/jobs.py | 27 ++++++++++++++++++++------- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index f8fc76c0631e..49b6103385ab 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -62,6 +62,22 @@ class JobAssociation(Model): ) +class JobInputAssociation(JobAssociation): + name: str = Field( + default=Required, + title="name", + description="Name of the job input parameter.", + ) + + +class JobOutputAssociation(JobAssociation): + name: str = Field( + default=Required, + title="name", + description="Name of the job output parameter.", + ) + + class ReportJobErrorPayload(Model): dataset_id: DecodedDatabaseIdField = Field( default=Required, diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index bb959c8b5c2f..8e9f7e5e0674 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -43,11 +43,12 @@ from galaxy.schema.jobs import ( DeleteJobPayload, EncodedJobDetails, - JobAssociation, JobDestinationParams, JobDisplayParametersSummary, JobErrorSummary, + JobInputAssociation, JobInputSummary, + JobOutputAssociation, ReportJobErrorPayload, SearchJobsPayload, ) @@ -294,7 +295,7 @@ def resume( self, id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, - ) -> List[JobAssociation]: + ) -> List[JobOutputAssociation]: job = self.service.get_job(trans, job_id=id) if not job: raise exceptions.ObjectNotFound("Could not access job with the given id") @@ -302,7 +303,11 @@ def resume( job.resume() else: exceptions.RequestParameterInvalidException(f"Job with id '{job.tool_id}' is not paused") - return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + associations = self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + output_associations = [] + for association in associations: + output_associations.append(JobOutputAssociation(name=association.name, dataset=association.dataset)) + return output_associations @router.post( "/api/jobs/{id}/error", @@ -346,9 +351,13 @@ def inputs( self, id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, - ) -> List[JobAssociation]: + ) -> List[JobInputAssociation]: job = self.service.get_job(trans=trans, job_id=id) - return self.service.dictify_associations(trans, job.input_datasets, job.input_library_datasets) + associations = self.service.dictify_associations(trans, job.input_datasets, job.input_library_datasets) + input_associations = [] + for association in associations: + input_associations.append(JobInputAssociation(name=association.name, dataset=association.dataset)) + return input_associations @router.get( "/api/jobs/{id}/outputs", @@ -359,9 +368,13 @@ def outputs( self, id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, - ) -> List[JobAssociation]: + ) -> List[JobOutputAssociation]: job = self.service.get_job(trans=trans, job_id=id) - return self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + associations = self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) + output_associations = [] + for association in associations: + output_associations.append(JobOutputAssociation(name=association.name, dataset=association.dataset)) + return output_associations @router.get( "/api/jobs/{job_id}/parameters_display", From 543a7a438151a5c05d9a4a9379e457539500aeea Mon Sep 17 00:00:00 2001 From: Tillman Date: Sun, 12 Nov 2023 11:12:48 +0100 Subject: [PATCH 094/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 54 +++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 009c55c586c0..372f9c5bedbf 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -6372,22 +6372,6 @@ export interface components { * @enum {string} */ ItemsFromSrc: "url" | "files" | "path" | "ftp_import" | "server_dir"; - /** - * JobAssociation - * @description Base model definition with common configuration used by all derived models. - */ - JobAssociation: { - /** - * dataset - * @description Reference to the associated item. - */ - dataset: components["schemas"]["EncodedDataItemSourceId"]; - /** - * name - * @description Name of the job parameter. - */ - name: string; - }; /** * JobDestinationParams * @description Base model definition with common configuration used by all derived models. @@ -6585,6 +6569,22 @@ export interface components { * @enum {string} */ JobIndexViewEnum: "collection" | "admin_job_list"; + /** + * JobInputAssociation + * @description Base model definition with common configuration used by all derived models. + */ + JobInputAssociation: { + /** + * dataset + * @description Reference to the associated item. + */ + dataset: components["schemas"]["EncodedDataItemSourceId"]; + /** + * name + * @description Name of the job input parameter. + */ + name: string; + }; /** * JobInputSummary * @description Base model definition with common configuration used by all derived models. @@ -6663,6 +6663,22 @@ export interface components { */ value: components["schemas"]["EncodedDataItemSourceId"]; }; + /** + * JobOutputAssociation + * @description Base model definition with common configuration used by all derived models. + */ + JobOutputAssociation: { + /** + * dataset + * @description Reference to the associated item. + */ + dataset: components["schemas"]["EncodedDataItemSourceId"]; + /** + * name + * @description Name of the job output parameter. + */ + name: string; + }; /** * JobParameter * @description Base model definition with common configuration used by all derived models. @@ -15356,7 +15372,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobAssociation"][]; + "application/json": components["schemas"]["JobInputAssociation"][]; }; }; /** @description Validation Error */ @@ -15383,7 +15399,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobAssociation"][]; + "application/json": components["schemas"]["JobOutputAssociation"][]; }; }; /** @description Validation Error */ @@ -15410,7 +15426,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobAssociation"][]; + "application/json": components["schemas"]["JobOutputAssociation"][]; }; }; /** @description Validation Error */ From 899608c390f7d27b266839132ff81b68aa04f83c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 13 Nov 2023 15:31:00 +0100 Subject: [PATCH 095/110] Use validator to load job search inputs --- lib/galaxy/schema/jobs.py | 12 ++++++++++-- lib/galaxy/webapps/galaxy/api/jobs.py | 3 +-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 49b6103385ab..8b4c48aff142 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,3 +1,4 @@ +import json from enum import Enum from typing import ( Any, @@ -11,6 +12,7 @@ Field, Required, UUID4, + validator, ) from galaxy.schema.fields import ( @@ -104,10 +106,10 @@ class SearchJobsPayload(Model): ) # TODO the inputs are actually a dict, but are passed as a JSON dump # maybe change it? - inputs: str = Field( + inputs: Dict[str, Any] = Field( default=Required, title="Inputs", - description="The inputs of the job as a JSON dump.", + description="The inputs of the job.", ) state: JobState = Field( default=Required, @@ -115,6 +117,12 @@ class SearchJobsPayload(Model): description="Current state of the job.", ) + @validator("inputs", pre=True) + def decode_json(cls, v): + if isinstance(v, str): + return json.loads(v) + return v + class Config: extra = Extra.allow # This is used for items named file_ and __file_ diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 8e9f7e5e0674..4ef21148871b 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -4,7 +4,6 @@ .. seealso:: :class:`galaxy.model.Jobs` """ -import json import logging from datetime import ( date, @@ -492,7 +491,7 @@ def search( raise exceptions.ObjectNotFound("Requested tool not found") # TODO the inputs are actually a dict, but are passed as a JSON dump # maybe change it? - inputs = json.loads(payload.inputs) + inputs = payload.inputs # Find files coming in as multipart file data and add to inputs. for k, v in payload.__annotations__.items(): if k.startswith("files_") or k.startswith("__files_"): From d38b8db26b191e022628e93f3dd6b7602c7b7b11 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 14 Nov 2023 18:24:55 +0100 Subject: [PATCH 096/110] Remove accidental code insertion in test_jobs.py --- lib/galaxy_test/api/test_jobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 3fe9dd89dfbe..f01f4925fe5b 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -1093,4 +1093,3 @@ def __jobs_index(self, **kwds): jobs = jobs_response.json() assert isinstance(jobs, list) return jobs - return jobs From 19c06a9cf3efb7a558aa3ab05a0c6b8857dfc4f9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 14 Nov 2023 18:35:55 +0100 Subject: [PATCH 097/110] Set Ids of JobBaseModel to encoded and undo previous model split --- lib/galaxy/schema/jobs.py | 11 +---------- lib/galaxy/schema/schema.py | 13 +++++-------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 8b4c48aff142..dbe494f36571 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -156,16 +156,7 @@ class EncodedDatasetJobInfo(EncodedDataItemSourceId): ) -class EncodedJobIDs(Model): - id: EncodedDatabaseIdField = EntityIdField - history_id: Optional[EncodedDatabaseIdField] = Field( - None, - title="History ID", - description="The encoded ID of the history associated with this item.", - ) - - -class EncodedJobDetails(JobSummary, EncodedJobIDs): +class EncodedJobDetails(JobSummary): command_version: str = Field( ..., title="Command Version", diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index c726ba43829e..c408e2538be0 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1828,16 +1828,13 @@ class JobIdResponse(Model): ) -class JobIDs(Model): - id: DecodedDatabaseIdField = EntityIdField - history_id: Optional[DecodedDatabaseIdField] = Field( +class JobBaseModel(Model): + id: EncodedDatabaseIdField = EntityIdField + history_id: Optional[EncodedDatabaseIdField] = Field( None, title="History ID", description="The encoded ID of the history associated with this item.", ) - - -class JobBaseModel(Model): model_class: JOB_MODEL_CLASS = ModelClassField(JOB_MODEL_CLASS) tool_id: str = Field( ..., @@ -1864,7 +1861,7 @@ class JobBaseModel(Model): ) -class JobImportHistoryResponse(JobBaseModel, JobIDs): +class JobImportHistoryResponse(JobBaseModel): message: str = Field( ..., title="Message", @@ -1949,7 +1946,7 @@ class DatasetJobInfo(DatasetSourceId): uuid: UUID4 = UuidField -class JobDetails(JobSummary, JobIDs): +class JobDetails(JobSummary): command_version: str = Field( ..., title="Command Version", From ec606f8508fbcaf5ccc7e3837e54f0930a74575d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 14 Nov 2023 18:40:59 +0100 Subject: [PATCH 098/110] Refine EncodedHdacaSourceId model --- lib/galaxy/schema/jobs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index dbe494f36571..cc978d8afc4a 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -1,5 +1,4 @@ import json -from enum import Enum from typing import ( Any, Dict, @@ -14,12 +13,14 @@ UUID4, validator, ) +from typing_extensions import Literal from galaxy.schema.fields import ( DecodedDatabaseIdField, EncodedDatabaseIdField, ) from galaxy.schema.schema import ( + DataItemSourceType, EncodedDataItemSourceId, EntityIdField, JobState, @@ -134,19 +135,17 @@ class DeleteJobPayload(Model): description="Stop message", ) +class SrcItem(Model): + src: DataItemSourceType -class EncodedHdcaSourceId(Model): - class Hdca(str, Enum): - hdca = "hdca" - +class EncodedHdcaSourceId(SrcItem): id: EncodedDatabaseIdField = EntityIdField - src: Hdca = Field( + src: Literal[DataItemSourceType.hdca] = Field( default=Required, title="Source", description="The source of this dataset, which in the case of the model can only be `hdca`.", ) - class EncodedDatasetJobInfo(EncodedDataItemSourceId): uuid: Optional[UUID4] = Field( default=None, From 140539157a4dadb6d0e46ed42ec9a1355ed78f3d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 14 Nov 2023 18:53:28 +0100 Subject: [PATCH 099/110] Regenerate client schema --- client/src/api/schema/schema.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 372f9c5bedbf..1cef5fd2d101 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4235,8 +4235,9 @@ export interface components { /** * Source * @description The source of this dataset, which in the case of the model can only be `hdca`. + * @enum {string} */ - src: components["schemas"]["Hdca"]; + src: "hdca"; }; /** * EncodedHistoryContentItem @@ -5736,12 +5737,6 @@ export interface components { */ type: "hdas"; }; - /** - * Hdca - * @description An enumeration. - * @enum {string} - */ - Hdca: "hdca"; /** * HdcaDataItemsFromTarget * @description Base model definition with common configuration used by all derived models. @@ -8684,9 +8679,9 @@ export interface components { SearchJobsPayload: { /** * Inputs - * @description The inputs of the job as a JSON dump. + * @description The inputs of the job. */ - inputs: string; + inputs: Record; /** * State * @description Current state of the job. From e4824b292b51a7396b57c9f808d7b6b892d9e934 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 14 Nov 2023 18:55:11 +0100 Subject: [PATCH 100/110] Fix codestyle issue --- lib/galaxy/schema/jobs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index cc978d8afc4a..34d1ad41fea6 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -135,9 +135,11 @@ class DeleteJobPayload(Model): description="Stop message", ) + class SrcItem(Model): src: DataItemSourceType + class EncodedHdcaSourceId(SrcItem): id: EncodedDatabaseIdField = EntityIdField src: Literal[DataItemSourceType.hdca] = Field( @@ -146,6 +148,7 @@ class EncodedHdcaSourceId(SrcItem): description="The source of this dataset, which in the case of the model can only be `hdca`.", ) + class EncodedDatasetJobInfo(EncodedDataItemSourceId): uuid: Optional[UUID4] = Field( default=None, From 66001b7919a61bbbcfb05755ce1ac02226c915eb Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 14 Nov 2023 19:46:50 +0100 Subject: [PATCH 101/110] Remove completed TODOs and clarify some path parameters --- lib/galaxy/webapps/galaxy/api/jobs.py | 33 ++++++++++++--------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 4ef21148871b..dc1f7fb893ba 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -195,7 +195,6 @@ ReportErrorBody = Body(default=Required, title="Report error", description="The values to report an Error") SearchJobBody = Body(default=Required, title="Search job", description="The values to search an Job") -# TODO The endpoint only stops/cancels a job, but is called delete settle for one name DeleteJobBody = Body(title="Delete/cancel job", description="The values to delete/cancel a job") @@ -256,16 +255,16 @@ def create( raise HTTPException(status_code=501, detail="Please POST to /api/tools instead.") @router.get( - "/api/jobs/{id}/common_problems", + "/api/jobs/{job_id}/common_problems", name="check_common_problems", summary="Check inputs and job for common potential problems to aid in error reporting", ) def common_problems( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, ) -> JobInputSummary: - job = self.service.get_job(trans=trans, job_id=id) + job = self.service.get_job(trans=trans, job_id=job_id) seen_ids = set() has_empty_inputs = False has_duplicate_inputs = False @@ -286,16 +285,16 @@ def common_problems( return JobInputSummary(has_empty_inputs=has_empty_inputs, has_duplicate_inputs=has_duplicate_inputs) @router.put( - "/api/jobs/{id}/resume", + "/api/jobs/{job_id}/resume", name="resume_paused_job", summary="Resumes a paused job.", ) def resume( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, ) -> List[JobOutputAssociation]: - job = self.service.get_job(trans, job_id=id) + job = self.service.get_job(trans, job_id=job_id) if not job: raise exceptions.ObjectNotFound("Could not access job with the given id") if job.state == job.states.PAUSED: @@ -309,21 +308,21 @@ def resume( return output_associations @router.post( - "/api/jobs/{id}/error", + "/api/jobs/{job_id}/error", name="report_error", summary="Submits a bug report via the API.", ) def error( self, payload: Annotated[ReportJobErrorPayload, ReportErrorBody], - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, ) -> JobErrorSummary: # Get dataset on which this error was triggered dataset_id = payload.dataset_id dataset = self.service.hda_manager.get_accessible(id=dataset_id, user=trans.user) # Get job - job = self.service.get_job(trans, id) + job = self.service.get_job(trans, job_id) if dataset.creating_job.id != job.id: raise exceptions.RequestParameterInvalidException("dataset_id was not created by job_id") tool = trans.app.toolbox.get_tool(job.tool_id, tool_version=job.tool_version) or None @@ -342,16 +341,16 @@ def error( return JobErrorSummary(messages=messages) @router.get( - "/api/jobs/{id}/inputs", + "/api/jobs/{job_id}/inputs", name="get_inputs", summary="Returns input datasets created by a job.", ) def inputs( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, ) -> List[JobInputAssociation]: - job = self.service.get_job(trans=trans, job_id=id) + job = self.service.get_job(trans=trans, job_id=job_id) associations = self.service.dictify_associations(trans, job.input_datasets, job.input_library_datasets) input_associations = [] for association in associations: @@ -359,16 +358,16 @@ def inputs( return input_associations @router.get( - "/api/jobs/{id}/outputs", + "/api/jobs/{job_id}/outputs", name="get_outputs", summary="Returns output datasets created by a job.", ) def outputs( self, - id: Annotated[DecodedDatabaseIdField, JobIdPathParam], + job_id: Annotated[DecodedDatabaseIdField, JobIdPathParam], trans: ProvidesUserContext = DependsOnTrans, ) -> List[JobOutputAssociation]: - job = self.service.get_job(trans=trans, job_id=id) + job = self.service.get_job(trans=trans, job_id=job_id) associations = self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) output_associations = [] for association in associations: @@ -489,8 +488,6 @@ def search( tool = trans.app.toolbox.get_tool(tool_id) if tool is None: raise exceptions.ObjectNotFound("Requested tool not found") - # TODO the inputs are actually a dict, but are passed as a JSON dump - # maybe change it? inputs = payload.inputs # Find files coming in as multipart file data and add to inputs. for k, v in payload.__annotations__.items(): From a8a4b91710717750c9a30fbb0aa79310109ad1b2 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 18 Nov 2023 11:11:34 +0100 Subject: [PATCH 102/110] Remove outdated TODO --- lib/galaxy/schema/jobs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index 34d1ad41fea6..b049bc07af89 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -105,8 +105,6 @@ class SearchJobsPayload(Model): title="Tool ID", description="The tool ID related to the job.", ) - # TODO the inputs are actually a dict, but are passed as a JSON dump - # maybe change it? inputs: Dict[str, Any] = Field( default=Required, title="Inputs", From 832ecd8948a8172192285f0be81c15eb64fd8ad4 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 18 Nov 2023 11:11:59 +0100 Subject: [PATCH 103/110] Add test for delete operation --- lib/galaxy_test/api/test_jobs.py | 33 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index f01f4925fe5b..af77499417d7 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -960,22 +960,23 @@ def test_get_inputs_and_outputs(self, history_id): assert job_first_output.get("dataset").get("id") == job_first_output_values.get("id") assert job_first_output.get("dataset").get("src") == job_first_output_values.get("src") - # @pytest.mark.require_new_history - # def test_delete_job(self, history_id): - # dataset_id = self.__history_with_ok_dataset(history_id) - # inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) - # search_response = self._create_and_search_job(history_id, inputs, tool_id="cat1") - # job_id = search_response.json()[0]["id"] - # # delete the job without message - # delete_job_response = self._delete(f"jobs/{job_id}") - # self._assert_status_code_is(delete_job_response, 200) - # # TODO the job is finished as such it is not stopped Think of another way to test it - # assert delete_job_response.json() == True - # # now that we deleted the job we should not find it anymore - # search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) - # empty_search_response = self._post("jobs/search", data=search_payload, json=True) - # self._assert_status_code_is(empty_search_response, 200) - # assert len(empty_search_response.json()) == 0 + @pytest.mark.require_new_history + def test_delete_job(self, history_id): + dataset_id = self.__history_with_ok_dataset(history_id) + inputs = json.dumps({"input1": {"src": "hda", "id": dataset_id}}) + search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) + # create a job + tool_response = self._post("tools", data=search_payload) + job_id = tool_response.json()["jobs"][0]["id"] + # delete the job without message + delete_job_response = self._delete(f"jobs/{job_id}") + self._assert_status_code_is(delete_job_response, 200) + assert delete_job_response.json() is True + # now that we deleted the job we should not find it anymore + search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) + empty_search_response = self._post("jobs/search", data=search_payload, json=True) + self._assert_status_code_is(empty_search_response, 200) + assert len(empty_search_response.json()) == 0 @pytest.mark.require_new_history def test_destination_params(self, history_id): From c2ca0a6e42427f38db3873dffa940157114554a2 Mon Sep 17 00:00:00 2001 From: Tillman Date: Sat, 18 Nov 2023 11:13:00 +0100 Subject: [PATCH 104/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 200 ++++++++++++++++---------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 1cef5fd2d101..1fd19f5d08c5 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -929,36 +929,28 @@ export interface paths { */ post: operations["search_jobs_api_jobs_search_post"]; }; - "/api/jobs/{id}/common_problems": { - /** Check inputs and job for common potential problems to aid in error reporting */ - get: operations["check_common_problems_api_jobs__id__common_problems_get"]; - }; - "/api/jobs/{id}/error": { - /** Submits a bug report via the API. */ - post: operations["report_error_api_jobs__id__error_post"]; - }; - "/api/jobs/{id}/inputs": { - /** Returns input datasets created by a job. */ - get: operations["get_inputs_api_jobs__id__inputs_get"]; - }; - "/api/jobs/{id}/outputs": { - /** Returns output datasets created by a job. */ - get: operations["get_outputs_api_jobs__id__outputs_get"]; - }; - "/api/jobs/{id}/resume": { - /** Resumes a paused job. */ - put: operations["resume_paused_job_api_jobs__id__resume_put"]; - }; "/api/jobs/{job_id}": { /** Return dictionary containing description of job data. */ get: operations["show_job_api_jobs__job_id__get"]; /** Cancels specified job */ delete: operations["cancel_job_api_jobs__job_id__delete"]; }; + "/api/jobs/{job_id}/common_problems": { + /** Check inputs and job for common potential problems to aid in error reporting */ + get: operations["check_common_problems_api_jobs__job_id__common_problems_get"]; + }; "/api/jobs/{job_id}/destination_params": { /** Return destination parameters for specified job. */ get: operations["destination_params_job_api_jobs__job_id__destination_params_get"]; }; + "/api/jobs/{job_id}/error": { + /** Submits a bug report via the API. */ + post: operations["report_error_api_jobs__job_id__error_post"]; + }; + "/api/jobs/{job_id}/inputs": { + /** Returns input datasets created by a job. */ + get: operations["get_inputs_api_jobs__job_id__inputs_get"]; + }; "/api/jobs/{job_id}/metrics": { /** Return job metrics for specified job. */ get: operations["get_metrics_api_jobs__job_id__metrics_get"]; @@ -970,6 +962,10 @@ export interface paths { */ get: operations["get_token_api_jobs__job_id__oidc_tokens_get"]; }; + "/api/jobs/{job_id}/outputs": { + /** Returns output datasets created by a job. */ + get: operations["get_outputs_api_jobs__job_id__outputs_get"]; + }; "/api/jobs/{job_id}/parameters_display": { /** * Resolve parameters as a list for nested display. @@ -984,6 +980,10 @@ export interface paths { */ get: operations["resolve_parameters_display_api_jobs__job_id__parameters_display_get"]; }; + "/api/jobs/{job_id}/resume": { + /** Resumes a paused job. */ + put: operations["resume_paused_job_api_jobs__job_id__resume_put"]; + }; "/api/libraries": { /** * Returns a list of summary data for all libraries. @@ -15292,23 +15292,27 @@ export interface operations { }; }; }; - check_common_problems_api_jobs__id__common_problems_get: { - /** Check inputs and job for common potential problems to aid in error reporting */ + show_job_api_jobs__job_id__get: { + /** Return dictionary containing description of job data. */ parameters: { + /** @description Show extra information. */ + query?: { + full?: boolean; + }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobInputSummary"]; + "application/json": Record; }; }; /** @description Validation Error */ @@ -15319,8 +15323,8 @@ export interface operations { }; }; }; - report_error_api_jobs__id__error_post: { - /** Submits a bug report via the API. */ + cancel_job_api_jobs__job_id__delete: { + /** Cancels specified job */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15328,19 +15332,19 @@ export interface operations { }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; - requestBody: { + requestBody?: { content: { - "application/json": components["schemas"]["ReportJobErrorPayload"]; + "application/json": components["schemas"]["DeleteJobPayload"]; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobErrorSummary"]; + "application/json": boolean; }; }; /** @description Validation Error */ @@ -15351,8 +15355,8 @@ export interface operations { }; }; }; - get_inputs_api_jobs__id__inputs_get: { - /** Returns input datasets created by a job. */ + check_common_problems_api_jobs__job_id__common_problems_get: { + /** Check inputs and job for common potential problems to aid in error reporting */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15360,14 +15364,14 @@ export interface operations { }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobInputAssociation"][]; + "application/json": components["schemas"]["JobInputSummary"]; }; }; /** @description Validation Error */ @@ -15378,8 +15382,8 @@ export interface operations { }; }; }; - get_outputs_api_jobs__id__outputs_get: { - /** Returns output datasets created by a job. */ + destination_params_job_api_jobs__job_id__destination_params_get: { + /** Return destination parameters for specified job. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15387,14 +15391,14 @@ export interface operations { }; /** @description The ID of the job */ path: { - id: string; + job_id: string; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobOutputAssociation"][]; + "application/json": components["schemas"]["JobDestinationParams"]; }; }; /** @description Validation Error */ @@ -15405,8 +15409,8 @@ export interface operations { }; }; }; - resume_paused_job_api_jobs__id__resume_put: { - /** Resumes a paused job. */ + report_error_api_jobs__job_id__error_post: { + /** Submits a bug report via the API. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -15414,14 +15418,19 @@ export interface operations { }; /** @description The ID of the job */ path: { - id: string; + job_id: string; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReportJobErrorPayload"]; }; }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobOutputAssociation"][]; + "application/json": components["schemas"]["JobErrorSummary"]; }; }; /** @description Validation Error */ @@ -15432,13 +15441,9 @@ export interface operations { }; }; }; - show_job_api_jobs__job_id__get: { - /** Return dictionary containing description of job data. */ + get_inputs_api_jobs__job_id__inputs_get: { + /** Returns input datasets created by a job. */ parameters: { - /** @description Show extra information. */ - query?: { - full?: boolean; - }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; @@ -15452,7 +15457,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["JobInputAssociation"][]; }; }; /** @description Validation Error */ @@ -15463,9 +15468,16 @@ export interface operations { }; }; }; - cancel_job_api_jobs__job_id__delete: { - /** Cancels specified job */ + get_metrics_api_jobs__job_id__metrics_get: { + /** Return job metrics for specified job. */ parameters: { + /** + * @deprecated + * @description Whether this dataset belongs to a history (HDA) or a library (LDDA). + */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; + }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; @@ -15475,16 +15487,11 @@ export interface operations { job_id: string; }; }; - requestBody?: { - content: { - "application/json": components["schemas"]["DeleteJobPayload"]; - }; - }; responses: { /** @description Successful Response */ 200: { content: { - "application/json": boolean; + "application/json": components["schemas"]["JobMetric"][]; }; }; /** @description Validation Error */ @@ -15495,14 +15502,22 @@ export interface operations { }; }; }; - destination_params_job_api_jobs__job_id__destination_params_get: { - /** Return destination parameters for specified job. */ + get_token_api_jobs__job_id__oidc_tokens_get: { + /** + * Get a fresh OIDC token + * @description Allows remote job running mechanisms to get a fresh OIDC token that can be used on remote side to authorize user. It is not meant to represent part of Galaxy's stable, user facing API + */ parameters: { + /** @description A key used to authenticate this request as acting onbehalf or a job runner for the specified job */ + /** @description OIDC provider name */ + query: { + job_key: string; + provider: string; + }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; - /** @description The ID of the job */ path: { job_id: string; }; @@ -15511,7 +15526,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobDestinationParams"]; + "text/plain": string; }; }; /** @description Validation Error */ @@ -15522,16 +15537,9 @@ export interface operations { }; }; }; - get_metrics_api_jobs__job_id__metrics_get: { - /** Return job metrics for specified job. */ + get_outputs_api_jobs__job_id__outputs_get: { + /** Returns output datasets created by a job. */ parameters: { - /** - * @deprecated - * @description Whether this dataset belongs to a history (HDA) or a library (LDDA). - */ - query?: { - hda_ldda?: components["schemas"]["DatasetSourceType"]; - }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; @@ -15545,7 +15553,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobMetric"][]; + "application/json": components["schemas"]["JobOutputAssociation"][]; }; }; /** @description Validation Error */ @@ -15556,22 +15564,31 @@ export interface operations { }; }; }; - get_token_api_jobs__job_id__oidc_tokens_get: { + resolve_parameters_display_api_jobs__job_id__parameters_display_get: { /** - * Get a fresh OIDC token - * @description Allows remote job running mechanisms to get a fresh OIDC token that can be used on remote side to authorize user. It is not meant to represent part of Galaxy's stable, user facing API + * Resolve parameters as a list for nested display. + * @description Resolve parameters as a list for nested display. More client logic + * here than is ideal but it is hard to reason about tool parameter + * types on the client relative to the server. Job accessibility checks + * are slightly different than dataset checks, so both methods are + * available. + * + * This API endpoint is unstable and tied heavily to Galaxy's JS client code, + * this endpoint will change frequently. */ parameters: { - /** @description A key used to authenticate this request as acting onbehalf or a job runner for the specified job */ - /** @description OIDC provider name */ - query: { - job_key: string; - provider: string; + /** + * @deprecated + * @description Whether this dataset belongs to a history (HDA) or a library (LDDA). + */ + query?: { + hda_ldda?: components["schemas"]["DatasetSourceType"]; }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; }; + /** @description The ID of the job */ path: { job_id: string; }; @@ -15580,7 +15597,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "text/plain": string; + "application/json": components["schemas"]["JobDisplayParametersSummary"]; }; }; /** @description Validation Error */ @@ -15591,26 +15608,9 @@ export interface operations { }; }; }; - resolve_parameters_display_api_jobs__job_id__parameters_display_get: { - /** - * Resolve parameters as a list for nested display. - * @description Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * - * This API endpoint is unstable and tied heavily to Galaxy's JS client code, - * this endpoint will change frequently. - */ + resume_paused_job_api_jobs__job_id__resume_put: { + /** Resumes a paused job. */ parameters: { - /** - * @deprecated - * @description Whether this dataset belongs to a history (HDA) or a library (LDDA). - */ - query?: { - hda_ldda?: components["schemas"]["DatasetSourceType"]; - }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string; @@ -15624,7 +15624,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["JobDisplayParametersSummary"]; + "application/json": components["schemas"]["JobOutputAssociation"][]; }; }; /** @description Validation Error */ From d9255270a9bbb5e08a9215b7259585d565f5be98 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 20 Nov 2023 13:01:42 +0100 Subject: [PATCH 105/110] Add expetion in case of missing parameter --- lib/galaxy/webapps/galaxy/services/jobs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index c0fea605da2e..95ceb3ede2b3 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -117,8 +117,10 @@ def get_job( dataset_instance = self.hda_manager.get_accessible(id=dataset_id, user=trans.user) else: dataset_instance = self.hda_manager.ldda_manager.get(trans, id=dataset_id) - # TODO Raise error if no ID passed? Never happens when called from Job API endpoints - return dataset_instance.creating_job + return dataset_instance.creating_job + else: + # Raise an exception if neither job_id nor dataset_id is provided + raise ValueError("Either job_id or dataset_id must be provided.") def dictify_associations(self, trans, *association_lists) -> List[JobAssociation]: rval: List[JobAssociation] = [] From 78fc9127cf6f49e27ec2ec1deab6f8cb6e5cc805 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 20 Nov 2023 13:11:49 +0100 Subject: [PATCH 106/110] Refine the JobDestinationParams model --- lib/galaxy/schema/jobs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index b049bc07af89..a69943343cdd 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -187,15 +187,15 @@ class EncodedJobDetails(JobSummary): class JobDestinationParams(Model): - runner: str = Field(default=Required, title="Runner", description="Job runner class", alias="Runner") - runner_job_id: str = Field( - default=Required, + runner: Optional[str] = Field(None, title="Runner", description="Job runner class", alias="Runner") + runner_job_id: Optional[str] = Field( + None, title="Runner Job ID", description="ID assigned to submitted job by external job running system", alias="Runner Job ID", ) - handler: str = Field( - default=Required, title="Handler", description="Name of the process that handled the job.", alias="Handler" + handler: Optional[str] = Field( + None, title="Handler", description="Name of the process that handled the job.", alias="Handler" ) From 05edd51c72f848e5ca0c6736cd4af132f28488c9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 20 Nov 2023 13:29:38 +0100 Subject: [PATCH 107/110] Regenerate the client schema --- client/src/api/schema/schema.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 1fd19f5d08c5..e30b9bad16f2 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -6376,17 +6376,17 @@ export interface components { * Handler * @description Name of the process that handled the job. */ - Handler: string; + Handler?: string; /** * Runner * @description Job runner class */ - Runner: string; + Runner?: string; /** * Runner Job ID * @description ID assigned to submitted job by external job running system */ - "Runner Job ID": string; + "Runner Job ID"?: string; }; /** * JobDisplayParametersSummary From c0d43cf120ad939a9a46b09a072ac48e5340ae9f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Nov 2023 18:06:26 +0100 Subject: [PATCH 108/110] Drop POST /api/jobs This is a remnant from the old routes setup that would automatically create thr CRUD methods ... much better to just drop this. --- client/src/api/schema/schema.ts | 34 --------------------------- lib/galaxy/webapps/galaxy/api/jobs.py | 14 ----------- 2 files changed, 48 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index e30b9bad16f2..62f8142fa725 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -914,11 +914,6 @@ export interface paths { "/api/jobs": { /** Index */ get: operations["index_api_jobs_get"]; - /** - * Not implemented. - * @description See the create method in tools.py in order to submit a job. - */ - post: operations["create_job_api_jobs_post"]; }; "/api/jobs/search": { /** @@ -15230,35 +15225,6 @@ export interface operations { }; }; }; - create_job_api_jobs_post: { - /** - * Not implemented. - * @description See the create method in tools.py in order to submit a job. - */ - parameters: { - query: { - kwd: Record; - }; - /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ - header?: { - "run-as"?: string; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - content: { - "application/json": Record; - }; - }; - /** @description Validation Error */ - 422: { - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; search_jobs_api_jobs_search_post: { /** * Return jobs for current user diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index dc1f7fb893ba..50e829fd5ee6 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -20,7 +20,6 @@ from fastapi import ( Body, Depends, - HTTPException, Path, Query, ) @@ -241,19 +240,6 @@ def index( ) return self.service.index(trans, payload) - @router.post( - "/api/jobs", - name="create_job", - summary="Not implemented.", - ) - def create( - self, - trans: ProvidesUserContext = DependsOnTrans, - **kwd, - ): - """See the create method in tools.py in order to submit a job.""" - raise HTTPException(status_code=501, detail="Please POST to /api/tools instead.") - @router.get( "/api/jobs/{job_id}/common_problems", name="check_common_problems", From f18a33b4a1321d996597d6428176ff5186feb153 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Nov 2023 19:03:50 +0100 Subject: [PATCH 109/110] Update description of '/api/jobs/{job_id}/parameters_display' --- client/src/api/schema/schema.ts | 28 ++++----------------------- lib/galaxy/managers/jobs.py | 2 ++ lib/galaxy/security/idencoding.py | 4 +++- lib/galaxy/webapps/galaxy/api/jobs.py | 14 ++------------ 4 files changed, 11 insertions(+), 37 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 62f8142fa725..525387aee7e1 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -178,12 +178,7 @@ export interface paths { /** * Resolve parameters as a list for nested display. * @deprecated - * @description Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * + * @description Resolve parameters as a list for nested display. * This API endpoint is unstable and tied heavily to Galaxy's JS client code, * this endpoint will change frequently. */ @@ -964,12 +959,7 @@ export interface paths { "/api/jobs/{job_id}/parameters_display": { /** * Resolve parameters as a list for nested display. - * @description Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * + * @description Resolve parameters as a list for nested display. * This API endpoint is unstable and tied heavily to Galaxy's JS client code, * this endpoint will change frequently. */ @@ -11055,12 +11045,7 @@ export interface operations { /** * Resolve parameters as a list for nested display. * @deprecated - * @description Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * + * @description Resolve parameters as a list for nested display. * This API endpoint is unstable and tied heavily to Galaxy's JS client code, * this endpoint will change frequently. */ @@ -15533,12 +15518,7 @@ export interface operations { resolve_parameters_display_api_jobs__job_id__parameters_display_get: { /** * Resolve parameters as a list for nested display. - * @description Resolve parameters as a list for nested display. More client logic - * here than is ideal but it is hard to reason about tool parameter - * types on the client relative to the server. Job accessibility checks - * are slightly different than dataset checks, so both methods are - * available. - * + * @description Resolve parameters as a list for nested display. * This API endpoint is unstable and tied heavily to Galaxy's JS client code, * this endpoint will change frequently. */ diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index fcb09f1b9a84..a1ac6728591a 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -880,6 +880,8 @@ def summarize_job_parameters(trans, job): Precondition: the caller has verified the job is accessible to the user represented by the trans parameter. """ + # More client logic here than is ideal but it is hard to reason about + # tool parameter types on the client relative to the server. def inputs_recursive(input_params, param_values, depth=1, upgrade_messages=None): if upgrade_messages is None: diff --git a/lib/galaxy/security/idencoding.py b/lib/galaxy/security/idencoding.py index 9c0b10ce2a20..e1eca5d2a314 100644 --- a/lib/galaxy/security/idencoding.py +++ b/lib/galaxy/security/idencoding.py @@ -35,9 +35,11 @@ def __init__(self, **config): per_kind_id_secret_base = config.get("per_kind_id_secret_base", self.id_secret) self.id_ciphers_for_kind = _cipher_cache(per_kind_id_secret_base) - def encode_id(self, obj_id, kind=None): + def encode_id(self, obj_id, kind=None, strict_integer=False): if obj_id is None: raise galaxy.exceptions.MalformedId("Attempted to encode None id") + if strict_integer and not isinstance(obj_id, int): + raise galaxy.exceptions.MalformedId("Attempted to encode id that is not an integer") id_cipher = self.__id_cipher(kind) # Convert to bytes s = smart_str(obj_id) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 50e829fd5ee6..bd19370dd6ae 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -372,12 +372,7 @@ def parameters_display_by_job( trans: ProvidesUserContext = DependsOnTrans, ) -> JobDisplayParametersSummary: """ - Resolve parameters as a list for nested display. More client logic - here than is ideal but it is hard to reason about tool parameter - types on the client relative to the server. Job accessibility checks - are slightly different than dataset checks, so both methods are - available. - + Resolve parameters as a list for nested display. This API endpoint is unstable and tied heavily to Galaxy's JS client code, this endpoint will change frequently. """ @@ -398,12 +393,7 @@ def parameters_display_by_dataset( trans: ProvidesUserContext = DependsOnTrans, ) -> JobDisplayParametersSummary: """ - Resolve parameters as a list for nested display. More client logic - here than is ideal but it is hard to reason about tool parameter - types on the client relative to the server. Job accessibility checks - are slightly different than dataset checks, so both methods are - available. - + Resolve parameters as a list for nested display. This API endpoint is unstable and tied heavily to Galaxy's JS client code, this endpoint will change frequently. """ From 4c4f949c0ed752656fb567ad0cde74efa51cc015 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Nov 2023 19:07:45 +0100 Subject: [PATCH 110/110] Drop BaseModel docstring --- client/src/api/schema/schema.ts | 832 +++++++------------------------- lib/galaxy/schema/schema.py | 2 - 2 files changed, 165 insertions(+), 669 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 525387aee7e1..d947c7b1f9c7 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -1787,10 +1787,7 @@ export interface components { */ link: string; }; - /** - * AnonUserModel - * @description Base model definition with common configuration used by all derived models. - */ + /** AnonUserModel */ AnonUserModel: { /** * Nice total disc usage @@ -1808,10 +1805,7 @@ export interface components { */ total_disk_usage: number; }; - /** - * ArchiveHistoryRequestPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ArchiveHistoryRequestPayload */ ArchiveHistoryRequestPayload: { /** * Export Record ID @@ -2042,10 +2036,7 @@ export interface components { */ url: string; }; - /** - * AsyncFile - * @description Base model definition with common configuration used by all derived models. - */ + /** AsyncFile */ AsyncFile: { /** * Storage Request Id @@ -2054,10 +2045,7 @@ export interface components { storage_request_id: string; task: components["schemas"]["AsyncTaskResultSummary"]; }; - /** - * AsyncTaskResultSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** AsyncTaskResultSummary */ AsyncTaskResultSummary: { /** * ID @@ -2098,10 +2086,7 @@ export interface components { ) | ("cloud" | "quota" | "no_quota" | "restricted"); }; - /** - * BasicRoleModel - * @description Base model definition with common configuration used by all derived models. - */ + /** BasicRoleModel */ BasicRoleModel: { /** * ID @@ -2150,10 +2135,7 @@ export interface components { /** Targets */ targets: Record; }; - /** - * BroadcastNotificationContent - * @description Base model definition with common configuration used by all derived models. - */ + /** BroadcastNotificationContent */ BroadcastNotificationContent: { /** * Action links @@ -2274,10 +2256,7 @@ export interface components { */ variant: components["schemas"]["NotificationVariant"]; }; - /** - * BrowsableFilesSourcePlugin - * @description Base model definition with common configuration used by all derived models. - */ + /** BrowsableFilesSourcePlugin */ BrowsableFilesSourcePlugin: { /** * Browsable @@ -2331,19 +2310,13 @@ export interface components { */ writable: boolean; }; - /** - * BulkOperationItemError - * @description Base model definition with common configuration used by all derived models. - */ + /** BulkOperationItemError */ BulkOperationItemError: { /** Error */ error: string; item: components["schemas"]["EncodedHistoryContentItem"]; }; - /** - * ChangeDatatypeOperationParams - * @description Base model definition with common configuration used by all derived models. - */ + /** ChangeDatatypeOperationParams */ ChangeDatatypeOperationParams: { /** Datatype */ datatype: string; @@ -2353,10 +2326,7 @@ export interface components { */ type: "change_datatype"; }; - /** - * ChangeDbkeyOperationParams - * @description Base model definition with common configuration used by all derived models. - */ + /** ChangeDbkeyOperationParams */ ChangeDbkeyOperationParams: { /** Dbkey */ dbkey: string; @@ -2366,10 +2336,7 @@ export interface components { */ type: "change_dbkey"; }; - /** - * CheckForUpdatesResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** CheckForUpdatesResponse */ CheckForUpdatesResponse: { /** * Message @@ -2399,10 +2366,7 @@ export interface components { */ type: string; }; - /** - * CleanableItemsSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** CleanableItemsSummary */ CleanableItemsSummary: { /** * Total Items @@ -2415,18 +2379,12 @@ export interface components { */ total_size: number; }; - /** - * CleanupStorageItemsRequest - * @description Base model definition with common configuration used by all derived models. - */ + /** CleanupStorageItemsRequest */ CleanupStorageItemsRequest: { /** Item Ids */ item_ids: string[]; }; - /** - * CloudDatasets - * @description Base model definition with common configuration used by all derived models. - */ + /** CloudDatasets */ CloudDatasets: { /** * Authentication ID @@ -2457,10 +2415,7 @@ export interface components { */ overwrite_existing?: boolean; }; - /** - * CloudDatasetsResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** CloudDatasetsResponse */ CloudDatasetsResponse: { /** * Bucket @@ -2478,10 +2433,7 @@ export interface components { */ sent_dataset_labels: string[]; }; - /** - * CloudObjects - * @description Base model definition with common configuration used by all derived models. - */ + /** CloudObjects */ CloudObjects: { /** * Authentication ID @@ -2511,10 +2463,7 @@ export interface components { */ objects: string[]; }; - /** - * CollectionElementIdentifier - * @description Base model definition with common configuration used by all derived models. - */ + /** CollectionElementIdentifier */ CollectionElementIdentifier: { /** * Collection Type @@ -2554,10 +2503,7 @@ export interface components { * @enum {string} */ ColletionSourceType: "hda" | "ldda" | "hdca" | "new_collection"; - /** - * CompositeDataElement - * @description Base model definition with common configuration used by all derived models. - */ + /** CompositeDataElement */ CompositeDataElement: { /** Md5 */ MD5?: string; @@ -2642,10 +2588,7 @@ export interface components { /** To posix lines */ to_posix_lines: boolean; }; - /** - * CompositeItems - * @description Base model definition with common configuration used by all derived models. - */ + /** CompositeItems */ CompositeItems: { /** Elements */ elements: ( @@ -2657,10 +2600,7 @@ export interface components { | components["schemas"]["FtpImportElement"] )[]; }; - /** - * ComputeDatasetHashPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ComputeDatasetHashPayload */ ComputeDatasetHashPayload: { /** * Extra Files Path @@ -2721,10 +2661,7 @@ export interface components { ConvertedDatasetsMap: { [key: string]: string | undefined; }; - /** - * CreateEntryPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateEntryPayload */ CreateEntryPayload: { /** * Name @@ -2738,10 +2675,7 @@ export interface components { */ target: string; }; - /** - * CreateHistoryContentFromStore - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateHistoryContentFromStore */ CreateHistoryContentFromStore: { model_store_format?: components["schemas"]["ModelStoreFormat"]; /** Store Content Uri */ @@ -2749,10 +2683,7 @@ export interface components { /** Store Dict */ store_dict?: Record; }; - /** - * CreateHistoryContentPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateHistoryContentPayload */ CreateHistoryContentPayload: { /** * Collection Type @@ -2826,10 +2757,7 @@ export interface components { */ type?: components["schemas"]["HistoryContentType"]; }; - /** - * CreateHistoryFromStore - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateHistoryFromStore */ CreateHistoryFromStore: { model_store_format?: components["schemas"]["ModelStoreFormat"]; /** Store Content Uri */ @@ -2837,10 +2765,7 @@ export interface components { /** Store Dict */ store_dict?: Record; }; - /** - * CreateLibrariesFromStore - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateLibrariesFromStore */ CreateLibrariesFromStore: { model_store_format?: components["schemas"]["ModelStoreFormat"]; /** Store Content Uri */ @@ -2848,10 +2773,7 @@ export interface components { /** Store Dict */ store_dict?: Record; }; - /** - * CreateLibraryFilePayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateLibraryFilePayload */ CreateLibraryFilePayload: { /** * From HDA ID @@ -2872,10 +2794,7 @@ export interface components { */ ldda_message?: string; }; - /** - * CreateLibraryFolderPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateLibraryFolderPayload */ CreateLibraryFolderPayload: { /** * Description @@ -2889,10 +2808,7 @@ export interface components { */ name: string; }; - /** - * CreateLibraryPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateLibraryPayload */ CreateLibraryPayload: { /** * Description @@ -2928,10 +2844,7 @@ export interface components { */ metrics?: components["schemas"]["Metric"][]; }; - /** - * CreateNewCollectionPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateNewCollectionPayload */ CreateNewCollectionPayload: { /** * Collection Type @@ -2980,10 +2893,7 @@ export interface components { */ name?: string; }; - /** - * CreatePagePayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CreatePagePayload */ CreatePagePayload: { /** * Annotation @@ -3019,10 +2929,7 @@ export interface components { */ title: string; }; - /** - * CreateQuotaParams - * @description Base model definition with common configuration used by all derived models. - */ + /** CreateQuotaParams */ CreateQuotaParams: { /** * Amount @@ -3109,10 +3016,7 @@ export interface components { */ url: string; }; - /** - * CreatedEntryResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** CreatedEntryResponse */ CreatedEntryResponse: { /** * External link @@ -3191,10 +3095,7 @@ export interface components { */ username: string; }; - /** - * CustomBuildCreationPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** CustomBuildCreationPayload */ CustomBuildCreationPayload: { /** * Length type @@ -3218,10 +3119,7 @@ export interface components { * @enum {string} */ CustomBuildLenType: "file" | "fasta" | "text"; - /** - * CustomBuildModel - * @description Base model definition with common configuration used by all derived models. - */ + /** CustomBuildModel */ CustomBuildModel: { /** * Count @@ -3262,10 +3160,7 @@ export interface components { * @description The custom builds associated with the user. */ CustomBuildsCollection: components["schemas"]["CustomBuildModel"][]; - /** - * CustomBuildsMetadataResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** CustomBuildsMetadataResponse */ CustomBuildsMetadataResponse: { /** * Fasta HDAs @@ -3381,10 +3276,7 @@ export interface components { */ populated?: boolean; }; - /** - * DataElementsFromTarget - * @description Base model definition with common configuration used by all derived models. - */ + /** DataElementsFromTarget */ DataElementsFromTarget: { /** * Auto Decompress @@ -3408,10 +3300,7 @@ export interface components { /** Url */ url?: string; }; - /** - * DataElementsTarget - * @description Base model definition with common configuration used by all derived models. - */ + /** DataElementsTarget */ DataElementsTarget: { /** * Auto Decompress @@ -3444,10 +3333,7 @@ export interface components { * @enum {string} */ DataItemSourceType: "hda" | "ldda" | "hdca" | "dce" | "dc"; - /** - * DatasetAssociationRoles - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetAssociationRoles */ DatasetAssociationRoles: { /** * Access Roles @@ -3468,10 +3354,7 @@ export interface components { */ modify_item_roles?: string[][]; }; - /** - * DatasetCollectionAttributesResult - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetCollectionAttributesResult */ DatasetCollectionAttributesResult: { /** * Dbkey @@ -3514,10 +3397,7 @@ export interface components { * @enum {string} */ DatasetContentType: "meta" | "attr" | "stats" | "data"; - /** - * DatasetErrorMessage - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetErrorMessage */ DatasetErrorMessage: { /** * Dataset @@ -3532,14 +3412,10 @@ export interface components { }; /** * DatasetInheritanceChain - * @description Base model definition with common configuration used by all derived models. * @default [] */ DatasetInheritanceChain: components["schemas"]["DatasetInheritanceChainEntry"][]; - /** - * DatasetInheritanceChainEntry - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetInheritanceChainEntry */ DatasetInheritanceChainEntry: { /** * Dep @@ -3570,10 +3446,7 @@ export interface components { */ manage?: string[]; }; - /** - * DatasetSourceId - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetSourceId */ DatasetSourceId: { /** * ID @@ -3611,10 +3484,7 @@ export interface components { | "failed_metadata" | "deferred" | "discarded"; - /** - * DatasetStorageDetails - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetStorageDetails */ DatasetStorageDetails: { /** * Badges @@ -3667,10 +3537,7 @@ export interface components { */ sources: Record[]; }; - /** - * DatasetSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetSummary */ DatasetSummary: { /** * Create Time @@ -3712,15 +3579,9 @@ export interface components { */ uuid: string; }; - /** - * DatasetSummaryList - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetSummaryList */ DatasetSummaryList: components["schemas"]["DatasetSummary"][]; - /** - * DatasetTextContentDetails - * @description Base model definition with common configuration used by all derived models. - */ + /** DatasetTextContentDetails */ DatasetTextContentDetails: { /** * Item Data @@ -3869,10 +3730,7 @@ export interface components { [key: string]: string | undefined; }; }; - /** - * DefaultQuota - * @description Base model definition with common configuration used by all derived models. - */ + /** DefaultQuota */ DefaultQuota: { /** * Model class @@ -3901,10 +3759,7 @@ export interface components { * @enum {string} */ DefaultQuotaValues: "unregistered" | "registered" | "no"; - /** - * DeleteDatasetBatchPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** DeleteDatasetBatchPayload */ DeleteDatasetBatchPayload: { /** * Datasets @@ -3918,10 +3773,7 @@ export interface components { */ purge?: boolean; }; - /** - * DeleteDatasetBatchResult - * @description Base model definition with common configuration used by all derived models. - */ + /** DeleteDatasetBatchResult */ DeleteDatasetBatchResult: { /** * Errors @@ -3934,10 +3786,7 @@ export interface components { */ success_count: number; }; - /** - * DeleteHistoryContentPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** DeleteHistoryContentPayload */ DeleteHistoryContentPayload: { /** * Purge @@ -3991,10 +3840,7 @@ export interface components { */ purge?: boolean; }; - /** - * DeleteJobPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** DeleteJobPayload */ DeleteJobPayload: { /** * Job message @@ -4002,10 +3848,7 @@ export interface components { */ message?: string; }; - /** - * DeleteLibraryPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** DeleteLibraryPayload */ DeleteLibraryPayload: { /** * Undelete @@ -4013,10 +3856,7 @@ export interface components { */ undelete: boolean; }; - /** - * DeleteQuotaPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** DeleteQuotaPayload */ DeleteQuotaPayload: { /** * Purge @@ -4025,10 +3865,7 @@ export interface components { */ purge?: boolean; }; - /** - * DeletedCustomBuild - * @description Base model definition with common configuration used by all derived models. - */ + /** DeletedCustomBuild */ DeletedCustomBuild: { /** * Deletion message @@ -4036,10 +3873,7 @@ export interface components { */ message: string; }; - /** - * DetailedUserModel - * @description Base model definition with common configuration used by all derived models. - */ + /** DetailedUserModel */ DetailedUserModel: { /** * Deleted @@ -4148,10 +3982,7 @@ export interface components { * @enum {string} */ ElementsFromType: "archive" | "bagit" | "bagit_archive" | "directory"; - /** - * EncodedDataItemSourceId - * @description Base model definition with common configuration used by all derived models. - */ + /** EncodedDataItemSourceId */ EncodedDataItemSourceId: { /** * ID @@ -4165,10 +3996,7 @@ export interface components { */ src: components["schemas"]["DataItemSourceType"]; }; - /** - * EncodedDatasetJobInfo - * @description Base model definition with common configuration used by all derived models. - */ + /** EncodedDatasetJobInfo */ EncodedDatasetJobInfo: { /** * ID @@ -4189,10 +4017,7 @@ export interface components { */ uuid?: string; }; - /** - * EncodedDatasetSourceId - * @description Base model definition with common configuration used by all derived models. - */ + /** EncodedDatasetSourceId */ EncodedDatasetSourceId: { /** * ID @@ -4206,10 +4031,7 @@ export interface components { */ src: components["schemas"]["DatasetSourceType"]; }; - /** - * EncodedHdcaSourceId - * @description Base model definition with common configuration used by all derived models. - */ + /** EncodedHdcaSourceId */ EncodedHdcaSourceId: { /** * ID @@ -4353,10 +4175,7 @@ export interface components { */ user_email?: string; }; - /** - * ExportHistoryArchivePayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ExportHistoryArchivePayload */ ExportHistoryArchivePayload: { /** * Directory URI @@ -4387,18 +4206,12 @@ export interface components { */ include_hidden?: boolean; }; - /** - * ExportObjectMetadata - * @description Base model definition with common configuration used by all derived models. - */ + /** ExportObjectMetadata */ ExportObjectMetadata: { request_data: components["schemas"]["ExportObjectRequestMetadata"]; result_data?: components["schemas"]["ExportObjectResultMetadata"]; }; - /** - * ExportObjectRequestMetadata - * @description Base model definition with common configuration used by all derived models. - */ + /** ExportObjectRequestMetadata */ ExportObjectRequestMetadata: { /** * Object Id @@ -4416,10 +4229,7 @@ export interface components { */ user_id?: string; }; - /** - * ExportObjectResultMetadata - * @description Base model definition with common configuration used by all derived models. - */ + /** ExportObjectResultMetadata */ ExportObjectResultMetadata: { /** Error */ error?: string; @@ -4466,15 +4276,9 @@ export interface components { */ target_uri: string; }; - /** - * ExportTaskListResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** ExportTaskListResponse */ ExportTaskListResponse: components["schemas"]["ObjectExportTaskResponse"][]; - /** - * ExtraFiles - * @description Base model definition with common configuration used by all derived models. - */ + /** ExtraFiles */ ExtraFiles: { /** * Fuzzy Root @@ -4486,10 +4290,7 @@ export interface components { items_from?: string; src: components["schemas"]["Src"]; }; - /** - * FavoriteObject - * @description Base model definition with common configuration used by all derived models. - */ + /** FavoriteObject */ FavoriteObject: { /** * Object ID @@ -4503,10 +4304,7 @@ export interface components { * @enum {string} */ FavoriteObjectType: "tools"; - /** - * FavoriteObjectsSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** FavoriteObjectsSummary */ FavoriteObjectsSummary: { /** * Favorite tools @@ -4514,10 +4312,7 @@ export interface components { */ tools: string[]; }; - /** - * FetchDataPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** FetchDataPayload */ FetchDataPayload: { /** * History ID @@ -4534,10 +4329,7 @@ export interface components { | components["schemas"]["FtpImportTarget"] )[]; }; - /** - * FileDataElement - * @description Base model definition with common configuration used by all derived models. - */ + /** FileDataElement */ FileDataElement: { /** Md5 */ MD5?: string; @@ -4590,10 +4382,7 @@ export interface components { */ to_posix_lines?: boolean; }; - /** - * FileLibraryFolderItem - * @description Base model definition with common configuration used by all derived models. - */ + /** FileLibraryFolderItem */ FileLibraryFolderItem: { /** Can Manage */ can_manage: boolean; @@ -4652,10 +4441,7 @@ export interface components { */ update_time: string; }; - /** - * FilesSourcePlugin - * @description Base model definition with common configuration used by all derived models. - */ + /** FilesSourcePlugin */ FilesSourcePlugin: { /** * Browsable @@ -4705,7 +4491,6 @@ export interface components { }; /** * FilesSourcePluginList - * @description Base model definition with common configuration used by all derived models. * @default [] * @example [ * { @@ -4723,10 +4508,7 @@ export interface components { | components["schemas"]["BrowsableFilesSourcePlugin"] | components["schemas"]["FilesSourcePlugin"] )[]; - /** - * FolderLibraryFolderItem - * @description Base model definition with common configuration used by all derived models. - */ + /** FolderLibraryFolderItem */ FolderLibraryFolderItem: { /** Can Manage */ can_manage: boolean; @@ -4765,10 +4547,7 @@ export interface components { */ update_time: string; }; - /** - * FtpImportElement - * @description Base model definition with common configuration used by all derived models. - */ + /** FtpImportElement */ FtpImportElement: { /** Md5 */ MD5?: string; @@ -4823,10 +4602,7 @@ export interface components { */ to_posix_lines?: boolean; }; - /** - * FtpImportTarget - * @description Base model definition with common configuration used by all derived models. - */ + /** FtpImportTarget */ FtpImportTarget: { /** * Auto Decompress @@ -4874,10 +4650,7 @@ export interface components { */ name: string; }; - /** - * GroupQuota - * @description Base model definition with common configuration used by all derived models. - */ + /** GroupQuota */ GroupQuota: { /** * Group @@ -4892,15 +4665,9 @@ export interface components { */ model_class: "GroupQuotaAssociation"; }; - /** - * GroupRoleListResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** GroupRoleListResponse */ GroupRoleListResponse: components["schemas"]["GroupRoleResponse"][]; - /** - * GroupRoleResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** GroupRoleResponse */ GroupRoleResponse: { /** * ID @@ -4920,15 +4687,9 @@ export interface components { */ url: string; }; - /** - * GroupUserListResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** GroupUserListResponse */ GroupUserListResponse: components["schemas"]["GroupUserResponse"][]; - /** - * GroupUserResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** GroupUserResponse */ GroupUserResponse: { /** * Email @@ -5711,10 +5472,7 @@ export interface components { * @enum {string} */ HashFunctionNameEnum: "MD5" | "SHA-1" | "SHA-256" | "SHA-512"; - /** - * HdaDestination - * @description Base model definition with common configuration used by all derived models. - */ + /** HdaDestination */ HdaDestination: { /** * Type @@ -5722,10 +5480,7 @@ export interface components { */ type: "hdas"; }; - /** - * HdcaDataItemsFromTarget - * @description Base model definition with common configuration used by all derived models. - */ + /** HdcaDataItemsFromTarget */ HdcaDataItemsFromTarget: { /** * Auto Decompress @@ -5751,10 +5506,7 @@ export interface components { /** Url */ url?: string; }; - /** - * HdcaDataItemsTarget - * @description Base model definition with common configuration used by all derived models. - */ + /** HdcaDataItemsTarget */ HdcaDataItemsTarget: { /** * Auto Decompress @@ -5783,10 +5535,7 @@ export interface components { /** Tags */ tags?: string[]; }; - /** - * HdcaDestination - * @description Base model definition with common configuration used by all derived models. - */ + /** HdcaDestination */ HdcaDestination: { /** * Type @@ -5794,10 +5543,7 @@ export interface components { */ type: "hdca"; }; - /** - * HistoryContentBulkOperationPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** HistoryContentBulkOperationPayload */ HistoryContentBulkOperationPayload: { /** Items */ items?: components["schemas"]["HistoryContentItem"][]; @@ -5808,10 +5554,7 @@ export interface components { | components["schemas"]["ChangeDbkeyOperationParams"] | components["schemas"]["TagOperationParams"]; }; - /** - * HistoryContentBulkOperationResult - * @description Base model definition with common configuration used by all derived models. - */ + /** HistoryContentBulkOperationResult */ HistoryContentBulkOperationResult: { /** Errors */ errors: components["schemas"]["BulkOperationItemError"][]; @@ -5856,10 +5599,7 @@ export interface components { * @enum {string} */ HistoryContentSource: "hda" | "hdca" | "library" | "library_folder" | "new_collection"; - /** - * HistoryContentStats - * @description Base model definition with common configuration used by all derived models. - */ + /** HistoryContentStats */ HistoryContentStats: { /** * Total Matches @@ -6136,10 +5876,7 @@ export interface components { */ text: string; }; - /** - * ImplicitCollectionJobsStateSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** ImplicitCollectionJobsStateSummary */ ImplicitCollectionJobsStateSummary: { /** * ID @@ -6175,10 +5912,7 @@ export interface components { | components["schemas"]["ImportToolDataBundleDatasetSource"] | components["schemas"]["ImportToolDataBundleUriSource"]; }; - /** - * ImportToolDataBundleDatasetSource - * @description Base model definition with common configuration used by all derived models. - */ + /** ImportToolDataBundleDatasetSource */ ImportToolDataBundleDatasetSource: { /** * ID @@ -6193,10 +5927,7 @@ export interface components { */ src: "hda" | "ldda"; }; - /** - * ImportToolDataBundleUriSource - * @description Base model definition with common configuration used by all derived models. - */ + /** ImportToolDataBundleUriSource */ ImportToolDataBundleUriSource: { /** * src @@ -6210,10 +5941,7 @@ export interface components { */ uri: string; }; - /** - * InputArguments - * @description Base model definition with common configuration used by all derived models. - */ + /** InputArguments */ InputArguments: { /** * Database Key @@ -6240,10 +5968,7 @@ export interface components { */ to_posix_lines?: "Yes" | boolean; }; - /** - * InstalledRepositoryToolShedStatus - * @description Base model definition with common configuration used by all derived models. - */ + /** InstalledRepositoryToolShedStatus */ InstalledRepositoryToolShedStatus: { /** * Latest installed revision @@ -6260,10 +5985,7 @@ export interface components { /** Revision Upgrade */ revision_upgrade?: string; }; - /** - * InstalledToolShedRepository - * @description Base model definition with common configuration used by all derived models. - */ + /** InstalledToolShedRepository */ InstalledToolShedRepository: { /** * Changeset revision @@ -6324,10 +6046,7 @@ export interface components { /** Uninstalled */ uninstalled: boolean; }; - /** - * ItemTagsPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ItemTagsPayload */ ItemTagsPayload: { /** * Item class @@ -6352,10 +6071,7 @@ export interface components { * @enum {string} */ ItemsFromSrc: "url" | "files" | "path" | "ftp_import" | "server_dir"; - /** - * JobDestinationParams - * @description Base model definition with common configuration used by all derived models. - */ + /** JobDestinationParams */ JobDestinationParams: { /** * Handler @@ -6373,10 +6089,7 @@ export interface components { */ "Runner Job ID"?: string; }; - /** - * JobDisplayParametersSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** JobDisplayParametersSummary */ JobDisplayParametersSummary: { /** * Has parameter errors @@ -6396,10 +6109,7 @@ export interface components { */ parameters: components["schemas"]["JobParameter"][]; }; - /** - * JobErrorSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** JobErrorSummary */ JobErrorSummary: { /** * Error messages @@ -6407,15 +6117,9 @@ export interface components { */ messages: string[][]; }; - /** - * JobExportHistoryArchiveListResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** JobExportHistoryArchiveListResponse */ JobExportHistoryArchiveListResponse: components["schemas"]["JobExportHistoryArchiveModel"][]; - /** - * JobExportHistoryArchiveModel - * @description Base model definition with common configuration used by all derived models. - */ + /** JobExportHistoryArchiveModel */ JobExportHistoryArchiveModel: { /** * Download URL @@ -6474,10 +6178,7 @@ export interface components { */ job_id: string; }; - /** - * JobImportHistoryResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** JobImportHistoryResponse */ JobImportHistoryResponse: { /** * Create Time @@ -6549,10 +6250,7 @@ export interface components { * @enum {string} */ JobIndexViewEnum: "collection" | "admin_job_list"; - /** - * JobInputAssociation - * @description Base model definition with common configuration used by all derived models. - */ + /** JobInputAssociation */ JobInputAssociation: { /** * dataset @@ -6565,10 +6263,7 @@ export interface components { */ name: string; }; - /** - * JobInputSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** JobInputSummary */ JobInputSummary: { /** * Duplicate inputs @@ -6591,7 +6286,6 @@ export interface components { }; /** * JobMetric - * @description Base model definition with common configuration used by all derived models. * @example { * "name": "start_epoch", * "plugin": "core", @@ -6627,10 +6321,7 @@ export interface components { */ value: string; }; - /** - * JobOutput - * @description Base model definition with common configuration used by all derived models. - */ + /** JobOutput */ JobOutput: { /** * Output label @@ -6643,10 +6334,7 @@ export interface components { */ value: components["schemas"]["EncodedDataItemSourceId"]; }; - /** - * JobOutputAssociation - * @description Base model definition with common configuration used by all derived models. - */ + /** JobOutputAssociation */ JobOutputAssociation: { /** * dataset @@ -6659,10 +6347,7 @@ export interface components { */ name: string; }; - /** - * JobParameter - * @description Base model definition with common configuration used by all derived models. - */ + /** JobParameter */ JobParameter: { /** * Depth @@ -6712,10 +6397,7 @@ export interface components { | "stop" | "stopped" | "skipped"; - /** - * JobStateSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** JobStateSummary */ JobStateSummary: { /** * ID @@ -6760,10 +6442,7 @@ export interface components { */ value: string; }; - /** - * LegacyLibraryPermissionsPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** LegacyLibraryPermissionsPayload */ LegacyLibraryPermissionsPayload: { /** * Access IDs @@ -6790,10 +6469,7 @@ export interface components { */ LIBRARY_MODIFY_in?: string[] | string; }; - /** - * LibraryAvailablePermissions - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryAvailablePermissions */ LibraryAvailablePermissions: { /** * Page @@ -6816,10 +6492,7 @@ export interface components { */ total: number; }; - /** - * LibraryCurrentPermissions - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryCurrentPermissions */ LibraryCurrentPermissions: { /** * Access Role List @@ -6842,10 +6515,7 @@ export interface components { */ modify_library_role_list: string[][]; }; - /** - * LibraryDestination - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryDestination */ LibraryDestination: { /** * Description @@ -6868,10 +6538,7 @@ export interface components { */ type: "library"; }; - /** - * LibraryFolderContentsIndexResult - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryFolderContentsIndexResult */ LibraryFolderContentsIndexResult: { /** Folder Contents */ folder_contents: ( @@ -6880,10 +6547,7 @@ export interface components { )[]; metadata: components["schemas"]["LibraryFolderMetadata"]; }; - /** - * LibraryFolderCurrentPermissions - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryFolderCurrentPermissions */ LibraryFolderCurrentPermissions: { /** * Add Role List @@ -6901,10 +6565,7 @@ export interface components { */ modify_folder_role_list: string[][]; }; - /** - * LibraryFolderDestination - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryFolderDestination */ LibraryFolderDestination: { /** * Library Folder Id @@ -6917,10 +6578,7 @@ export interface components { */ type: "library_folder"; }; - /** - * LibraryFolderDetails - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryFolderDetails */ LibraryFolderDetails: { /** * Deleted @@ -6987,10 +6645,7 @@ export interface components { */ update_time: string; }; - /** - * LibraryFolderMetadata - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryFolderMetadata */ LibraryFolderMetadata: { /** Can Add Library Item */ can_add_library_item: boolean; @@ -7016,10 +6671,7 @@ export interface components { * @enum {string} */ LibraryFolderPermissionAction: "set_permissions"; - /** - * LibraryFolderPermissionsPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryFolderPermissionsPayload */ LibraryFolderPermissionsPayload: { /** * Action @@ -7045,10 +6697,7 @@ export interface components { */ "modify_ids[]"?: string[] | string; }; - /** - * LibraryLegacySummary - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryLegacySummary */ LibraryLegacySummary: { /** * Create Time @@ -7109,10 +6758,7 @@ export interface components { * @enum {string} */ LibraryPermissionScope: "current" | "available"; - /** - * LibraryPermissionsPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** LibraryPermissionsPayload */ LibraryPermissionsPayload: { /** * Access IDs @@ -7144,10 +6790,7 @@ export interface components { */ "modify_ids[]"?: string[] | string; }; - /** - * LibrarySummary - * @description Base model definition with common configuration used by all derived models. - */ + /** LibrarySummary */ LibrarySummary: { /** * Can User Add @@ -7224,7 +6867,6 @@ export interface components { }; /** * LibrarySummaryList - * @description Base model definition with common configuration used by all derived models. * @default [] */ LibrarySummaryList: components["schemas"]["LibrarySummary"][]; @@ -7338,10 +6980,7 @@ export interface components { * @enum {string} */ MandatoryNotificationCategory: "broadcast"; - /** - * MaterializeDatasetInstanceAPIRequest - * @description Base model definition with common configuration used by all derived models. - */ + /** MaterializeDatasetInstanceAPIRequest */ MaterializeDatasetInstanceAPIRequest: { /** * Content @@ -7358,10 +6997,7 @@ export interface components { */ source: components["schemas"]["DatasetSourceType"]; }; - /** - * MessageNotificationContent - * @description Base model definition with common configuration used by all derived models. - */ + /** MessageNotificationContent */ MessageNotificationContent: { /** * Category @@ -7516,10 +7152,7 @@ export interface components { * @enum {string} */ ModelStoreFormat: "tgz" | "tar" | "tar.gz" | "bag.zip" | "bag.tar" | "bag.tgz" | "rocrate.zip" | "bco.json"; - /** - * NestedElement - * @description Base model definition with common configuration used by all derived models. - */ + /** NestedElement */ NestedElement: { /** Md5 */ MD5?: string; @@ -7580,10 +7213,7 @@ export interface components { */ to_posix_lines?: boolean; }; - /** - * NewSharedItemNotificationContent - * @description Base model definition with common configuration used by all derived models. - */ + /** NewSharedItemNotificationContent */ NewSharedItemNotificationContent: { /** * Category @@ -7737,10 +7367,7 @@ export interface components { */ recipients: components["schemas"]["NotificationRecipients"]; }; - /** - * NotificationCreatedResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** NotificationCreatedResponse */ NotificationCreatedResponse: { /** * Notification @@ -7865,10 +7492,7 @@ export interface components { * @enum {string} */ NotificationVariant: "info" | "warning" | "urgent"; - /** - * NotificationsBatchRequest - * @description Base model definition with common configuration used by all derived models. - */ + /** NotificationsBatchRequest */ NotificationsBatchRequest: { /** * Notification IDs @@ -7887,10 +7511,7 @@ export interface components { */ updated_count: number; }; - /** - * ObjectExportTaskResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** ObjectExportTaskResponse */ ObjectExportTaskResponse: { /** * Create Time @@ -7949,10 +7570,7 @@ export interface components { * @enum {string} */ PageContentFormat: "markdown" | "html"; - /** - * PageDetails - * @description Base model definition with common configuration used by all derived models. - */ + /** PageDetails */ PageDetails: { /** * Content @@ -8049,10 +7667,7 @@ export interface components { */ username: string; }; - /** - * PageSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** PageSummary */ PageSummary: { /** * Create Time @@ -8129,14 +7744,10 @@ export interface components { }; /** * PageSummaryList - * @description Base model definition with common configuration used by all derived models. * @default [] */ PageSummaryList: components["schemas"]["PageSummary"][]; - /** - * PastedDataElement - * @description Base model definition with common configuration used by all derived models. - */ + /** PastedDataElement */ PastedDataElement: { /** Md5 */ MD5?: string; @@ -8194,10 +7805,7 @@ export interface components { */ to_posix_lines?: boolean; }; - /** - * PathDataElement - * @description Base model definition with common configuration used by all derived models. - */ + /** PathDataElement */ PathDataElement: { /** Md5 */ MD5?: string; @@ -8267,10 +7875,7 @@ export interface components { * @enum {string} */ PluginKind: "rfs" | "drs" | "rdm" | "stock"; - /** - * PrepareStoreDownloadPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** PrepareStoreDownloadPayload */ PrepareStoreDownloadPayload: { /** * Bco Merge History Metadata @@ -8446,7 +8051,6 @@ export interface components { }; /** * QuotaSummaryList - * @description Base model definition with common configuration used by all derived models. * @default [] */ QuotaSummaryList: components["schemas"]["QuotaSummary"][]; @@ -8459,10 +8063,7 @@ export interface components { /** Reloaded */ reloaded: string[]; }; - /** - * RemoteDirectory - * @description Base model definition with common configuration used by all derived models. - */ + /** RemoteDirectory */ RemoteDirectory: { /** * Class @@ -8485,10 +8086,7 @@ export interface components { */ uri: string; }; - /** - * RemoteFile - * @description Base model definition with common configuration used by all derived models. - */ + /** RemoteFile */ RemoteFile: { /** * Class @@ -8533,10 +8131,7 @@ export interface components { * @enum {string} */ RemoteFilesFormat: "flat" | "jstree" | "uri"; - /** - * RemoteUserCreationPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** RemoteUserCreationPayload */ RemoteUserCreationPayload: { /** * Email @@ -8544,10 +8139,7 @@ export interface components { */ remote_user_email: string; }; - /** - * ReportJobErrorPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ReportJobErrorPayload */ ReportJobErrorPayload: { /** * History Dataset Association ID @@ -8586,10 +8178,7 @@ export interface components { * @enum {string} */ Requirement: "logged_in" | "new_history" | "admin"; - /** - * RoleDefinitionModel - * @description Base model definition with common configuration used by all derived models. - */ + /** RoleDefinitionModel */ RoleDefinitionModel: { /** * Description @@ -8612,15 +8201,9 @@ export interface components { */ user_ids?: string[]; }; - /** - * RoleListResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** RoleListResponse */ RoleListResponse: components["schemas"]["RoleModelResponse"][]; - /** - * RoleModelResponse - * @description Base model definition with common configuration used by all derived models. - */ + /** RoleModelResponse */ RoleModelResponse: { /** * Description @@ -8657,10 +8240,7 @@ export interface components { */ url: string; }; - /** - * SearchJobsPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** SearchJobsPayload */ SearchJobsPayload: { /** * Inputs @@ -8678,10 +8258,7 @@ export interface components { */ tool_id: string; }; - /** - * ServerDirElement - * @description Base model definition with common configuration used by all derived models. - */ + /** ServerDirElement */ ServerDirElement: { /** Md5 */ MD5?: string; @@ -8826,10 +8403,7 @@ export interface components { */ version: string; }; - /** - * SetSlugPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** SetSlugPayload */ SetSlugPayload: { /** * New Slug @@ -8837,10 +8411,7 @@ export interface components { */ new_slug: string; }; - /** - * ShareWithExtra - * @description Base model definition with common configuration used by all derived models. - */ + /** ShareWithExtra */ ShareWithExtra: { /** * Can Share @@ -8849,10 +8420,7 @@ export interface components { */ can_share?: boolean; }; - /** - * ShareWithPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ShareWithPayload */ ShareWithPayload: { /** * Share Option @@ -8869,10 +8437,7 @@ export interface components { */ user_ids: (string | string)[]; }; - /** - * ShareWithStatus - * @description Base model definition with common configuration used by all derived models. - */ + /** ShareWithStatus */ ShareWithStatus: { /** * Encoded Email @@ -8934,10 +8499,7 @@ export interface components { * @enum {string} */ SharingOptions: "make_public" | "make_accessible_to_shared" | "no_changes"; - /** - * SharingStatus - * @description Base model definition with common configuration used by all derived models. - */ + /** SharingStatus */ SharingStatus: { /** * Encoded Email @@ -8982,10 +8544,7 @@ export interface components { */ users_shared_with?: components["schemas"]["UserEmail"][]; }; - /** - * ShortTermStoreExportPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** ShortTermStoreExportPayload */ ShortTermStoreExportPayload: { /** Duration */ duration?: number | number; @@ -9024,10 +8583,7 @@ export interface components { * @enum {string} */ Src: "url" | "pasted" | "files" | "path" | "composite" | "ftp_import" | "server_dir"; - /** - * StorageItemCleanupError - * @description Base model definition with common configuration used by all derived models. - */ + /** StorageItemCleanupError */ StorageItemCleanupError: { /** Error */ error: string; @@ -9037,10 +8593,7 @@ export interface components { */ item_id: string; }; - /** - * StorageItemsCleanupResult - * @description Base model definition with common configuration used by all derived models. - */ + /** StorageItemsCleanupResult */ StorageItemsCleanupResult: { /** Errors */ errors: components["schemas"]["StorageItemCleanupError"][]; @@ -9051,10 +8604,7 @@ export interface components { /** Total Item Count */ total_item_count: number; }; - /** - * StoreExportPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** StoreExportPayload */ StoreExportPayload: { /** * Include deleted @@ -9080,10 +8630,7 @@ export interface components { */ model_store_format?: components["schemas"]["ModelStoreFormat"]; }; - /** - * StoredItem - * @description Base model definition with common configuration used by all derived models. - */ + /** StoredItem */ StoredItem: { /** * Id @@ -9109,10 +8656,7 @@ export interface components { * @enum {string} */ StoredItemOrderBy: "name-asc" | "name-dsc" | "size-asc" | "size-dsc" | "update_time-asc" | "update_time-dsc"; - /** - * SuitableConverter - * @description Base model definition with common configuration used by all derived models. - */ + /** SuitableConverter */ SuitableConverter: { /** * Name @@ -9150,10 +8694,7 @@ export interface components { * ] */ TagCollection: string[]; - /** - * TagOperationParams - * @description Base model definition with common configuration used by all derived models. - */ + /** TagOperationParams */ TagOperationParams: { /** Tags */ tags: string[]; @@ -9468,10 +9009,7 @@ export interface components { */ visible?: boolean; }; - /** - * UpdateLibraryFolderPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** UpdateLibraryFolderPayload */ UpdateLibraryFolderPayload: { /** * Description @@ -9484,10 +9022,7 @@ export interface components { */ name?: string; }; - /** - * UpdateLibraryPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** UpdateLibraryPayload */ UpdateLibraryPayload: { /** * Description @@ -9505,10 +9040,7 @@ export interface components { */ synopsis?: string; }; - /** - * UpdateQuotaParams - * @description Base model definition with common configuration used by all derived models. - */ + /** UpdateQuotaParams */ UpdateQuotaParams: { /** * Amount @@ -9576,10 +9108,7 @@ export interface components { [key: string]: components["schemas"]["NotificationCategorySettings"] | undefined; }; }; - /** - * UrlDataElement - * @description Base model definition with common configuration used by all derived models. - */ + /** UrlDataElement */ UrlDataElement: { /** Md5 */ MD5?: string; @@ -9637,10 +9166,7 @@ export interface components { */ url: string; }; - /** - * UserBeaconSetting - * @description Base model definition with common configuration used by all derived models. - */ + /** UserBeaconSetting */ UserBeaconSetting: { /** * Enabled @@ -9648,10 +9174,7 @@ export interface components { */ enabled: boolean; }; - /** - * UserCreationPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** UserCreationPayload */ UserCreationPayload: { /** * Email @@ -9669,10 +9192,7 @@ export interface components { */ username: string; }; - /** - * UserDeletionPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** UserDeletionPayload */ UserDeletionPayload: { /** * Purge user @@ -9680,10 +9200,7 @@ export interface components { */ purge: boolean; }; - /** - * UserEmail - * @description Base model definition with common configuration used by all derived models. - */ + /** UserEmail */ UserEmail: { /** * Email @@ -9877,10 +9394,7 @@ export interface components { */ notification_ids: string[]; }; - /** - * UserQuota - * @description Base model definition with common configuration used by all derived models. - */ + /** UserQuota */ UserQuota: { /** * Model class @@ -9917,15 +9431,9 @@ export interface components { /** Error Type */ type: string; }; - /** - * Visualization - * @description Base model definition with common configuration used by all derived models. - */ + /** Visualization */ Visualization: Record; - /** - * VisualizationSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** VisualizationSummary */ VisualizationSummary: { /** * Annotation @@ -9993,14 +9501,10 @@ export interface components { }; /** * VisualizationSummaryList - * @description Base model definition with common configuration used by all derived models. * @default [] */ VisualizationSummaryList: components["schemas"]["VisualizationSummary"][]; - /** - * WorkflowInvocationStateSummary - * @description Base model definition with common configuration used by all derived models. - */ + /** WorkflowInvocationStateSummary */ WorkflowInvocationStateSummary: { /** * ID @@ -10029,10 +9533,7 @@ export interface components { [key: string]: number | undefined; }; }; - /** - * WriteInvocationStoreToPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** WriteInvocationStoreToPayload */ WriteInvocationStoreToPayload: { /** * Bco Merge History Metadata @@ -10095,10 +9596,7 @@ export interface components { */ target_uri: string; }; - /** - * WriteStoreToPayload - * @description Base model definition with common configuration used by all derived models. - */ + /** WriteStoreToPayload */ WriteStoreToPayload: { /** * Include deleted diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index c408e2538be0..6920fe5dbb94 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -257,8 +257,6 @@ class DatasetCollectionPopulatedState(str, Enum): class Model(BaseModel): - """Base model definition with common configuration used by all derived models.""" - class Config: use_enum_values = True # when using .dict() allow_population_by_field_name = True