Test and fix containerized metadata with pulsar

galaxyproject · Dec 12, 2023 · 96516d2 · 96516d2
1 parent fc44060
commit 96516d2
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 6 deletions.
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
@@ -561,7 +561,12 @@ def _find_container(
             job_wrapper.set_container(container)
         return container
 
-    def _get_metadata_container(self, job_wrapper):
+    def _get_metadata_container(
+        self,
+        job_wrapper,
+        job_directory_type: typing.Literal["galaxy", "pulsar"] = "galaxy",
+        working_directory: typing.Optional[str] = None,
+    ):
         destination_info = job_wrapper.job_destination.params
         if destination_info.get("metadata_config", {}).get("containerize"):
             image = destination_info["metadata_config"].get("image", "galaxyproject/galaxy-job-execution")
@@ -577,12 +582,12 @@ def _get_metadata_container(self, job_wrapper):
                 profile=23.2,
             )
             job_info = JobInfo(
-                working_directory=job_wrapper.working_directory,
+                working_directory=working_directory or job_wrapper.working_directory,
                 tool_directory=None,
                 job_directory=None,
                 tmp_directory=None,
                 home_directory=None,
-                job_directory_type="galaxy",
+                job_directory_type=job_directory_type,
             )
 
             return self.app.container_finder.find_container(tool_info, destination_info, job_info)

diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
@@ -522,6 +522,9 @@ def __prepare_job(self, job_wrapper, job_destination):
                     compute_tool_directory=remote_tool_directory,
                     compute_job_directory=remote_job_directory,
                 )
+            metadata_container = self._get_metadata_container(
+                job_wrapper, job_directory_type="pulsar", working_directory=remote_job_directory
+            )
 
             # Pulsar handles ``create_tool_working_directory`` and
             # ``include_work_dir_outputs`` details.
@@ -534,6 +537,7 @@ def __prepare_job(self, job_wrapper, job_destination):
                 include_work_dir_outputs=False,
                 remote_command_params=remote_command_params,
                 remote_job_directory=remote_job_directory,
+                metadata_container=metadata_container,
             )
         except UnsupportedPulsarException:
             log.exception("failure running job %d, unsupported Pulsar target", job_wrapper.job_id)

diff --git a/test/integration/embedded_pulsar_docker_job_conf.yml b/test/integration/embedded_pulsar_docker_job_conf.yml
@@ -20,6 +20,10 @@ execution:
       docker_required: true
       container_monitor_result: callback
       remote_metadata: true
+      metadata_config:
+        containerize: True
+        engine: docker
+        image: "galaxyproject/galaxy-job-execution"
 
 tools:
 - id: upload1

diff --git a/test/integration/test_metadata_containerized.py b/test/integration/test_metadata_containerized.py
@@ -9,8 +9,14 @@
     skip_if_container_type_unavailable,
 )
 
-DOCKERIZED_JOB_CONFIG = {
-    "runners": {"local": {"load": "galaxy.jobs.runners.local:LocalJobRunner", "workers": 1}},
+DOCKERIZED_METADATA_JOB_CONFIG = {
+    "runners": {
+        "local": {"load": "galaxy.jobs.runners.local:LocalJobRunner"},
+        "pulsar_embed": {
+            "load": "galaxy.jobs.runners.pulsar:PulsarEmbeddedJobRunner",
+            "pulsar_app_config": {"conda_auto_init": False, "conda_auto_install": False},
+        },
+    },
     "execution": {
         "default": "local_docker",
         "environments": {
@@ -23,8 +29,22 @@
                     "image": "galaxyproject/galaxy-job-execution",
                 },
             },
+            "pulsar_embed": {
+                "runner": "pulsar_embed",
+                "docker_enabled": True,
+                "remote_metadata": True,
+                "metadata_config": {
+                    "containerize": True,
+                    "engine": "docker",
+                    "image": "galaxyproject/galaxy-job-execution",
+                },
+            },
         },
     },
+    "tools": [
+        {"id": "metadata_bam", "environment": "local_docker"},
+        {"id": "composite_output", "environment": "local_docker"},
+    ],
 }
 
 
@@ -36,7 +56,7 @@ class ContainerizedMetadataIntegrationTestCase(integration_util.IntegrationTestC
     @classmethod
     def handle_galaxy_config_kwds(cls, config) -> None:
         super().handle_galaxy_config_kwds(config)
-        config["job_config"] = DOCKERIZED_JOB_CONFIG
+        config["job_config"] = DOCKERIZED_METADATA_JOB_CONFIG
         config["enable_celery_tasks"] = False
         config["metadata_strategy"] = "extended"
         disable_dependency_resolution(config)
@@ -67,5 +87,6 @@ def setUpClass(cls) -> None:
 test_tools = integration_util.integration_tool_runner(
     [
         "metadata_bam",
+        "composite_output",
     ]
 )