Test kubernetes in CI

.github/workflows/ci-k8s.yaml

@@ -0,0 +1,53 @@
+name: Parsl
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+
+jobs:
+  k8s-kind-suite:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Create k8s Kind Cluster
+      uses: helm/kind-action@v1
+      with:
+        # kind tooling uses this name by default, but kind-action uses
+        # a different default name
+        cluster_name: kind
+
+    - name: Build docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        tags: parsl:ci
+
+    - name: Push docker image into kubernetes cluster
+      run: |
+        kind load docker-image parsl:ci
+
+    - name: set liberal permissions
+      run: |
+        kubectl create clusterrolebinding serviceaccounts-cluster-admin   --clusterrole=cluster-admin   --group=system:serviceaccounts
+
+    - name: launch pytest Job
+      run: |
+        free -h
+        kubectl create -f ./pytest-task.yaml 
+
+    - name: wait for pytest Job
+      run: |
+        kubectl wait --timeout=600s --for=condition=Complete Job pytest
+
+    - name: report some info
+      if: ${{ always() }}
+      run: |
+        free -h
+        kubectl describe pods
+        kubectl describe jobs
+        kubectl logs --timestamps Job/pytest

Dockerfile

@@ -0,0 +1,40 @@
+FROM debian:trixie
+
+RUN apt-get update && apt-get upgrade -y
+
+RUN apt-get update && apt-get install -y sudo openssh-server
+
+RUN apt-get update && apt-get install -y curl less vim
+
+# git is needed for parsl to figure out it's own repo-specific
+# version string
+RUN apt-get update && apt-get install -y git
+
+# useful stuff to have around
+RUN apt-get update && apt-get install -y procps
+
+# for building documentation
+RUN apt-get update && apt-get install -y pandoc
+
+# for monitoring visualization
+RUN apt-get update && apt-get install -y graphviz wget
+
+# for commandline access to monitoring database
+RUN apt-get update && apt-get install -y sqlite3
+
+RUN apt-get update && apt-get install -y python3.12 python3.12-dev
+RUN apt-get update && apt-get install -y python3.12-venv
+
+RUN apt-get update && apt-get install -y gcc build-essential make pkg-config mpich
+
+RUN python3.12 -m venv /venv
+
+ADD . /parsl
+WORKDIR /
+RUN git clone https://github.com/cooperative-computing-lab/cctools
+WORKDIR /cctools
+RUN . /venv/bin/activate && apt install swig && ./configure --prefix=/ && make && make install
+
+WORKDIR /parsl
+RUN . /venv/bin/activate && pip3 install '.[kubernetes]' cloudpickle -r test-requirements.txt
+

htex_k8s_kind.py

@@ -0,0 +1,26 @@
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import KubernetesProvider
+
+
+def fresh_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="executorname",
+                storage_access=[],
+                worker_debug=True,
+                cores_per_worker=1,
+                encrypted=False,  # needs certificate fs to be mounted in same place...
+                provider=KubernetesProvider(
+                    worker_init=". /venv/bin/activate",
+                    # pod_name="override-pod-name", # can't use default name because of dots, without own bugfix
+                    image="parsl:ci",
+                    max_mem="2048Gi"  # was getting OOM-killing of workers with default... maybe this will help.
+                    ),
+            )
+        ],
+        strategy='none',
+    )

parsl/executors/taskvine/executor.py

@@ -110,6 +110,8 @@ def __init__(self,
                  storage_access: Optional[List[Staging]] = None):
 
         # Set worker launch option for this executor
+        # This is to make repr work - otherwise it raises an attribute error
+        self.worker_launch_method = worker_launch_method
         if worker_launch_method == 'factory' or worker_launch_method == 'manual':
             provider = None
 
@@ -582,11 +584,18 @@ def shutdown(self, *args, **kwargs):
         logger.debug("TaskVine shutdown started")
         self._should_stop.set()
 
-        # Remove the workers that are still going
-        kill_ids = [self.blocks_to_job_id[block] for block in self.blocks_to_job_id.keys()]
-        if self.provider:
-            logger.debug("Cancelling blocks")
-            self.provider.cancel(kill_ids)
+        # BENC: removed this bit because the scaling code does this,
+        # and the kubernetes provider fails trying to scale in blocks
+        # that have already been deleted by the scaling code shutdown.
+        # This code assumes it can enumerate all blocks by using the
+        # blocks_to_job_id structure, but there's a not-very-strong
+        # principle that: you should not enumerate blocks_to_job_id
+        # (or the job to block map) and you should not call cancel on
+        # already cancelled blocks.
+        # kill_ids = [self.blocks_to_job_id[block] for block in self.blocks_to_job_id.keys()]
+        # if self.provider:
+        #    logger.debug("Cancelling blocks")
+        #    self.provider.cancel(kill_ids)
 
         # Join all processes before exiting
         logger.debug("Joining on submit process")

parsl/executors/taskvine/manager_config.py

@@ -25,6 +25,7 @@ class TaskVineManagerConfig:
     address: Optional[str]
         Address of the local machine.
         If None, socket.gethostname() will be used to determine the address.
+        XXXX ^ if None, looks like get_any_address is being used and in my kubernetes setup, choosing 127.0.0.1
 
     project_name: Optional[str]
         If given, TaskVine will periodically report its status and performance

parsl/providers/kubernetes/kube.py

@@ -189,6 +189,7 @@ def submit(self, cmd_string: str, tasks_per_node: int, job_name: str = "parsl.ku
                                                worker_init=self.worker_init)
 
         logger.debug("Pod name: %s", pod_name)
+
         self._create_pod(image=self.image,
                          pod_name=pod_name,
                          job_id=job_id,

pytest-task.yaml

@@ -0,0 +1,15 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pytest
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: pytest
+        image: parsl:ci
+        command: ["bash", "runme.sh"]
+

runme.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+source /venv/bin/activate
+
+pytest parsl/tests/ --config ./htex_k8s_kind.py -k 'not issue3328 and not staging_required and not shared_fs' -x --random-order
+
+
+# I tried letting staging_required tests run here but they do not -- a bit confused about this comment in taskvine:
+#
+#            # Absolute paths are assumed to be in shared filesystem, and thus
+#            # not staged by taskvine.
+# which I guess is saying something is making assumptions about the presence of a shared filesystem even when defaulting to shared_fs=False in the taskvine config?
+
+
+PYTHONPATH=/usr/lib/python3.12/site-packages/ pytest parsl/tests/ --config ./taskvine_k8s_kind.py -k 'not issue3328 and not staging_required and not shared_fs' -x --random-order --log-cli-level=DEBUG

taskvine_k8s_kind.py

@@ -0,0 +1,19 @@
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.launchers import SimpleLauncher
+from parsl.providers import KubernetesProvider
+from parsl.addresses import address_by_hostname
+
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+
+def fresh_config():
+    return Config(executors=[TaskVineExecutor(manager_config=TaskVineManagerConfig(address=address_by_hostname(), port=9000),
+                                              worker_launch_method='provider',
+                provider=KubernetesProvider(
+                    worker_init=". /venv/bin/activate",
+                    # pod_name="override-pod-name", # can't use default name because of dots, without own bugfix
+                    image="parsl:ci",
+                    max_mem="2048Gi"  # was getting OOM-killing of workers with default... maybe this will help.
+                    ),
+
+                )])