fix: remove creation of mlpipeline_minio_artifact secret

.github/workflows/integrate.yaml

@@ -75,6 +75,10 @@ jobs:
       run: kubectl get all -A
       if: failure()
 
+    - name: Get all workflows
+      run: kubectl get workflows -A -oyaml
+      if: failure()
+
     - name: Describe deployments
       run: kubectl describe deployments -A
       if: failure()

charms/argo-controller/requirements-integration.in

@@ -2,6 +2,7 @@ aiohttp
 jinja2
 # Pinning to <4.0 due to compatibility with the 3.1 controller version
 juju<4.0
+lightkube
 pytest-operator
 requests
 tenacity

charms/argo-controller/requirements-integration.txt

@@ -8,6 +8,8 @@ aiohttp==3.8.5
     # via -r requirements-integration.in
 aiosignal==1.3.1
     # via aiohttp
+anyio==4.0.0
+    # via httpcore
 asttokens==2.4.0
     # via stack-data
 async-timeout==4.0.3
@@ -22,6 +24,8 @@ cachetools==5.3.1
     # via google-auth
 certifi==2023.7.22
     # via
+    #   httpcore
+    #   httpx
     #   kubernetes
     #   requests
 cffi==1.15.1
@@ -39,7 +43,9 @@ decorator==5.1.1
     #   ipdb
     #   ipython
 exceptiongroup==1.1.3
-    # via pytest
+    # via
+    #   anyio
+    #   pytest
 executing==1.2.0
     # via stack-data
 frozenlist==1.4.0
@@ -48,10 +54,18 @@ frozenlist==1.4.0
     #   aiosignal
 google-auth==2.22.0
     # via kubernetes
+h11==0.14.0
+    # via httpcore
+httpcore==0.18.0
+    # via httpx
+httpx==0.25.0
+    # via lightkube
 hvac==1.2.0
     # via juju
 idna==3.4
     # via
+    #   anyio
+    #   httpx
     #   requests
     #   yarl
 iniconfig==2.0.0
@@ -72,6 +86,10 @@ juju==3.2.2
     #   pytest-operator
 kubernetes==27.2.0
     # via juju
+lightkube==0.14.0
+    # via -r requirements-integration.in
+lightkube-models==1.28.1.4
+    # via lightkube
 macaroonbakery==1.3.1
     # via juju
 markupsafe==2.1.3
@@ -148,6 +166,7 @@ pyyaml==6.0.1
     # via
     #   juju
     #   kubernetes
+    #   lightkube
     #   pytest-operator
 requests==2.31.0
     # via
@@ -169,6 +188,11 @@ six==1.16.0
     #   paramiko
     #   pymacaroons
     #   python-dateutil
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpcore
+    #   httpx
 stack-data==0.6.2
     # via ipython
 tenacity==8.2.3

charms/argo-controller/src/charm.py

@@ -8,7 +8,6 @@
 """
 
 import logging
-from base64 import b64encode
 
 import lightkube
 from charmed_kubeflow_chisme.components import SdiRelationDataReceiverComponent
@@ -43,7 +42,6 @@
     "src/templates/auth_manifests.yaml.j2",
     "src/templates/crds.yaml",
     "src/templates/minio_configmap.yaml.j2",
-    "src/templates/mlpipeline_minio_artifact_secret.yaml.j2",
 ]
 METRICS_PATH = "/metrics"
 
@@ -146,16 +144,13 @@ def __init__(self, *args):
 
     @property
     def _context_callable(self):
+        # The mlpipeline_minio_artifact_secret is hardcoded to mlpipeline-minio-artifact
+        # because that is the name the secret will have when deployed by kfp-profile-controller
+        # We are assuming that argo-workflows is always going to be deployed alongside kfp-operators
         return lambda: {
             "app_name": self.app.name,
             "namespace": self.model.name,
-            "access_key": b64encode(
-                self.object_storage_relation.component.get_data()["access-key"].encode("utf-8")
-            ).decode("utf-8"),
-            "secret_key": b64encode(
-                self.object_storage_relation.component.get_data()["secret-key"].encode("utf-8")
-            ).decode("utf-8"),
-            "mlpipeline_minio_artifact_secret": "mlpipeline-minio-artifact-secret",
+            "mlpipeline_minio_artifact_secret": "mlpipeline-minio-artifact",
             "argo_controller_configmap": ARGO_CONTROLLER_CONFIGMAP,
             "s3_bucket": self.model.config["bucket"],
             "s3_minio_endpoint": (

charms/argo-controller/src/templates/mlpipeline_minio_artifact_secret.yaml.j2 → charms/argo-controller/tests/integration/secret.yaml

@@ -1,11 +1,8 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  labels:
-    app: minio
-  name: {{ mlpipeline_minio_artifact_secret }}
+  name: mlpipeline-minio-artifact
   namespace: {{ namespace }}
-stringData:
+data:
   accesskey: {{ access_key }}
   secretkey: {{ secret_key }}
-type: Opaque

charms/argo-controller/tests/integration/test_charm.py

@@ -3,12 +3,16 @@
 
 import json
 import logging
+from base64 import b64encode
 from pathlib import Path
 
+import lightkube
 import pytest
 import requests
 import tenacity
 import yaml
+from jinja2 import Template
+from lightkube import ApiError, codecs
 from pytest_operator.plugin import OpsTest
 
 log = logging.getLogger(__name__)
@@ -50,6 +54,29 @@ async def test_build_and_deploy_with_relations(ops_test: OpsTest):
     await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", timeout=1000)
 
 
+async def create_mlpipeline_minio_secret(ops_test: OpsTest):
+    """Creates a Secret with name mlpipeline-minio-artifact.
+
+    This simulates the behaviour of having the kfp-profile-controller
+    present in the deployment for providing this Secret, but without
+    having to actually deploy it in the testing environment.
+    """
+    lightkube_client = lightkube.Client()
+    secret_file = Path("./tests/integration/secret.yaml").read_text()
+    secret_template = Template(secret_file)
+    context = {
+        "access_key": b64encode(MINIO_CONFIG["access-key"].encode("utf-8")).decode("utf-8"),
+        "secret_key": b64encode(MINIO_CONFIG["secret-key"].encode("utf-8")).decode("utf-8"),
+        "namespace": ops_test.model_name,
+    }
+    rendered_secret_template = secret_template.render(**context)
+    for obj in codecs.load_all_yaml(rendered_secret_template):
+        try:
+            lightkube_client.apply(obj, field_manager="test")
+        except ApiError as e:
+            raise e
+
+
 async def create_artifact_bucket(ops_test: OpsTest):
     # Ensure bucket is available
     model_name = ops_test.model_name
@@ -123,6 +150,8 @@ async def submit_workflow_using_artifact(ops_test: OpsTest):
 
 
 async def test_workflow_using_artifacts(ops_test: OpsTest):
+    # Create the mlpipeline-minio-artifact secret
+    await create_mlpipeline_minio_secret(ops_test)
     # Argo will fail if the artifact bucket it uses does not exist
     await create_artifact_bucket(ops_test)
 

charms/argo-controller/tests/unit/test_charm.py

@@ -140,7 +140,7 @@ def test_kubernetes_created_method(
 
     # Assert
     # FIXME: why is it counting 50?
-    assert mocked_lightkube_client.apply.call_count == 50
+    assert mocked_lightkube_client.apply.call_count == 48
     assert isinstance(harness.charm.kubernetes_resources.status, ActiveStatus)