Add logging relation kubeflow-dashboard (#204)

Added logging relation and cos integration tests from chisme.

fixes: #195

metadata.yaml

@@ -75,3 +75,6 @@ requires:
           - service-port
     versions: [v1]
     __schema_source: https://raw.githubusercontent.com/canonical/operator-schemas/master/k8s-service.yaml
+  logging:
+    interface: loki_push_api
+    optional: true

requirements-integration.in

@@ -9,6 +9,8 @@ pytest-operator
 pyyaml
 selenium
 selenium-wire
+# This is required due to the abstraction of cos integration
+charmed-kubeflow-chisme>=0.4.0
 # This is needed only for this charm as the integration test files in
 # test/integration/ have imports from the charm code in src/charm.py
 -r requirements.in

requirements-integration.txt

@@ -44,10 +44,14 @@ cffi==1.15.1
     # via
     #   cryptography
     #   pynacl
-charmed-kubeflow-chisme==0.2.0
-    # via -r requirements.in
+charmed-kubeflow-chisme==0.4.0
+    # via
+    #   -r requirements-integration.in
+    #   -r requirements.in
 charset-normalizer==3.2.0
     # via requests
+cosl==0.0.12
+    # via -r requirements.in
 cryptography==41.0.3
     # via
     #   paramiko
@@ -116,6 +120,7 @@ jsonschema==4.17.3
 juju==3.2.2
     # via
     #   -r requirements-integration.in
+    #   charmed-kubeflow-chisme
     #   pytest-operator
 kaitaistruct==0.10
     # via selenium-wire
@@ -149,6 +154,7 @@ ops==2.14.0
     #   -r requirements-integration.in
     #   -r requirements.in
     #   charmed-kubeflow-chisme
+    #   cosl
     #   serialized-data-interface
 ordered-set==4.1.0
     # via deepdiff
@@ -230,6 +236,7 @@ pyyaml==6.0.1
     # via
     #   -r requirements-integration.in
     #   -r requirements.in
+    #   cosl
     #   juju
     #   kubernetes
     #   lightkube
@@ -249,7 +256,7 @@ rsa==4.9
     # via google-auth
 ruamel-yaml==0.17.32
     # via charmed-kubeflow-chisme
-ruamel-yaml-clib==0.2.7
+ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
 selenium==4.12.0
     # via
@@ -300,6 +307,7 @@ trio-websocket==0.10.4
     # via selenium
 typing-extensions==4.7.1
     # via
+    #   cosl
     #   ipython
     #   typing-inspect
 typing-inspect==0.9.0

requirements-unit.txt

@@ -19,6 +19,8 @@ charmed-kubeflow-chisme==0.2.0
     #   -r requirements.in
 charset-normalizer==3.2.0
     # via requests
+cosl==0.0.12
+    # via -r requirements.in
 coverage==7.3.0
     # via -r requirements-unit.in
 deepdiff==6.2.1
@@ -60,6 +62,7 @@ ops==2.14.0
     #   -r requirements-unit.in
     #   -r requirements.in
     #   charmed-kubeflow-chisme
+    #   cosl
     #   serialized-data-interface
 ordered-set==4.1.0
     # via deepdiff
@@ -84,14 +87,15 @@ pyyaml==6.0.1
     # via
     #   -r requirements-unit.in
     #   -r requirements.in
+    #   cosl
     #   lightkube
     #   ops
     #   serialized-data-interface
 requests==2.31.0
     # via serialized-data-interface
 ruamel-yaml==0.17.32
     # via charmed-kubeflow-chisme
-ruamel-yaml-clib==0.2.7
+ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
 serialized-data-interface==0.7.0
     # via
@@ -106,6 +110,8 @@ tenacity==8.2.2
     # via charmed-kubeflow-chisme
 tomli==2.0.1
     # via pytest
+typing-extensions==4.12.2
+    # via cosl
 urllib3==2.0.4
     # via requests
 websocket-client==1.6.1

requirements.in

@@ -5,3 +5,5 @@ lightkube
 ops
 pyyaml
 serialized-data-interface
+# from loki_k8s.v1.loki_push_api.py
+cosl

requirements.txt

@@ -17,6 +17,8 @@ charmed-kubeflow-chisme==0.2.0
     # via -r requirements.in
 charset-normalizer==3.2.0
     # via requests
+cosl==0.0.12
+    # via -r requirements.in
 deepdiff==6.2.1
     # via charmed-kubeflow-chisme
 exceptiongroup==1.1.2
@@ -50,6 +52,7 @@ ops==2.14.0
     # via
     #   -r requirements.in
     #   charmed-kubeflow-chisme
+    #   cosl
     #   serialized-data-interface
 ordered-set==4.1.0
     # via deepdiff
@@ -60,6 +63,7 @@ pyrsistent==0.19.3
 pyyaml==6.0.1
     # via
     #   -r requirements.in
+    #   cosl
     #   lightkube
     #   ops
     #   serialized-data-interface
@@ -80,6 +84,8 @@ sniffio==1.3.0
     #   httpx
 tenacity==8.2.2
     # via charmed-kubeflow-chisme
+typing-extensions==4.12.2
+    # via cosl
 urllib3==2.0.4
     # via requests
 websocket-client==1.6.1

src/charm.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 import json
@@ -13,6 +13,7 @@
     DASHBOARD_LINK_LOCATIONS,
     KubeflowDashboardLinksProvider,
 )
+from charms.loki_k8s.v1.loki_push_api import LogForwarder
 from charms.observability_libs.v1.kubernetes_service_patch import KubernetesServicePatch
 from lightkube import ApiError
 from lightkube.generic_resource import load_in_cluster_generic_resources
@@ -93,6 +94,7 @@ def __init__(self, *args):
             relation_name=DASHBOARD_LINKS_RELATION_NAME,
         )
         self.framework.observe(self.dashboard_link_provider.on.updated, self.main)
+        self._logging = LogForwarder(charm=self)
 
     @property
     def profiles_service(self):
@@ -169,10 +171,12 @@ def _kubeflow_dashboard_operator_layer(self) -> Layer:
                         "USERID_HEADER": "kubeflow-userid",
                         "USERID_PREFIX": "",
                         "PROFILES_KFAM_SERVICE_HOST": f"{self.profiles_service}.{self.model.name}",
-                        "REGISTRATION_FLOW": self._registration_flow,
+                        "REGISTRATION_FLOW": str(
+                            self._registration_flow
+                        ).lower(),  # convert to a string because the applied layer will have it  # noqa E501
                         "DASHBOARD_CONFIGMAP": self._configmap_name,
                         "LOGOUT_URL": "/authservice/logout",
-                        "POD_NAMESPACE": self.model.name,  # Added due to https://github.com/canonical/bundle-kubeflow/issues/698   # noqa E501
+                        "POD_NAMESPACE": self.model.name,  # Added due to https://github.com/canonical/bundle-kubeflow/issues/698  # noqa E501
                     },
                 }
             },

tests/integration/test_charm.py

@@ -11,6 +11,11 @@
 import pytest
 import pytest_asyncio
 import yaml
+from charmed_kubeflow_chisme.testing import (
+    GRAFANA_AGENT_APP,
+    assert_logging,
+    deploy_and_assert_grafana_agent,
+)
 from charms.kubeflow_dashboard.v0.kubeflow_dashboard_links import (
     DASHBOARD_LINK_LOCATIONS,
     DashboardLink,
@@ -32,6 +37,7 @@
     "tests/integration/dashboard_links_requirer_tester_charm"
 ).absolute()
 TESTER_CHARM_NAME = "kubeflow-dashboard-requirer-tester"
+TESTER_CHARMS = [f"{TESTER_CHARM_NAME}{suffix}" for suffix in ["1", "2"]]
 
 DEFAULT_DOCUMENTATION_TEXTS = [
     "Getting started with Charmed Kubeflow",
@@ -73,12 +79,17 @@ async def test_build_and_deploy(ops_test: OpsTest):
         == "Add required relation to kubeflow-profiles"
     )
 
+    # Deploying grafana-agent-k8s and add all relations
+    await deploy_and_assert_grafana_agent(
+        ops_test.model, CHARM_NAME, metrics=False, dashboard=False, logging=True
+    )
+
 
 @pytest.mark.asyncio
 @pytest.mark.abort_on_fail
 async def test_add_profile_relation(ops_test: OpsTest):
     await ops_test.model.deploy(PROFILES_CHARM_NAME, channel="latest/edge", trust=True)
-    await ops_test.model.relate(PROFILES_CHARM_NAME, CHARM_NAME)
+    await ops_test.model.integrate(PROFILES_CHARM_NAME, CHARM_NAME)
     await ops_test.model.wait_for_idle(
         [PROFILES_CHARM_NAME, CHARM_NAME],
         status="active",
@@ -138,8 +149,7 @@ async def test_configmap_contents_with_relations(
 
     expected_links = {name: [] for name in DASHBOARD_LINK_LOCATIONS}
 
-    for tester_suffix in ["1", "2"]:
-        tester = f"{TESTER_CHARM_NAME}{tester_suffix}"
+    for tester in TESTER_CHARMS:
         await ops_test.model.deploy(charm, application_name=tester)
         for location in ["menu", "documentation"]:
             link_texts = [location]
@@ -149,12 +159,13 @@ async def test_configmap_contents_with_relations(
             )
             expected_links[location].extend(these_links)
 
-        await ops_test.model.relate(CHARM_NAME, tester)
+        await ops_test.model.integrate(CHARM_NAME, tester)
 
     # Wait for everything to settle
     await ops_test.model.wait_for_idle(
+        apps=[CHARM_NAME, PROFILES_CHARM_NAME, *TESTER_CHARMS],
         raise_on_error=True,
-        raise_on_blocked=True,
+        raise_on_blocked=False,  # grafana-agent-k8s is expected to be blocked
         status="active",
         timeout=150,
     )
@@ -184,8 +195,9 @@ async def test_configmap_contents_with_menu_links_from_config(
 
     # Wait for everything to settle
     await ops_test.model.wait_for_idle(
+        apps=[CHARM_NAME, PROFILES_CHARM_NAME, *TESTER_CHARMS],
         raise_on_error=True,
-        raise_on_blocked=True,
+        raise_on_blocked=False,  # grafana-agent-k8s is expected to be blocked
         status="active",
         timeout=150,
     )
@@ -227,8 +239,9 @@ async def test_configmap_contents_with_menu_links_from_config(
 
     # Wait for everything to settle
     await ops_test.model.wait_for_idle(
+        apps=[CHARM_NAME, PROFILES_CHARM_NAME, *TESTER_CHARMS],
         raise_on_error=True,
-        raise_on_blocked=True,
+        raise_on_blocked=False,  # grafana-agent-k8s is expected to be blocked
         status="active",
         timeout=150,
     )
@@ -267,8 +280,9 @@ async def test_configmap_contents_with_ordering(ops_test: OpsTest, lightkube_cli
 
     # Wait for everything to settle
     await ops_test.model.wait_for_idle(
+        apps=[CHARM_NAME, PROFILES_CHARM_NAME, *TESTER_CHARMS],
         raise_on_error=True,
-        raise_on_blocked=True,
+        raise_on_blocked=False,  # grafana-agent-k8s is expected to be blocked
         status="active",
         timeout=150,
     )
@@ -340,3 +354,9 @@ async def test_dashboard_access(ops_test: OpsTest, lightkube_client: Client):
     assert result_status == 200
     # And that the title is the one expected
     assert "<title>Kubeflow Central Dashboard</title>" in result_text
+
+
+async def test_logging(ops_test: OpsTest):
+    """Test logging is defined in relation data bag."""
+    app = ops_test.model.applications[GRAFANA_AGENT_APP]
+    await assert_logging(app)

tests/unit/test_operator.py

@@ -104,6 +104,12 @@ def harness_with_profiles(harness: Harness) -> Harness:
 
 
 class TestCharm:
+    @patch("charm.KubernetesServicePatch", lambda x, y: None)
+    def test_log_forwarding(self, harness: Harness):
+        with patch("charm.LogForwarder") as mock_logging:
+            harness.begin()
+            mock_logging.assert_called_once_with(charm=harness.charm)
+
     @patch("charm.KubernetesServicePatch", lambda x, y: None)
     def test_check_leader_failure(self, harness: Harness):
         harness.begin_with_initial_hooks()