Merge pull request 2i2c-org#4935 from sgibson91/cost-attribution/auto…

…mate-dashboard-deploy

Automate cost attribution grafana dashboard deployment

deployer/commands/grafana/deploy_dashboards.py

@@ -7,7 +7,7 @@
 from deployer.cli_app import grafana_app
 from deployer.utils.rendering import print_colour
 
-from .utils import get_grafana_token, get_grafana_url
+from .utils import get_cluster_cloud_provider, get_grafana_token, get_grafana_url
 
 
 @grafana_app.command()
@@ -30,31 +30,50 @@ def deploy_dashboards(
     """
     grafana_url = get_grafana_url(cluster_name)
     grafana_token = get_grafana_token(cluster_name)
+    cluster_provider = get_cluster_cloud_provider(cluster_name)
 
     print_colour("Cloning jupyterhub/grafana-dashboards...")
     subprocess.check_call(
         [
             "git",
             "clone",
             "https://github.com/jupyterhub/grafana-dashboards",
+            "jupyterhub-grafana-dashboards",
         ]
     )
 
     # Add GRAFANA_TOKEN to the environment variables for
     # jupyterhub/grafana-dashboards' deploy.py script
     deploy_script_env = os.environ.copy()
     deploy_script_env.update({"GRAFANA_TOKEN": grafana_token})
+
     try:
         print_colour(f"Deploying grafana dashboards to {cluster_name}...")
         subprocess.check_call(
             ["./deploy.py", grafana_url, f"--dashboards-dir={dashboards_dir}"],
             env=deploy_script_env,
-            cwd="grafana-dashboards",
+            cwd="jupyterhub-grafana-dashboards",
         )
+
+        if cluster_provider == "aws":
+            print_colour("Deploying cost attribution dashboards to an AWS cluster...")
+            subprocess.check_call(
+                [
+                    "./deploy.py",
+                    grafana_url,
+                    "--dashboards-dir=../grafana-dashboards",
+                    "--folder-name='Cost Attribution Dashboards'",
+                    "--folder-uid=aws-ce-grafana-backend",
+                ],
+                env=deploy_script_env,
+                cwd="jupyterhub-grafana-dashboards",
+            )
+
         print_colour(f"Done! Dashboards deployed to {grafana_url}.")
+
     finally:
         # Delete the directory where we cloned the repo.
         # The deployer cannot call jsonnet to deploy the dashboards if using a temp directory here.
         # Might be because opening more than once of a temp file is tried
         # (https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile)
-        shutil.rmtree("grafana-dashboards")
+        shutil.rmtree("jupyterhub-grafana-dashboards")

deployer/commands/grafana/utils.py

@@ -200,3 +200,20 @@ def update_central_grafana_token(cluster_name, token):
 
     # Encrypt the private key
     subprocess.check_call(["sops", "--in-place", "--encrypt", grafana_token_file])
+
+
+def get_cluster_cloud_provider(cluster_name):
+    """
+    Get the cloud provider a cluster is hosted on from its cluster.yaml file
+
+    Args:
+        cluster_name (str): name of the cluster
+        provider (str): name of the cloud provider the cluster is hosted on
+    """
+    cluster_config_dir_path = find_absolute_path_to_cluster_file(cluster_name).parent
+    config_filename = cluster_config_dir_path.joinpath("cluster.yaml")
+
+    with open(config_filename) as f:
+        cluster_config = yaml.load(f)
+
+    return cluster_config["provider"]

docs/howto/cost-attribution/aws.md

@@ -153,42 +153,15 @@ deployer deploy-support $CLUSTER_NAME
 
 ### 5. Deploy a Grafana dashboard
 
-```{important}
-We doesn't yet have no automation for this, and the dashboard definition will be
-duplicated every time we deploy because its not centralized. With that in mind,
-please treat the Grafana dashboard in https://grafana.openscapes.2i2c.cloud as
-the source of truth for now, and act as changes to dashboards in other cluster
-will get discarded.
+Use the deployer to deploy the Grafana dashboards:
+
+```bash
+deployer grafana deploy-dashboards $CLUSTER_NAME
 ```
 
-To manually deploy the dashboard we currently have:
-
-1. Login to https://grafana.openscapes.2i2c.cloud as the admin user
-2. Visit [openscapes's dashboard settings] and save the JSON blob to a text
-   editor
-3. Login to Grafana instance of the cluster you're working with now as the admin
-   user
-4. Navigate to the `plugins/yesoreyeram-infinity-datasource` path (Plugins -> Search for infinity)
-5. First press the "Install" button, and then press the "Add new datasource"
-   button
-6. You'll arrive at a page with a path like
-   `connections/datasources/edit/ddyyd0z19aznke`, where the last part is an ID
-   of the datasource you just added. Copy that id, which in this example is
-   `ddyyd0z19aznke`.
-7. In the text editor with the JSON blob from openscapes dashboard definition,
-   find all references to the old infinity datasource ID listed under `uid`, and
-   replace all those with the ID you copied just before.
-8. Navigate to the `/dashboard/import` path (Dashboards -> New dashboard ->
-   Import dashboard)
-9. Paste the JSON blob you updated in the text editor and create a new dashboard
-
-The same dashboard definition is besides updating the datasource ID expected to
-work out of the box, because there is nothing hardcoded to the openscapes
-cluster within it. It simply reads data from
-`aws-ce-grafana-backend.support.svc.cluster.local`, which is a k8s cluster local
-address to the k8s Service `aws-ce-grafana-backend` in the `support` namespace.
-
-[openscapes's dashboard settings]: https://grafana.openscapes.2i2c.cloud/d/edw06h7udjwg0b/cloud-cost-attribution?orgId=1&editview=dashboard_json
+```{warning}
+Running this command will overwrite any changes or new dashboards made via the UI.
+```
 
 ## Troubleshooting