From 83251c668ede439bcc9ca6c5ccf3c9e79a583ad0 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 3 Oct 2024 16:40:06 +0100 Subject: [PATCH 1/5] Write a function that reads which cloud a cluster is on from cluster.yaml --- deployer/commands/grafana/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/deployer/commands/grafana/utils.py b/deployer/commands/grafana/utils.py index a4f535efa4..42a518f4af 100644 --- a/deployer/commands/grafana/utils.py +++ b/deployer/commands/grafana/utils.py @@ -200,3 +200,20 @@ def update_central_grafana_token(cluster_name, token): # Encrypt the private key subprocess.check_call(["sops", "--in-place", "--encrypt", grafana_token_file]) + + +def get_cluster_cloud_provider(cluster_name): + """ + Get the cloud provider a cluster is hosted on from its cluster.yaml file + + Args: + cluster_name (str): name of the cluster + provider (str): name of the cloud provider the cluster is hosted on + """ + cluster_config_dir_path = find_absolute_path_to_cluster_file(cluster_name).parent + config_filename = cluster_config_dir_path.joinpath("cluster.yaml") + + with open(config_filename) as f: + cluster_config = yaml.load(f) + + return cluster_config["provider"] From 228d26e72238626dd0941a302ab5429acce228d3 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 3 Oct 2024 16:42:20 +0100 Subject: [PATCH 2/5] Add scaffolding code to install cost attribution dashboards if the cluster is on AWS --- deployer/commands/grafana/deploy_dashboards.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deployer/commands/grafana/deploy_dashboards.py b/deployer/commands/grafana/deploy_dashboards.py index f30e1f306c..04c2e1de2d 100644 --- a/deployer/commands/grafana/deploy_dashboards.py +++ b/deployer/commands/grafana/deploy_dashboards.py @@ -7,7 +7,7 @@ from deployer.cli_app import grafana_app from deployer.utils.rendering import print_colour -from .utils import get_grafana_token, get_grafana_url +from .utils import get_cluster_cloud_provider, get_grafana_token, get_grafana_url @grafana_app.command() @@ -30,6 +30,7 @@ def deploy_dashboards( """ grafana_url = get_grafana_url(cluster_name) grafana_token = get_grafana_token(cluster_name) + cluster_provider = get_cluster_cloud_provider(cluster_name) print_colour("Cloning jupyterhub/grafana-dashboards...") subprocess.check_call( @@ -58,3 +59,8 @@ def deploy_dashboards( # Might be because opening more than once of a temp file is tried # (https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile) shutil.rmtree("grafana-dashboards") + + if cluster_provider == "aws": + # FIXME: Add code to deploy cost attribution dashboards here + pass + \ No newline at end of file From b7626c4472d1a2dd196e5c860856dc9addee4dea Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 4 Oct 2024 14:54:21 +0100 Subject: [PATCH 3/5] Rename folder jupyterhub/grafana-dashboards is cloned into Now there is a grafana-dashboards folder in the infrastructure repository, the git clone command will fail trying to clone into an existing, non-empty folder --- deployer/commands/grafana/deploy_dashboards.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployer/commands/grafana/deploy_dashboards.py b/deployer/commands/grafana/deploy_dashboards.py index 04c2e1de2d..0acaae962b 100644 --- a/deployer/commands/grafana/deploy_dashboards.py +++ b/deployer/commands/grafana/deploy_dashboards.py @@ -38,6 +38,7 @@ def deploy_dashboards( "git", "clone", "https://github.com/jupyterhub/grafana-dashboards", + "jupyterhub-grafana-dashboards", ] ) @@ -50,7 +51,7 @@ def deploy_dashboards( subprocess.check_call( ["./deploy.py", grafana_url, f"--dashboards-dir={dashboards_dir}"], env=deploy_script_env, - cwd="grafana-dashboards", + cwd="jupyterhub-grafana-dashboards", ) print_colour(f"Done! Dashboards deployed to {grafana_url}.") finally: @@ -58,7 +59,7 @@ def deploy_dashboards( # The deployer cannot call jsonnet to deploy the dashboards if using a temp directory here. # Might be because opening more than once of a temp file is tried # (https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile) - shutil.rmtree("grafana-dashboards") + shutil.rmtree("jupyterhub-grafana-dashboards") if cluster_provider == "aws": # FIXME: Add code to deploy cost attribution dashboards here From 116cad0fa77dcc4f6be2fee0f9c8f2d8844240b1 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 4 Oct 2024 15:52:43 +0100 Subject: [PATCH 4/5] Add the command to deploy the cost attribution dashboards to AWS clusters --- .../commands/grafana/deploy_dashboards.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/deployer/commands/grafana/deploy_dashboards.py b/deployer/commands/grafana/deploy_dashboards.py index 0acaae962b..8ceb31ed25 100644 --- a/deployer/commands/grafana/deploy_dashboards.py +++ b/deployer/commands/grafana/deploy_dashboards.py @@ -46,6 +46,7 @@ def deploy_dashboards( # jupyterhub/grafana-dashboards' deploy.py script deploy_script_env = os.environ.copy() deploy_script_env.update({"GRAFANA_TOKEN": grafana_token}) + try: print_colour(f"Deploying grafana dashboards to {cluster_name}...") subprocess.check_call( @@ -53,15 +54,26 @@ def deploy_dashboards( env=deploy_script_env, cwd="jupyterhub-grafana-dashboards", ) + + if cluster_provider == "aws": + print_colour("Deploying cost attribution dashboards to an AWS cluster...") + subprocess.check_call( + [ + "./deploy.py", + grafana_url, + "--dashboards-dir=../grafana-dashboards", + "--folder-name='Cost Attribution Dashboards'", + "--folder-uid=aws-ce-grafana-backend", + ], + env=deploy_script_env, + cwd="jupyterhub-grafana-dashboards", + ) + print_colour(f"Done! Dashboards deployed to {grafana_url}.") + finally: # Delete the directory where we cloned the repo. # The deployer cannot call jsonnet to deploy the dashboards if using a temp directory here. # Might be because opening more than once of a temp file is tried # (https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile) shutil.rmtree("jupyterhub-grafana-dashboards") - - if cluster_provider == "aws": - # FIXME: Add code to deploy cost attribution dashboards here - pass - \ No newline at end of file From f7ee00f1c68b9e9f6ffe385748a2b5541fdf3665 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 4 Oct 2024 16:04:16 +0100 Subject: [PATCH 5/5] Update cost attribution docs to use the deployer to install the grafana dashboards --- docs/howto/cost-attribution/aws.md | 41 +++++------------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/docs/howto/cost-attribution/aws.md b/docs/howto/cost-attribution/aws.md index 140b615b01..3d53e565f2 100644 --- a/docs/howto/cost-attribution/aws.md +++ b/docs/howto/cost-attribution/aws.md @@ -153,42 +153,15 @@ deployer deploy-support $CLUSTER_NAME ### 5. Deploy a Grafana dashboard -```{important} -We doesn't yet have no automation for this, and the dashboard definition will be -duplicated every time we deploy because its not centralized. With that in mind, -please treat the Grafana dashboard in https://grafana.openscapes.2i2c.cloud as -the source of truth for now, and act as changes to dashboards in other cluster -will get discarded. +Use the deployer to deploy the Grafana dashboards: + +```bash +deployer grafana deploy-dashboards $CLUSTER_NAME ``` -To manually deploy the dashboard we currently have: - -1. Login to https://grafana.openscapes.2i2c.cloud as the admin user -2. Visit [openscapes's dashboard settings] and save the JSON blob to a text - editor -3. Login to Grafana instance of the cluster you're working with now as the admin - user -4. Navigate to the `plugins/yesoreyeram-infinity-datasource` path (Plugins -> Search for infinity) -5. First press the "Install" button, and then press the "Add new datasource" - button -6. You'll arrive at a page with a path like - `connections/datasources/edit/ddyyd0z19aznke`, where the last part is an ID - of the datasource you just added. Copy that id, which in this example is - `ddyyd0z19aznke`. -7. In the text editor with the JSON blob from openscapes dashboard definition, - find all references to the old infinity datasource ID listed under `uid`, and - replace all those with the ID you copied just before. -8. Navigate to the `/dashboard/import` path (Dashboards -> New dashboard -> - Import dashboard) -9. Paste the JSON blob you updated in the text editor and create a new dashboard - -The same dashboard definition is besides updating the datasource ID expected to -work out of the box, because there is nothing hardcoded to the openscapes -cluster within it. It simply reads data from -`aws-ce-grafana-backend.support.svc.cluster.local`, which is a k8s cluster local -address to the k8s Service `aws-ce-grafana-backend` in the `support` namespace. - -[openscapes's dashboard settings]: https://grafana.openscapes.2i2c.cloud/d/edw06h7udjwg0b/cloud-cost-attribution?orgId=1&editview=dashboard_json +```{warning} +Running this command will overwrite any changes or new dashboards made via the UI. +``` ## Troubleshooting