diff --git a/deployer/README.md b/deployer/README.md index 5bc4562c0b..141dabfa84 100644 --- a/deployer/README.md +++ b/deployer/README.md @@ -369,9 +369,9 @@ This allows us to validate that all required values are present and have the cor ### The `cilogon-client` sub-command for CILogon OAuth client management Deployer sub-command for managing CILogon clients for 2i2c hubs. -#### `cilogon-client create/delete/get/get-all/update` +#### `cilogon-client create/delete/get/get-all/update/cleanup` -create/delete/get/get-all/update/ CILogon clients using the 2i2c administrative client provided by CILogon. +create/delete/get/get-all/update/cleanup CILogon clients using the 2i2c administrative client provided by CILogon. ### The `exec` sub-command for executing shells and debugging commands diff --git a/deployer/commands/cilogon.py b/deployer/commands/cilogon.py index 0e20c5ff33..43e1fd4312 100644 --- a/deployer/commands/cilogon.py +++ b/deployer/commands/cilogon.py @@ -13,10 +13,12 @@ - `delete` a CILogon client application when a hub is removed or changes auth methods - `get` details about an existing hub CILogon client - `get-all` existing 2i2c CILogon client applications +- `cleanup` duplicated CILogon applications """ import base64 import json +from collections import Counter from pathlib import Path import requests @@ -27,6 +29,8 @@ from deployer.cli_app import cilogon_client_app from deployer.utils.file_acquisition import ( build_absolute_path_to_hub_encrypted_config_file, + find_absolute_path_to_cluster_file, + get_cluster_names_list, get_decrypted_file, persist_config_in_encrypted_file, remove_jupyterhub_hub_config_key_from_encrypted_file, @@ -264,52 +268,19 @@ def get_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None): return client_details -def delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None): - """Deletes the client associated with the id. - - Args: - id (str): Id of the client to delete +def delete_client(admin_id, admin_secret, client_id=None): + """Deletes a CILogon client. Returns status code if response.ok or None if the `delete` request returned a status code not in the range 200-299. See: https://github.com/ncsa/OA4MP/blob/HEAD/oa4mp-server-admin-oauth2/src/main/scripts/oidc-cm-scripts/cm-delete.sh """ - config_filename = build_absolute_path_to_hub_encrypted_config_file( - cluster_name, hub_name - ) - - if not client_id: - if Path(config_filename).is_file(): - client_id = load_client_id_from_file(config_filename) - # Nothing to do if no client has been found - if not client_id: - print_colour( - "No `client_id` to delete was provided and couldn't find any in `config_filename`", - "red", - ) - return - else: - print_colour( - f"No `client_id` to delete was provided and couldn't find any {config_filename} file", - "red", - ) - return + if client_id is None: + print("Deleting a CILogon client for unknown ID") else: - if not stored_client_id_same_with_cilogon_records( - admin_id, - admin_secret, - cluster_name, - hub_name, - client_id, - ): - print_colour( - "CILogon records are different than the client app stored in the configuration file. Consider updating the file.", - "red", - ) - return + print(f"Deleting the CILogon client details for {client_id}...") - print(f"Deleting the CILogon client details for {client_id}...") headers = build_request_headers(admin_id, admin_secret) response = requests.delete(build_request_url(client_id), headers=headers, timeout=5) if not response.ok: @@ -318,19 +289,6 @@ def delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None print_colour("Done!") - # Delete client credentials from config file also if file exists - if Path(config_filename).is_file(): - print(f"Deleting the CILogon client details from the {config_filename} also...") - key = "CILogonOAuthenticator" - try: - remove_jupyterhub_hub_config_key_from_encrypted_file(config_filename, key) - except KeyError: - print_colour(f"No {key} found to delete from {config_filename}", "yellow") - return - print_colour(f"CILogonAuthenticator config removed from {config_filename}") - if not Path(config_filename).is_file(): - print_colour(f"Empty {config_filename} file also deleted.", "yellow") - def get_all_clients(admin_id, admin_secret): print("Getting all existing CILogon client applications...") @@ -344,8 +302,67 @@ def get_all_clients(admin_id, admin_secret): return clients = response.json() - for c in clients["clients"]: - print(c) + return [c for c in clients["clients"]] + + +def find_duplicated_clients(clients): + """Determine duplicated CILogon clients by comparing client names + + Args: + clients (list[dict]): A list of dictionaries containing information about + the existing CILogon clients. Generated by get_all_clients function. + + Returns: + list: A list of duplicated client names + """ + client_names = [c["name"] for c in clients] + client_names_count = Counter(client_names) + return [k for k, v in client_names_count.items() if v > 1] + + +def find_orphaned_clients(clients): + """Find CILogon clients for which an associated cluster or hub no longer + exists and can safely be deleted. + + Args: + clients (list[dict]): A list of existing CILogon client info + + Returns: + list[dict]: A list of 'orphaned' CILogon clients which don't have an + associated cluster or hub, which can be deleted + """ + clients_to_be_deleted = [] + clusters = get_cluster_names_list() + + for client in clients: + cluster = next((cl for cl in clusters if cl in client["name"]), "") + + if cluster: + cluster_config_file = find_absolute_path_to_cluster_file(cluster) + with open(cluster_config_file) as f: + cluster_config = yaml.load(f) + + hub = next( + ( + hub["name"] + for hub in cluster_config["hubs"] + if hub["name"] in client["name"] + ), + "", + ) + + if not hub: + print( + f"A hub pertaining to client {client['name']} does NOT exist. Marking client for deletion." + ) + clients_to_be_deleted.append(client) + else: + print( + f"A cluster pertaining to client {client['name']} does NOT exist. Marking client for deletion." + ) + clients_to_be_deleted.append(client) + + return clients_to_be_deleted def get_2i2c_cilogon_admin_credentials(): @@ -415,7 +432,13 @@ def get( def get_all(): """Retrieve details about all existing 2i2c CILogon clients.""" admin_id, admin_secret = get_2i2c_cilogon_admin_credentials() - get_all_clients(admin_id, admin_secret) + clients = get_all_clients(admin_id, admin_secret) + for c in clients: + print(c) + + # Our plan with CILogon only permits 100 clients, so provide feedback on that + # number here. Change this if our plan updates. + print_colour(f"{len(clients)} / 100 clients used", "yellow") @cilogon_client_app.command() @@ -432,7 +455,123 @@ def delete( """, ), ): - """Delete an existing CILogon client. This deletes both the CILogon client application, - and the client credentials from the configuration file.""" + """ + Delete an existing CILogon client. This deletes both the CILogon client application, + and the client credentials from the configuration file. + """ + config_filename = build_absolute_path_to_hub_encrypted_config_file( + cluster_name, hub_name + ) admin_id, admin_secret = get_2i2c_cilogon_admin_credentials() + + if not client_id: + if Path(config_filename).is_file(): + client_id = load_client_id_from_file(config_filename) + # Nothing to do if no client has been found + if not client_id: + print_colour( + "No `client_id` to delete was provided and couldn't find any in `config_filename`", + "red", + ) + return + else: + print_colour( + f"No `client_id` to delete was provided and couldn't find any {config_filename} file", + "red", + ) + return + else: + if not stored_client_id_same_with_cilogon_records( + admin_id, + admin_secret, + cluster_name, + hub_name, + client_id, + ): + print_colour( + "CILogon records are different than the client app stored in the configuration file. Consider updating the file.", + "red", + ) + return + delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id) + + # Delete client credentials from config file also if file exists + if Path(config_filename).is_file(): + print(f"Deleting the CILogon client details from the {config_filename} also...") + key = "CILogonOAuthenticator" + try: + remove_jupyterhub_hub_config_key_from_encrypted_file(config_filename, key) + except KeyError: + print_colour(f"No {key} found to delete from {config_filename}", "yellow") + return + print_colour(f"CILogonAuthenticator config removed from {config_filename}") + if not Path(config_filename).is_file(): + print_colour(f"Empty {config_filename} file also deleted.", "yellow") + + +@cilogon_client_app.command() +def cleanup( + delete: bool = typer.Option( + False, help="Proceed with deleting duplicated CILogon apps" + ) +): + """Identify duplicated CILogon clients and which ID is being actively used in config, + and optionally delete unused duplicates. + + Args: + delete (bool, optional): Delete unused duplicate CILogon apps. Defaults to False. + """ + clients_to_be_deleted = [] + + admin_id, admin_secret = get_2i2c_cilogon_admin_credentials() + clients = get_all_clients(admin_id, admin_secret) + duplicated_clients = find_duplicated_clients(clients) + + # Cycle over each duplicated client name + for duped_client in duplicated_clients: + # Finds all the client IDs associated with a duplicated name + ids = [c["client_id"] for c in clients if c["name"] == duped_client] + + # Establish the cluster and hub name from the client name and build the + # absolute path to the encrypted hub values file + cluster_name, hub_name = duped_client.split("-") + config_filename = build_absolute_path_to_hub_encrypted_config_file( + cluster_name, hub_name + ) + + with get_decrypted_file(config_filename) as decrypted_path: + with open(decrypted_path) as f: + secret_config = yaml.load(f) + + if ( + "CILogonOAuthenticator" + not in secret_config["jupyterhub"]["hub"]["config"].keys() + ): + print( + f"Hub {hub_name} on cluster {cluster_name} doesn't use CILogonOAuthenticator." + ) + else: + # Extract the client ID *currently in use* from the encrypted config and remove it from the list of IDs + config_client_id = secret_config["jupyterhub"]["hub"]["config"][ + "CILogonOAuthenticator" + ]["client_id"] + ids.remove(config_client_id) + + clients_to_be_deleted.extend( + [{"client_name": duped_client, "client_id": id} for id in ids] + ) + + # Remove the duplicated clients from the client list + clients = [c for c in clients if c["name"] != duped_client] + + orphaned_clients = find_orphaned_clients(clients) + clients_to_be_deleted.extend(orphaned_clients) + + print_colour("CILogon clients to be deleted...") + for c in clients_to_be_deleted: + print(c) + + if delete: + for c in clients_to_be_deleted: + delete_client(admin_id, admin_secret, client_id=c["client_id"]) diff --git a/deployer/utils/file_acquisition.py b/deployer/utils/file_acquisition.py index f78ac8fb20..0f0a626f0a 100644 --- a/deployer/utils/file_acquisition.py +++ b/deployer/utils/file_acquisition.py @@ -248,3 +248,12 @@ def get_all_cluster_yaml_files(): for path in CONFIG_CLUSTERS_PATH.glob("**/cluster.yaml") if "templates" not in path.as_posix() } + + +def get_cluster_names_list(): + """ + Returns a list of all the clusters currently listed under config/clusters + """ + return [ + d.name for d, _, _ in CONFIG_CLUSTERS_PATH.walk() if "templates" not in str(d) + ]