From 7dbbe29fdfbdeb93d3ca78067fa460914a0efddf Mon Sep 17 00:00:00 2001 From: Parag Kamble Date: Tue, 28 May 2024 15:51:28 +0530 Subject: [PATCH] Added Azure KMS support in cluster deployment (#9731) Signed-off-by: Parag Kamble --- ...i_1az_rhcos_3m_3w_encryption_azure_kv.yaml | 21 ++ ocs_ci/ocs/constants.py | 14 + .../csi-kms-connection-details-azurekv.yaml | 6 + .../azurekv/azure-client-secrets.yaml | 8 + .../azurekv/ocs-kms-connection-details.yaml | 12 + ocs_ci/utility/kms.py | 267 +++++++++++++++++- setup.py | 1 + 7 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 conf/deployment/azure/ipi_1az_rhcos_3m_3w_encryption_azure_kv.yaml create mode 100644 ocs_ci/templates/CSI/rbd/csi-kms-connection-details-azurekv.yaml create mode 100644 ocs_ci/templates/openshift-infra/azurekv/azure-client-secrets.yaml create mode 100644 ocs_ci/templates/openshift-infra/azurekv/ocs-kms-connection-details.yaml diff --git a/conf/deployment/azure/ipi_1az_rhcos_3m_3w_encryption_azure_kv.yaml b/conf/deployment/azure/ipi_1az_rhcos_3m_3w_encryption_azure_kv.yaml new file mode 100644 index 00000000000..4dd16140640 --- /dev/null +++ b/conf/deployment/azure/ipi_1az_rhcos_3m_3w_encryption_azure_kv.yaml @@ -0,0 +1,21 @@ +--- +DEPLOYMENT: + openshift_install_timeout: 4800 + allow_lower_instance_requirements: false + kms_deployment: true +ENV_DATA: + platform: 'azure' + deployment_type: 'ipi' + region: 'eastus' + azure_base_domain_resource_group_name: 'odfqe' + worker_availability_zones: + - '1' + master_availability_zones: + - '1' + worker_replicas: 3 + master_replicas: 3 + master_instance_type: 'Standard_D8s_v3' + worker_instance_type: 'Standard_D16s_v3' + encryption_at_rest: true + sc_encryption: true + KMS_PROVIDER: azure-kv diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 5c62e50c155..889e750b6dc 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -975,6 +975,20 @@ KMIP_OCS_KMS_SECRET = os.path.join(KMIP_KMS_TEMPLATES, "thales-kmip-ocs-secret.yaml") KMIP_CSI_KMS_SECRET = os.path.join(TEMPLATE_CSI_RBD_DIR, "thales-kmip-csi-secret.yaml") +# Azure KV KMS yamls +AZURE_KV_PROVIDER_NAME = "azure-kv" +AZURE_KV_CSI_CONNECTION_DETAILS = "csi-kms-connection-details" +AZURE_KV_CONNECTION_DETAILS_RESOURCE = "ocs-kms-connection-details" +AZURE_KV_TEMPLATES = os.path.join(TEMPLATE_OPENSHIFT_INFRA_DIR, "azurekv") +AZURE_OCS_KMS_CONNECTION_DETAILS = os.path.join( + AZURE_KV_TEMPLATES, "ocs-kms-connection-details.yaml" +) +AZURE_CSI_KMS_CONNECTION_DETAILS = os.path.join( + TEMPLATE_CSI_RBD_DIR, "csi-kms-connection-details-azurekv.yaml" +) +AZURE_CLIENT_SECRETS = os.path.join(AZURE_KV_TEMPLATES, "azure-client-secrets.yaml") + + # Multicluster related yamls ODF_MULTICLUSTER_ORCHESTRATOR = os.path.join( TEMPLATE_MULTICLUSTER_DIR, "odf_multicluster_orchestrator.yaml" diff --git a/ocs_ci/templates/CSI/rbd/csi-kms-connection-details-azurekv.yaml b/ocs_ci/templates/CSI/rbd/csi-kms-connection-details-azurekv.yaml new file mode 100644 index 00000000000..bd41aca31c6 --- /dev/null +++ b/ocs_ci/templates/CSI/rbd/csi-kms-connection-details-azurekv.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +data: +kind: ConfigMap +metadata: + name: csi-kms-connection-details + namespace: openshift-storage diff --git a/ocs_ci/templates/openshift-infra/azurekv/azure-client-secrets.yaml b/ocs_ci/templates/openshift-infra/azurekv/azure-client-secrets.yaml new file mode 100644 index 00000000000..9e854381c20 --- /dev/null +++ b/ocs_ci/templates/openshift-infra/azurekv/azure-client-secrets.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: + namespace: +data: + CLIENT_CERT: +type: Opaque diff --git a/ocs_ci/templates/openshift-infra/azurekv/ocs-kms-connection-details.yaml b/ocs_ci/templates/openshift-infra/azurekv/ocs-kms-connection-details.yaml new file mode 100644 index 00000000000..f5fa810fdf0 --- /dev/null +++ b/ocs_ci/templates/openshift-infra/azurekv/ocs-kms-connection-details.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ocs-kms-connection-details + namespace: openshift-storage +data: + AZURE_CERT_SECRET_NAME: + AZURE_CLIENT_ID: + AZURE_TENANT_ID: + AZURE_VAULT_URL: + KMS_PROVIDER: + KMS_SERVICE_NAME: diff --git a/ocs_ci/utility/kms.py b/ocs_ci/utility/kms.py index 3f6dda855f7..805fd964503 100644 --- a/ocs_ci/utility/kms.py +++ b/ocs_ci/utility/kms.py @@ -48,6 +48,11 @@ encode, prepare_bin_dir, ) +from fauxfactory import gen_alphanumeric +from azure.identity import CertificateCredential +from azure.keyvault.secrets import SecretClient +from azure.core.exceptions import AzureError +from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs logger = logging.getLogger(__name__) @@ -1797,7 +1802,265 @@ def cleanup(self): logger.info("Keys deleted from CipherTrust Manager") -kms_map = {"vault": Vault, "hpcs": HPCS, "kmip": KMIP} +class AzureKV(KMS): + """ + Represents an Azure Key Vault implementation of KMS. + """ + + def __init__(self, namespace=config.ENV_DATA["cluster_namespace"]): + super().__init__(constants.AZURE_KV_PROVIDER_NAME) + self.namespace = namespace + self.kms_provider = constants.AZURE_KV_PROVIDER_NAME + self.azure_kms_connection_name = ( + f"azure-kv-conn-{gen_alphanumeric(length=5).lower()}" + ) + azure_auth = config.AUTH.get("azure_auth") + self.azure_kv_name = azure_auth.get("AZURE_KV_NAME") + self.azure_kv_certificate = azure_auth.get("AZURE_CERTIFICATE") + self.vault_url = azure_auth.get("AZURE_KV_URL") + self.vault_client_id = azure_auth.get("AZURE_KV_CLIENT_ID") + self.vault_tenant_id = azure_auth.get("AZURE_KV_TENANT_ID") + self.vault_cert_path = self._azure_kv_cert_path() + + self.conn_data = { + "KMS_PROVIDER": self.kms_provider, + "KMS_SERVICE_NAME": self.azure_kms_connection_name, + "AZURE_CLIENT_ID": self.vault_client_id, + "AZURE_VAULT_URL": self.vault_url, + "AZURE_TENANT_ID": self.vault_tenant_id, + } + + def deploy(self): + """ + This Function will create the Azure KV connection details in the ConfigMap. + """ + if not config.ENV_DATA.get("platform") == "azure": + raise VaultDeploymentError( + "Azure_KV deployment only supports on Azure platform." + ) + + self.create_azure_kv_csi_kms_connection_details() + if config.ENV_DATA.get("encryption_at_rest"): + self.create_azure_kv_ocs_csi_kms_connection_details() + + def post_deploy_verification(self): + """ + Post Deploy Verification For Azure Key Vault. + """ + if config.ENV_DATA.get("encryption_at_rest"): + if not self.verify_osd_keys_present_on_azure_kv(): + raise ValueError("OSD keys Not present on Azure Key Vault.") + logger.info("OSD Keys Are present on Azure Key Vault.") + + def is_azure_kv_connection_exists(self): + """ + Checks if the Azure KV connection exists in the ConfigMap + """ + + csi_kms_configmap = ocp.OCP( + kind=constants.CONFIGMAP, + resource_name=constants.VAULT_KMS_CSI_CONNECTION_DETAILS, + namespace=self.namespace, + ) + + if not csi_kms_configmap.is_exist(): + raise ValueError( + f"ConfigMap {csi_kms_configmap.resource_name} Not found in the namespace {self.namespace}" + ) + + if self.azure_kms_connection_name not in csi_kms_configmap.data["data"]: + raise ValueError( + f"Azure Key vault connection {self.azure_kms_connection_name} not exists." + ) + + def create_azure_kv_secrets(self, prefix="azure-ocs-"): + """ + Creates Azure KV secrets. + """ + secret_name = gen_alphanumeric(length=18, start=prefix).lower() + client_secret = templating.load_yaml(constants.AZURE_CLIENT_SECRETS) + + client_secret["metadata"]["name"] = secret_name + client_secret["metadata"]["namespace"] = self.namespace + client_secret["data"]["CLIENT_CERT"] = base64.b64encode( + self.azure_kv_certificate.encode() + ).decode() + logger.info(f"Creating a Azure Secret : {secret_name}") + self.create_resource(client_secret, prefix=prefix) + return secret_name + + def create_azure_kv_csi_kms_connection_details(self): + """ + Create Azure specific csi-kms-connection-details + configmap resource + """ + + # Check is already configmap exists + csi_kms_configmap = ocp.OCP( + kind=constants.CONFIGMAP, + resource_name=constants.AZURE_KV_CSI_CONNECTION_DETAILS, + namespace=self.namespace, + ) + + # Create a Connection data. + azure_conn = self.conn_data + azure_conn["AZURE_CERT_SECRET_NAME"] = self.create_azure_kv_secrets( + prefix="azure-csi-" + ) + + if not csi_kms_configmap.is_exist(): + logger.info( + f"Creating Configmap {constants.AZURE_KV_CSI_CONNECTION_DETAILS}" + ) + + csi_kms_conn_details = templating.load_yaml( + constants.AZURE_CSI_KMS_CONNECTION_DETAILS + ) + + # Updating Templet data. + csi_kms_conn_details["data"] = { + self.azure_kms_connection_name: json.dumps(azure_conn) + } + + csi_kms_conn_details["metadata"]["namespace"] = self.namespace + self.create_resource(csi_kms_conn_details, prefix="csiazureconn") + else: + # Append the connection details to existing ConfigMap. + logger.info( + f"Adding Azure connection to existing ConfigMap {constants.AZURE_KV_CSI_CONNECTION_DETAILS}" + ) + param = ( + f'[{{"op": "add", "path": "/data/{self.azure_kms_connection_name}", ' + f'"value": "{json.dumps(azure_conn)}"}}]' + ) + csi_kms_configmap.patch(params=param, format_type="merge") + + # verifying ConfigMap is created or not. + self.is_azure_kv_connection_exists() + + def create_azure_kv_ocs_csi_kms_connection_details(self): + """ + Creates Azure KV OCS CSI KMS connection details ConfigMap. + """ + + # Creating ConfigMap for OCS CSI KMS connection details. + azure_data = self.conn_data + azure_data["AZURE_CERT_SECRET_NAME"] = self.create_azure_kv_secrets( + prefix="azure-ocs-" + ) + + # loading ConfigMap template + ocs_kms_conn_details = templating.load_yaml( + constants.AZURE_OCS_KMS_CONNECTION_DETAILS + ) + ocs_kms_conn_details["metadata"]["namespace"] = self.namespace + ocs_kms_conn_details["data"] = azure_data + + # creating ConfigMap Rsource + logger.info( + f"creating ConfigMap resource for {constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE}" + ) + self.create_resource(ocs_kms_conn_details, prefix="ocsazureconn") + + # Verify ConfigMap is created or not. + ocs_kms_configmap = ocp.OCP( + kind=constants.CONFIGMAP, + resource_name=constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE, + namespace=self.namespace, + ) + + if not ocs_kms_configmap.is_exist(): + raise ValueError( + f"ConfigMap Resource {constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE}" + f" is not created in namespace {self.namespace}" + ) + + logger.info( + f"Successfully Created configmap {constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE} " + f"in {self.namespace} namespace" + ) + + def _azure_kv_cert_path(self): + """ + Create a temporary certificate file and write the Azure Key Vault certificate to it. + """ + try: + temp_dir = tempfile.mkdtemp() + cert_file = os.path.join(temp_dir, "certificate.pem") + + with open(cert_file, "w") as fd: + fd.write(self.azure_kv_certificate) + + return cert_file + except Exception as ex: + raise ValueError(f"Error Creating Azure certificate file : {ex}") + + def azure_kv_secrets(self): + """ + List the secrets in the Azure Key Vault. + """ + try: + # Create a CertificateCredential using the certificate + credential = CertificateCredential( + vault_url=self.vault_url, + tenant_id=self.vault_tenant_id, + client_id=self.vault_client_id, + certificate_path=self.vault_cert_path, + ) + + # Create a SecretClient using the certificate for authentication + secret_client = SecretClient( + vault_url=self.vault_url, credential=credential + ) + + # Get the list of secrets + secrets = secret_client.list_properties_of_secrets() + + # Extract and return the list of secret names + secret_names = [secret.name for secret in secrets] + return secret_names + + except AzureError as az_error: + print(f"AzureError occurred: {az_error.message}") + return None + except Exception as e: + print(f"An error occurred: {e}") + return None + + def azure_kv_osd_keys(self): + """ + List of OSD keys found in Azure Key Vault + """ + azure_kv_secrets = self.azure_kv_secrets() + deviceset = [pvc.name for pvc in get_deviceset_pvcs()] + + found_osd_keys = [ + kv_secret + for kv_secret in azure_kv_secrets + if [dev for dev in deviceset if dev in kv_secret] + ] + + logger.info(f"OSD Keys on Azure KV: {found_osd_keys}") + + return found_osd_keys + + def verify_osd_keys_present_on_azure_kv(self): + """ + Verify if all OSD keys are present in Azure Key Vault + """ + + osd_keys = self.azure_kv_osd_keys() + deviceset = [pvc.name for pvc in get_deviceset_pvcs()] + + if len(osd_keys) != len(deviceset): + logger.info("Not all OSD keys present in the Azure KV") + return False + + logger.info("All OSD keys are present in the Azure KV ") + return True + + +kms_map = {"vault": Vault, "hpcs": HPCS, "kmip": KMIP, "azure-kv": AzureKV} def update_csi_kms_vault_connection_details(update_config): @@ -1845,7 +2108,7 @@ def get_kms_deployment(): try: return kms_map[provider]() except KeyError: - raise KMSNotSupported("Not a supported KMS deployment") + raise KMSNotSupported(f"Not a supported KMS deployment , provider: {provider}") def is_kms_enabled(dont_raise=False): diff --git a/setup.py b/setup.py index 2de197bac96..97dd2c9bc94 100644 --- a/setup.py +++ b/setup.py @@ -97,6 +97,7 @@ "googleapis-common-protos==1.59.0", "urllib3==1.26.18", "psycopg2-binary==2.9.9", + "azure-keyvault-secrets==4.8.0", ], entry_points={ "console_scripts": [