Added Azure KMS support in cluster deployment (#9731)

Signed-off-by: Parag Kamble <[email protected]>

conf/deployment/azure/ipi_1az_rhcos_3m_3w_encryption_azure_kv.yaml

@@ -0,0 +1,21 @@
+---
+DEPLOYMENT:
+  openshift_install_timeout: 4800
+  allow_lower_instance_requirements: false
+  kms_deployment: true
+ENV_DATA:
+  platform: 'azure'
+  deployment_type: 'ipi'
+  region: 'eastus'
+  azure_base_domain_resource_group_name: 'odfqe'
+  worker_availability_zones:
+    - '1'
+  master_availability_zones:
+    - '1'
+  worker_replicas: 3
+  master_replicas: 3
+  master_instance_type: 'Standard_D8s_v3'
+  worker_instance_type: 'Standard_D16s_v3'
+  encryption_at_rest: true
+  sc_encryption: true
+  KMS_PROVIDER: azure-kv

ocs_ci/ocs/constants.py

@@ -975,6 +975,20 @@
 KMIP_OCS_KMS_SECRET = os.path.join(KMIP_KMS_TEMPLATES, "thales-kmip-ocs-secret.yaml")
 KMIP_CSI_KMS_SECRET = os.path.join(TEMPLATE_CSI_RBD_DIR, "thales-kmip-csi-secret.yaml")
 
+# Azure KV KMS yamls
+AZURE_KV_PROVIDER_NAME = "azure-kv"
+AZURE_KV_CSI_CONNECTION_DETAILS = "csi-kms-connection-details"
+AZURE_KV_CONNECTION_DETAILS_RESOURCE = "ocs-kms-connection-details"
+AZURE_KV_TEMPLATES = os.path.join(TEMPLATE_OPENSHIFT_INFRA_DIR, "azurekv")
+AZURE_OCS_KMS_CONNECTION_DETAILS = os.path.join(
+    AZURE_KV_TEMPLATES, "ocs-kms-connection-details.yaml"
+)
+AZURE_CSI_KMS_CONNECTION_DETAILS = os.path.join(
+    TEMPLATE_CSI_RBD_DIR, "csi-kms-connection-details-azurekv.yaml"
+)
+AZURE_CLIENT_SECRETS = os.path.join(AZURE_KV_TEMPLATES, "azure-client-secrets.yaml")
+
+
 # Multicluster related yamls
 ODF_MULTICLUSTER_ORCHESTRATOR = os.path.join(
     TEMPLATE_MULTICLUSTER_DIR, "odf_multicluster_orchestrator.yaml"

ocs_ci/templates/CSI/rbd/csi-kms-connection-details-azurekv.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+data:
+kind: ConfigMap
+metadata:
+  name: csi-kms-connection-details
+  namespace: openshift-storage

ocs_ci/templates/openshift-infra/azurekv/azure-client-secrets.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: 
+  namespace: 
+data:
+  CLIENT_CERT: 
+type: Opaque

ocs_ci/templates/openshift-infra/azurekv/ocs-kms-connection-details.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ocs-kms-connection-details
+  namespace: openshift-storage
+data:
+  AZURE_CERT_SECRET_NAME: 
+  AZURE_CLIENT_ID: 
+  AZURE_TENANT_ID: 
+  AZURE_VAULT_URL: 
+  KMS_PROVIDER: 
+  KMS_SERVICE_NAME: 

ocs_ci/utility/kms.py

@@ -48,6 +48,11 @@
     encode,
     prepare_bin_dir,
 )
+from fauxfactory import gen_alphanumeric
+from azure.identity import CertificateCredential
+from azure.keyvault.secrets import SecretClient
+from azure.core.exceptions import AzureError
+from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs
 
 
 logger = logging.getLogger(__name__)
@@ -1797,7 +1802,265 @@ def cleanup(self):
             logger.info("Keys deleted from CipherTrust Manager")
 
 
-kms_map = {"vault": Vault, "hpcs": HPCS, "kmip": KMIP}
+class AzureKV(KMS):
+    """
+    Represents an Azure Key Vault implementation of KMS.
+    """
+
+    def __init__(self, namespace=config.ENV_DATA["cluster_namespace"]):
+        super().__init__(constants.AZURE_KV_PROVIDER_NAME)
+        self.namespace = namespace
+        self.kms_provider = constants.AZURE_KV_PROVIDER_NAME
+        self.azure_kms_connection_name = (
+            f"azure-kv-conn-{gen_alphanumeric(length=5).lower()}"
+        )
+        azure_auth = config.AUTH.get("azure_auth")
+        self.azure_kv_name = azure_auth.get("AZURE_KV_NAME")
+        self.azure_kv_certificate = azure_auth.get("AZURE_CERTIFICATE")
+        self.vault_url = azure_auth.get("AZURE_KV_URL")
+        self.vault_client_id = azure_auth.get("AZURE_KV_CLIENT_ID")
+        self.vault_tenant_id = azure_auth.get("AZURE_KV_TENANT_ID")
+        self.vault_cert_path = self._azure_kv_cert_path()
+
+        self.conn_data = {
+            "KMS_PROVIDER": self.kms_provider,
+            "KMS_SERVICE_NAME": self.azure_kms_connection_name,
+            "AZURE_CLIENT_ID": self.vault_client_id,
+            "AZURE_VAULT_URL": self.vault_url,
+            "AZURE_TENANT_ID": self.vault_tenant_id,
+        }
+
+    def deploy(self):
+        """
+        This Function will create the Azure KV connection details in the ConfigMap.
+        """
+        if not config.ENV_DATA.get("platform") == "azure":
+            raise VaultDeploymentError(
+                "Azure_KV deployment only supports on Azure platform."
+            )
+
+        self.create_azure_kv_csi_kms_connection_details()
+        if config.ENV_DATA.get("encryption_at_rest"):
+            self.create_azure_kv_ocs_csi_kms_connection_details()
+
+    def post_deploy_verification(self):
+        """
+        Post Deploy Verification For Azure Key Vault.
+        """
+        if config.ENV_DATA.get("encryption_at_rest"):
+            if not self.verify_osd_keys_present_on_azure_kv():
+                raise ValueError("OSD keys Not present on Azure Key Vault.")
+            logger.info("OSD Keys Are present on Azure Key Vault.")
+
+    def is_azure_kv_connection_exists(self):
+        """
+        Checks if the Azure KV connection exists in the ConfigMap
+        """
+
+        csi_kms_configmap = ocp.OCP(
+            kind=constants.CONFIGMAP,
+            resource_name=constants.VAULT_KMS_CSI_CONNECTION_DETAILS,
+            namespace=self.namespace,
+        )
+
+        if not csi_kms_configmap.is_exist():
+            raise ValueError(
+                f"ConfigMap {csi_kms_configmap.resource_name} Not found in the namespace {self.namespace}"
+            )
+
+        if self.azure_kms_connection_name not in csi_kms_configmap.data["data"]:
+            raise ValueError(
+                f"Azure Key vault connection {self.azure_kms_connection_name} not exists."
+            )
+
+    def create_azure_kv_secrets(self, prefix="azure-ocs-"):
+        """
+        Creates Azure KV secrets.
+        """
+        secret_name = gen_alphanumeric(length=18, start=prefix).lower()
+        client_secret = templating.load_yaml(constants.AZURE_CLIENT_SECRETS)
+
+        client_secret["metadata"]["name"] = secret_name
+        client_secret["metadata"]["namespace"] = self.namespace
+        client_secret["data"]["CLIENT_CERT"] = base64.b64encode(
+            self.azure_kv_certificate.encode()
+        ).decode()
+        logger.info(f"Creating a Azure Secret : {secret_name}")
+        self.create_resource(client_secret, prefix=prefix)
+        return secret_name
+
+    def create_azure_kv_csi_kms_connection_details(self):
+        """
+        Create Azure specific csi-kms-connection-details
+        configmap resource
+        """
+
+        # Check is already configmap exists
+        csi_kms_configmap = ocp.OCP(
+            kind=constants.CONFIGMAP,
+            resource_name=constants.AZURE_KV_CSI_CONNECTION_DETAILS,
+            namespace=self.namespace,
+        )
+
+        # Create a Connection data.
+        azure_conn = self.conn_data
+        azure_conn["AZURE_CERT_SECRET_NAME"] = self.create_azure_kv_secrets(
+            prefix="azure-csi-"
+        )
+
+        if not csi_kms_configmap.is_exist():
+            logger.info(
+                f"Creating Configmap {constants.AZURE_KV_CSI_CONNECTION_DETAILS}"
+            )
+
+            csi_kms_conn_details = templating.load_yaml(
+                constants.AZURE_CSI_KMS_CONNECTION_DETAILS
+            )
+
+            # Updating Templet data.
+            csi_kms_conn_details["data"] = {
+                self.azure_kms_connection_name: json.dumps(azure_conn)
+            }
+
+            csi_kms_conn_details["metadata"]["namespace"] = self.namespace
+            self.create_resource(csi_kms_conn_details, prefix="csiazureconn")
+        else:
+            # Append the connection details to existing ConfigMap.
+            logger.info(
+                f"Adding Azure connection to existing ConfigMap {constants.AZURE_KV_CSI_CONNECTION_DETAILS}"
+            )
+            param = (
+                f'[{{"op": "add", "path": "/data/{self.azure_kms_connection_name}", '
+                f'"value": "{json.dumps(azure_conn)}"}}]'
+            )
+            csi_kms_configmap.patch(params=param, format_type="merge")
+
+        # verifying ConfigMap is created or not.
+        self.is_azure_kv_connection_exists()
+
+    def create_azure_kv_ocs_csi_kms_connection_details(self):
+        """
+        Creates Azure KV OCS CSI KMS connection details ConfigMap.
+        """
+
+        # Creating ConfigMap for OCS CSI KMS connection details.
+        azure_data = self.conn_data
+        azure_data["AZURE_CERT_SECRET_NAME"] = self.create_azure_kv_secrets(
+            prefix="azure-ocs-"
+        )
+
+        # loading ConfigMap template
+        ocs_kms_conn_details = templating.load_yaml(
+            constants.AZURE_OCS_KMS_CONNECTION_DETAILS
+        )
+        ocs_kms_conn_details["metadata"]["namespace"] = self.namespace
+        ocs_kms_conn_details["data"] = azure_data
+
+        # creating ConfigMap Rsource
+        logger.info(
+            f"creating ConfigMap resource for {constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE}"
+        )
+        self.create_resource(ocs_kms_conn_details, prefix="ocsazureconn")
+
+        # Verify ConfigMap is created or not.
+        ocs_kms_configmap = ocp.OCP(
+            kind=constants.CONFIGMAP,
+            resource_name=constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE,
+            namespace=self.namespace,
+        )
+
+        if not ocs_kms_configmap.is_exist():
+            raise ValueError(
+                f"ConfigMap Resource {constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE}"
+                f" is not created in namespace {self.namespace}"
+            )
+
+        logger.info(
+            f"Successfully Created configmap {constants.AZURE_KV_CONNECTION_DETAILS_RESOURCE} "
+            f"in {self.namespace} namespace"
+        )
+
+    def _azure_kv_cert_path(self):
+        """
+        Create a temporary certificate file and write the Azure Key Vault certificate to it.
+        """
+        try:
+            temp_dir = tempfile.mkdtemp()
+            cert_file = os.path.join(temp_dir, "certificate.pem")
+
+            with open(cert_file, "w") as fd:
+                fd.write(self.azure_kv_certificate)
+
+            return cert_file
+        except Exception as ex:
+            raise ValueError(f"Error Creating Azure certificate file : {ex}")
+
+    def azure_kv_secrets(self):
+        """
+        List the secrets in the Azure Key Vault.
+        """
+        try:
+            # Create a CertificateCredential using the certificate
+            credential = CertificateCredential(
+                vault_url=self.vault_url,
+                tenant_id=self.vault_tenant_id,
+                client_id=self.vault_client_id,
+                certificate_path=self.vault_cert_path,
+            )
+
+            # Create a SecretClient using the certificate for authentication
+            secret_client = SecretClient(
+                vault_url=self.vault_url, credential=credential
+            )
+
+            # Get the list of secrets
+            secrets = secret_client.list_properties_of_secrets()
+
+            # Extract and return the list of secret names
+            secret_names = [secret.name for secret in secrets]
+            return secret_names
+
+        except AzureError as az_error:
+            print(f"AzureError occurred: {az_error.message}")
+            return None
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+    def azure_kv_osd_keys(self):
+        """
+        List of OSD keys found in Azure Key Vault
+        """
+        azure_kv_secrets = self.azure_kv_secrets()
+        deviceset = [pvc.name for pvc in get_deviceset_pvcs()]
+
+        found_osd_keys = [
+            kv_secret
+            for kv_secret in azure_kv_secrets
+            if [dev for dev in deviceset if dev in kv_secret]
+        ]
+
+        logger.info(f"OSD Keys on Azure KV: {found_osd_keys}")
+
+        return found_osd_keys
+
+    def verify_osd_keys_present_on_azure_kv(self):
+        """
+        Verify if all OSD keys are present in Azure Key Vault
+        """
+
+        osd_keys = self.azure_kv_osd_keys()
+        deviceset = [pvc.name for pvc in get_deviceset_pvcs()]
+
+        if len(osd_keys) != len(deviceset):
+            logger.info("Not all OSD keys present in the Azure KV")
+            return False
+
+        logger.info("All OSD keys are present in the Azure KV ")
+        return True
+
+
+kms_map = {"vault": Vault, "hpcs": HPCS, "kmip": KMIP, "azure-kv": AzureKV}
 
 
 def update_csi_kms_vault_connection_details(update_config):
@@ -1845,7 +2108,7 @@ def get_kms_deployment():
     try:
         return kms_map[provider]()
     except KeyError:
-        raise KMSNotSupported("Not a supported KMS deployment")
+        raise KMSNotSupported(f"Not a supported KMS deployment , provider: {provider}")
 
 
 def is_kms_enabled(dont_raise=False):

setup.py

@@ -97,6 +97,7 @@
         "googleapis-common-protos==1.59.0",
         "urllib3==1.26.18",
         "psycopg2-binary==2.9.9",
+        "azure-keyvault-secrets==4.8.0",
     ],
     entry_points={
         "console_scripts": [