Add certificate support (#5867)

Co-authored-by: Wolfgang Kulhanek <[email protected]>

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/defaults/main.yml

@@ -42,6 +42,13 @@ ocp4_workload_deploy_hosted_cluster_user_base: user
 ocp4_workload_deploy_hosted_cluster_user_password: ""
 ocp4_workload_deploy_hosted_cluster_user_password_length: 16
 
+# Cert manager variables - from a secret
+ocp4_workload_deploy_hosted_cluster_certmanager_aws_access_key_id: ""
+ocp4_workload_deploy_hosted_cluster_certmanager_aws_secret_access_key: ""
+ocp4_workload_deploy_hosted_cluster_certmanager_aws_hostedzone_id: ""
+ocp4_workload_deploy_hosted_cluster_certmanager_aws_region: >-
+  {{ ocp4_workload_deploy_hosted_cluster_aws_region }}
+
 # Internal variables, do not set:
 _ocp4_workload_deploy_hosted_cluster_admin_password: ""
 _ocp4_workload_deploy_hosted_cluster_admin_password_hash: ""

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/tasks/remove_workload.yml

@@ -2,6 +2,30 @@
 # Implement your workload removal tasks here
 # ------------------------------------------
 
+- name: Delete cert-manager policy
+  kubernetes.core.k8s:
+    state: absent
+    api_version: policy.open-cluster-management.io/v1
+    kind: Policy
+    name: cert-{{ guid }}
+    namespace: rhdp-policies
+
+- name: Delete cert-manager placement binding
+  kubernetes.core.k8s:
+    state: absent
+    api_version: policy.open-cluster-management.io/v1
+    kind: PlacementBinding
+    name: cert-{{ guid }}
+    namespace: rhdp-policies
+
+- name: Delete cert-manager placement rule
+  kubernetes.core.k8s:
+    state: absent
+    api_version: apps.open-cluster-management.io/v1
+    kind: PlacementRule
+    name: cert-{{ guid }}
+    namespace: rhdp-policies
+
 - name: Delete klusterlet addon config
   kubernetes.core.k8s:
     state: absent
@@ -28,12 +52,16 @@
   register: r_hypershift_destroy
   ignore_errors: true
 
-- name: Delete htpasswd-{{ guid }} secret
+- name: Delete hosted cluster secrets
   kubernetes.core.k8s:
     state: absent
     api_version: v1
     kind: secret
-    name: "htpasswd-{{ guid }}"
+    name: "{{ item }}"
+    namespace: local-cluster
+  loop:
+  - "htpasswd-{{ guid }}"
+  - "oauth-{{ guid }}"
 
 # Leave this as the last task in the playbook.
 # --------------------------------------------

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/tasks/setup_authentication.yml

@@ -0,0 +1,80 @@
+---
+- name: Generate admin user password
+  when: ocp4_workload_deploy_hosted_cluster_admin_password | default('') | length == 0
+  ansible.builtin.set_fact:
+    _ocp4_workload_deploy_hosted_cluster_admin_password: >-
+      {{ lookup('password', '/dev/null chars=ascii_letters,digits '
+          ~ 'length=' ~ ocp4_workload_deploy_hosted_cluster_admin_password_length
+      ) }}
+
+- name: Use provided admin password
+  when: ocp4_workload_deploy_hosted_cluster_admin_password | default('') | length > 0
+  ansible.builtin.set_fact:
+    _ocp4_workload_deploy_hosted_cluster_admin_password: "{{ ocp4_workload_deploy_hosted_cluster_admin_password }}"
+
+- name: Generate htpasswd hash for admin user
+  ansible.builtin.shell: >-
+    htpasswd -nb "admin" "{{ _ocp4_workload_deploy_hosted_cluster_admin_password }}" | cut -d: -f2
+  register: r_htpasswd_admin_line
+
+- name: Set htpasswd admin password hash
+  when: r_htpasswd_admin_line is succeeded
+  ansible.builtin.set_fact:
+    _ocp4_workload_deploy_hosted_cluster_admin_password_hash: "{{ r_htpasswd_admin_line.stdout }}"
+
+- name: Generate user passwords
+  when: ocp4_workload_deploy_hosted_cluster_user_password | default('') | length == 0
+  ansible.builtin.set_fact:
+    _ocp4_workload_deploy_hosted_cluster_user_password: >-
+      {{ lookup('password', '/dev/null chars=ascii_letters,digits '
+          ~ 'length=' ~ ocp4_workload_deploy_hosted_cluster_user_password_length
+      ) }}
+
+- name: Use provided user passwords
+  when: ocp4_workload_deploy_hosted_cluster_user_password | default('') | length > 0
+  ansible.builtin.set_fact:
+    _ocp4_workload_deploy_hosted_cluster_user_password: "{{ ocp4_workload_deploy_hosted_cluster_user_password }}"
+
+- name: Generate htpasswd hash for user passwords
+  ansible.builtin.shell: >-
+    htpasswd -nb "userN" "{{ _ocp4_workload_deploy_hosted_cluster_user_password }}" | cut -d: -f2
+  register: r_htpasswd_user_line
+
+- name: Set htpasswd user password hash
+  when: r_htpasswd_user_line is succeeded
+  ansible.builtin.set_fact:
+    _ocp4_workload_deploy_hosted_cluster_user_password_hash: "{{ r_htpasswd_user_line.stdout }}"
+
+- name: Generate htpasswd file
+  ansible.builtin.template:
+    src: htpasswd.j2
+    dest: /home/opentlc-mgr/users.htpasswd
+    owner: opentlc-mgr
+    mode: 0664
+
+- name: Read contents of htpasswd file
+  ansible.builtin.slurp:
+    src: /home/opentlc-mgr/users.htpasswd
+  register: r_htpasswd_file
+
+- name: Remove generated htpasswd file
+  ansible.builtin.file:
+    path: /home/opentlc-mgr/users.htpasswd
+    state: absent
+
+- name: Ensure secret htpasswd-{{ guid }} is absent
+  kubernetes.core.k8s:
+    state: absent
+    api_version: v1
+    kind: Secret
+    name: "htpasswd-{{ guid }}"
+    namespace: local-cluster
+  register: r_htpasswd_secret_absent
+  retries: 5
+  delay: 10
+  until: r_htpasswd_secret_absent is success
+
+- name: Create secret htpasswd-{{ guid }}
+  kubernetes.core.k8s:
+    state: present
+    definition: "{{ lookup('template', 'secret-htpasswd.yaml.j2' ) | from_yaml }}"

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/tasks/setup_oauth_certificate.yml

@@ -0,0 +1,32 @@
+---
+- name: Get the IngressController definition from hub cluster
+  kubernetes.core.k8s_info:
+    api_version: operator.openshift.io/v1
+    kind: IngressController
+    name: default
+    namespace: openshift-ingress-operator
+  register: r_ingresscontroller
+
+- name: Determine if the hub cluster has certificates installed
+  when:
+  - r_ingresscontroller.resources | length == 1
+  - r_ingresscontroller.resources[0].spec.defaultCertificate is defined
+  - r_ingresscontroller.resources[0].spec.defaultCertificate.name is defined
+  block:
+  - name: Retrieve current ingress controller certificate
+    kubernetes.core.k8s_info:
+      api_version: v1
+      kind: Secret
+      name: "{{ r_ingresscontroller.resources[0].spec.defaultCertificate.name }}"
+      namespace: openshift-ingress
+    register: r_ingresscontroller_cert
+
+  - name: Create secret oauth-{{ guid }}
+    when:
+    - r_ingresscontroller_cert.resources | length == 1
+    - r_ingresscontroller_cert.resources[0].data is defined
+    - r_ingresscontroller_cert.resources[0].data['tls.key'] is defined
+    - r_ingresscontroller_cert.resources[0].data['tls.crt'] is defined
+    kubernetes.core.k8s:
+      state: present
+      definition: "{{ lookup('template', 'secret-oauth.yaml.j2' ) | from_yaml }}"

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/tasks/workload.yml

@@ -3,85 +3,11 @@
   ansible.builtin.debug:
     msg: "Setting up workload for user ocp_username = {{ ocp_username }}"
 
-- name: Generate admin user password
-  when: ocp4_workload_deploy_hosted_cluster_admin_password | default('') | length == 0
-  ansible.builtin.set_fact:
-    _ocp4_workload_deploy_hosted_cluster_admin_password: >-
-      {{ lookup('password', '/dev/null chars=ascii_letters,digits '
-          ~ 'length=' ~ ocp4_workload_deploy_hosted_cluster_admin_password_length
-      ) }}
-
-- name: Use provided admin password
-  when: ocp4_workload_deploy_hosted_cluster_admin_password | default('') | length > 0
-  ansible.builtin.set_fact:
-    _ocp4_workload_deploy_hosted_cluster_admin_password: "{{ ocp4_workload_deploy_hosted_cluster_admin_password }}"
-
-- name: Generate htpasswd hash for admin user
-  ansible.builtin.shell: >-
-    htpasswd -nb "admin" "{{ _ocp4_workload_deploy_hosted_cluster_admin_password }}" | cut -d: -f2
-  register: r_htpasswd_admin_line
-
-- name: Set htpasswd admin password hash
-  when: r_htpasswd_admin_line is succeeded
-  ansible.builtin.set_fact:
-    _ocp4_workload_deploy_hosted_cluster_admin_password_hash: "{{ r_htpasswd_admin_line.stdout }}"
+- name: Setup authentication
+  include_tasks: setup_authentication.yml
 
-- name: Generate user passwords
-  when: ocp4_workload_deploy_hosted_cluster_user_password | default('') | length == 0
-  ansible.builtin.set_fact:
-    _ocp4_workload_deploy_hosted_cluster_user_password: >-
-      {{ lookup('password', '/dev/null chars=ascii_letters,digits '
-          ~ 'length=' ~ ocp4_workload_deploy_hosted_cluster_user_password_length
-      ) }}
-
-- name: Use provided user passwords
-  when: ocp4_workload_deploy_hosted_cluster_user_password | default('') | length > 0
-  ansible.builtin.set_fact:
-    _ocp4_workload_deploy_hosted_cluster_user_password: "{{ ocp4_workload_deploy_hosted_cluster_user_password }}"
-
-- name: Generate htpasswd hash for user passwords
-  ansible.builtin.shell: >-
-    htpasswd -nb "userN" "{{ _ocp4_workload_deploy_hosted_cluster_user_password }}" | cut -d: -f2
-  register: r_htpasswd_user_line
-
-- name: Set htpasswd user password hash
-  when: r_htpasswd_user_line is succeeded
-  ansible.builtin.set_fact:
-    _ocp4_workload_deploy_hosted_cluster_user_password_hash: "{{ r_htpasswd_user_line.stdout }}"
-
-- name: Generate htpasswd file
-  ansible.builtin.template:
-    src: htpasswd.j2
-    dest: /home/opentlc-mgr/users.htpasswd
-    owner: opentlc-mgr
-    mode: 0664
-
-- name: Read contents of htpasswd file
-  ansible.builtin.slurp:
-    src: /home/opentlc-mgr/users.htpasswd
-  register: r_htpasswd_file
-
-- name: Remove generated htpasswd file
-  ansible.builtin.file:
-    path: /home/opentlc-mgr/users.htpasswd
-    state: absent
-
-- name: Ensure secret htpasswd-{{ guid }} is absent
-  kubernetes.core.k8s:
-    state: absent
-    api_version: v1
-    kind: Secret
-    name: "htpasswd-{{ guid }}"
-    namespace: local-cluster
-  register: r_htpasswd_secret_absent
-  retries: 5
-  delay: 10
-  until: r_htpasswd_secret_absent is success
-
-- name: Create secret htpasswd-{{ guid }}
-  kubernetes.core.k8s:
-    state: present
-    definition: "{{ lookup('template',  'secret-htpasswd.yaml.j2' ) | from_yaml }}"
+- name: Setup oauth certificate
+  include_tasks: setup_oauth_certificate.yml
 
 - name: Run hypershift CLI to deploy hosted cluster
   ansible.builtin.command: >-
@@ -104,7 +30,7 @@
       --namespace local-cluster
   register: r_hypershift_create_cluster
 
-- name: Patch hosted cluster to add authentication
+- name: Patch hosted cluster to add authentication and oauth certs
   kubernetes.core.k8s:
     state: patched
     definition: "{{ lookup('template', 'hosted-cluster-auth.yaml.j2' ) | from_yaml }}"
@@ -124,6 +50,15 @@
     _ocp4_workload_deploy_hosted_cluster_console_url: >-
       https://console-openshift-console.apps.{{ ocp4_workload_deploy_hosted_cluster_name }}.{{ ocp4_workload_deploy_hosted_cluster_base_domain }}
 
+- name: Create cert-manager policy
+  kubernetes.core.k8s:
+    state: present
+    definition: "{{ lookup('template', item ) | from_yaml }}"
+  loop:
+  - certmanager/policy.yaml.j2
+  - certmanager/placementrule.yaml.j2
+  - certmanager/placementbinding.yaml.j2
+
 - name: Save common user information
   agnosticd_user_info:
     data:

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/templates/certmanager/placementbinding.yaml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: PlacementBinding
+metadata:
+  name: cert-{{ guid }}
+  namespace: rhdp-policies
+placementRef:
+  apiGroup: apps.open-cluster-management.io
+  kind: PlacementRule
+  name: cert-{{ guid }}
+subjects:
+- apiGroup: policy.open-cluster-management.io
+  kind: Policy
+  name: cert-{{ guid }}

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/templates/certmanager/placementrule.yaml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: apps.open-cluster-management.io/v1
+kind: PlacementRule
+metadata:
+  name: cert-{{ guid }}
+  namespace: rhdp-policies
+spec:
+  clusterSelector:
+    matchLabels:
+      name: {{ ocp4_workload_deploy_hosted_cluster_name }}
+      vendor: OpenShift
+{% if ocp4_workload_deploy_hosted_cluster_labels | default("") | length > 0 %}
+      {{ ocp4_workload_deploy_hosted_cluster_labels | to_nice_yaml | indent(6) }}
+{% endif %}

ansible/roles_ocp_workloads/ocp4_workload_deploy_hosted_cluster/templates/certmanager/policy.yaml.j2

@@ -0,0 +1,73 @@
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: Policy
+metadata:
+  name: cert-{{ guid }}
+  namespace: rhdp-policies
+spec:
+  remediationAction: enforce
+  disabled: false
+  policy-templates:
+  - objectDefinition:
+      apiVersion: policy.open-cluster-management.io/v1
+      kind: ConfigurationPolicy
+      metadata:
+        name: cert-{{ guid }}
+      spec:
+        remediationAction: enforce
+        severity: medium
+        object-templates:
+        - complianceType: musthave
+          objectDefinition:
+            apiVersion: cert-manager.io/v1
+            kind: ClusterIssuer
+            metadata:
+              name: letsencrypt-production-route53
+            spec:
+              acme:
+                email: [email protected]
+                server: https://acme-v02.api.letsencrypt.org/directory
+                privateKeySecretRef:
+                  name: cluster-issuer-le-production
+                solvers:
+                - selector:
+                    dnsZones:
+                    - {{ ocp4_workload_deploy_hosted_cluster_base_domain }}
+                  dns01:
+                    route53:
+                      region: {{ ocp4_workload_deploy_hosted_cluster_certmanager_aws_region }}
+                      hostedZoneID: {{ ocp4_workload_deploy_hosted_cluster_certmanager_aws_hostedzone_id }}
+                      accessKeyID: {{ ocp4_workload_deploy_hosted_cluster_certmanager_aws_access_key_id }}
+                      secretAccessKeySecretRef:
+                        name: aws-secret-access-key
+                        key: secret-access-key
+        - complianceType: musthave
+          objectDefinition:
+            apiVersion: cert-manager.io/v1
+            kind: Certificate
+            metadata:
+              name: ingress-cert
+              namespace: openshift-ingress
+            spec:
+              secretName: ingress-cert
+              duration: 2160h
+              renewBefore: 360h
+              usages:
+              - server auth
+              - client auth
+              dnsNames:
+              - "*.apps.{{ ocp4_workload_deploy_hosted_cluster_name }}.{{ ocp4_workload_deploy_hosted_cluster_base_domain }}"
+              issuerRef:
+                kind: ClusterIssuer
+                name: letsencrypt-production-route53
+                group: cert-manager.io
+        - complianceType: musthave
+          objectDefinition:
+            apiVersion: operator.openshift.io/v1
+            kind: IngressController
+            metadata:
+              name: default
+              namespace: openshift-ingress-operator
+            spec:
+              defaultCertificate:
+                name: ingress-cert