diff --git a/roles/sap_hypervisor_node_preconfigure/handlers/main.yml b/roles/sap_hypervisor_node_preconfigure/handlers/main.yml index f920c7196..5150d6bdd 100644 --- a/roles/sap_hypervisor_node_preconfigure/handlers/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/handlers/main.yml @@ -1,4 +1,4 @@ --- -- name: SAP certified hypervisor node preconfigure - Include Handler Tasks for {{ sap_hypervisor_node_platform }} +- name: hypervisor node preconfigure - Include Handler Tasks for {{ sap_hypervisor_node_platform }} ansible.builtin.include_tasks: "{{ role_path }}/handlers/platform/{{ sap_hypervisor_node_platform }}/main.yml" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 index 183bfb353..32064a8d7 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 @@ -2,9 +2,8 @@ apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: - kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} - machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} - name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} + machineconfiguration.openshift.io/role: worker + name: 99-kargs-worker spec: config: ignition: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 82fec7ac3..068bbf1d8 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -7,12 +7,6 @@ index_var: __sap_hypervisor_node_preconfigure_register_worker_network_nr when: __sap_hypervisor_node_preconfigure_register_worker.networks is defined -- name: "Create MCP for {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - kubernetes.core.k8s: - template: - path: "mcp.yml.j2" - state: present - - name: Pause so cluster can process config ansible.builtin.pause: minutes: 1 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml deleted file mode 100644 index 6dd050ea9..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: "Download rhel 8.6 image" - kubernetes.core.k8s: - state: present - definition: - apiVersion: cdi.kubevirt.io/v1beta1 - kind: DataVolume - metadata: - namespace: openshift-virtualization-os-images - name: rhel-86 - annotations: - cdi.kubevirt.io/storage.bind.immediate.requested: 'true' - spec: - source: - registry: - url: 'docker://registry.redhat.io/rhel8/rhel-guest-image:8.6.0' - pullMethod: node - storage: - resources: - requests: - storage: 10Gi diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml deleted file mode 100644 index 25e88c1b2..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -- name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ - feature.node.kubernetes.io/cpu-feature-invtsc=true --overwrite=true" - register: __sap_hypervisor_node_preconfigure_register_label_node - changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 - -- name: Enable CPU Manager by patching MCP of "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - kubernetes.core.k8s: - state: patched - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfigPool - metadata: - name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - labels: - custom-kubelet: "cpumanager-enabled" - -- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined - kubernetes.core.k8s: - state: present - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: KubeletConfig - metadata: - name: "cpumanager-enabled" - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - spec: - machineConfigPoolSelector: - matchLabels: - custom-kubelet: "cpumanager-enabled" - kubeletConfig: - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s - reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" - -- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined - kubernetes.core.k8s: - state: present - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: KubeletConfig - metadata: - name: "cpumanager-enabled" - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - spec: - machineConfigPoolSelector: - matchLabels: - custom-kubelet: "cpumanager-enabled" - kubeletConfig: - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s - -- name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" - register: __sap_hypervisor_node_preconfigure_register_label_node - changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index fc1c7ab7a..e846f712d 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -29,6 +29,37 @@ fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster success_msg: Configured nodes found +# Determine available memory on first worker node. +# This amount will be used for all nodes, so make sure all have an identical amount. +- name: Get worker name + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_name: + "{{ __sap_hypervisor_node_preconfigure_register_nodes[0]['metadata']['labels']['kubernetes.io/hostname'] }}" + +- name: Get memory of worker node + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_memory_gb: + "{{ (__sap_hypervisor_node_preconfigure_register_nodes[0]['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" + +- name: Check if host has minimal amount of memory (96Gb) + ansible.builtin.assert: + that: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 96 + fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" + +# calculate memory to be allocated as hugepages +# if system < 512GB memory use 32GB as upper boundary, 64GB otherwise as upper boundary +- name: Calculate amount of hugepages to reserve (host memory < 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int < 512 + +- name: Calculate amount of hugepages to reserve (host memory >= 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 + # - meta: end_play - name: Include prepare @@ -66,3 +97,4 @@ - name: Include local storage creation (HPP) ansible.builtin.include_tasks: install-hpp.yml when: sap_hypervisor_node_preconfigure_install_hpp + diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 deleted file mode 100644 index 1a39d0a06..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfigPool -metadata: - labels: - machineconfiguration.openshift.io/mco-built-in: "" - name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} -spec: - configuration: - source: - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 00-worker - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 01-worker-container-runtime - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 01-worker-kubelet - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-crio-capabilities - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-crio-seccomp-use-default - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-kubelet - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-registries - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-ssh - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} - machineConfigSelector: - matchExpressions: - - key: machineconfiguration.openshift.io/role - operator: In - values: - - worker - - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} - nodeSelector: - matchLabels: - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - paused: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml index 2dc78034f..e144f1a9b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml @@ -1,11 +1,4 @@ --- -- name: Include configure kargs - ansible.builtin.include_tasks: configure-kargs-per-node.yml - with_items: "{{ __sap_hypervisor_node_preconfigure_register_nodes }}" - loop_control: - loop_var: __sap_hypervisor_node_preconfigure_register_worker - index_var: __sap_hypervisor_node_preconfigure_register_worker_nr - - name: Include configure worker ansible.builtin.include_tasks: configure-worker-node.yml with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" @@ -24,8 +17,7 @@ labels: custom-kubelet: cpumanager-enabled - -- name: Create kubletconfig for cpumanager worker +- name: Create kubletconfig for cpumanager worker with CPUs reserved for kubernetes when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined kubernetes.core.k8s: state: absent @@ -44,7 +36,7 @@ reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" - name: Create kubletconfig for cpumanager worker - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined kubernetes.core.k8s: state: present definition: @@ -52,6 +44,7 @@ kind: KubeletConfig metadata: name: cpumanager-enabled + machineconfiguration.openshift.io/role: worker spec: machineConfigPoolSelector: matchLabels: @@ -59,22 +52,15 @@ kubeletConfig: cpuManagerPolicy: static cpuManagerReconcilePeriod: 5s - reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" -- name: Create kubletconfig for cpumanager worker - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined +- name: Personalize template + ansible.builtin.template: + src: 99-kargs-worker.yml.j2 + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" + mode: "0644" + +- name: Enable hugepages kubernetes.core.k8s: state: present - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: KubeletConfig - metadata: - name: cpumanager-enabled - machineconfiguration.openshift.io/role: worker - spec: - machineConfigPoolSelector: - matchLabels: - custom-kubelet: cpumanager-enabled - kubeletConfig: - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" +