From 3d4ae5bedbc40fb8084745ccf895b7e27bfdcb86 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Wed, 21 Jun 2023 16:34:30 +0200 Subject: [PATCH] added redhat_ocp_virt platform to sap_hypervisor_node_preconfigure --- ...ypervisor-redhat_ocp_virt-preconfigure.yml | 11 + ...pervisor-node-preconfigure-rh_ocp_virt.yml | 89 ++++++++ .../README.md | 209 +++++++++++++++++- .../cnv-namespace-operator-subscription.yml | 33 +++ .../sriov-enabled-unsupported-nics.sh | 5 + .../sriov-namespace-operator-subscription.yml | 26 +++ .../platform/redhat_rhel_kvm/main.yml | 7 +- .../meta/main.yml | 9 +- .../redhat_ocp_virt/99-kargs-worker.yml.j2 | 18 ++ .../configure-kargs-per-node.yml | 34 +++ .../redhat_ocp_virt/configure-worker-node.yml | 24 ++ .../redhat_ocp_virt/create-sap-bridge.yml | 49 ++++ .../redhat_ocp_virt/download-rhel-images.yml | 21 ++ .../redhat_ocp_virt/enable-cpumanager.yml | 61 +++++ .../redhat_ocp_virt/install-cnv-operator.yml | 74 +++++++ .../install-nmstate-operator.yml | 89 ++++++++ .../install-sriov-operator.yml | 55 +++++ .../redhat_ocp_virt/install-trident.yml | 48 ++++ .../redhat_ocp_virt/install-virtctl.yml | 15 ++ .../tasks/platform/redhat_ocp_virt/kargs.yml | 11 + .../redhat_ocp_virt/label-worker-invtsc.yml | 11 + .../tasks/platform/redhat_ocp_virt/main.yml | 61 +++++ .../tasks/platform/redhat_ocp_virt/mcp.yml.j2 | 47 ++++ .../platform/redhat_ocp_virt/node-network.yml | 99 +++++++++ .../platform/redhat_ocp_virt/prepare.yml | 9 + .../redhat_ocp_virt/setup-worker-node.yml | 80 +++++++ .../sriov-enabled-unsupported-nics.sh | 5 + .../templates/99-kargs-worker.yml.j2 | 18 ++ .../templates/templates/mcp.yml.j2 | 47 ++++ .../redhat_ocp_virt/trident-backend.json.j2 | 18 ++ .../redhat_ocp_virt/tuned-virtual-host.yml | 21 ++ .../redhat_rhel_kvm/assert-configuration.yml | 46 ++-- .../redhat_rhel_kvm/assert-rhv-hooks.yml | 4 + .../assert-set-tuned-profile.yml | 1 + .../redhat_rhel_kvm/configuration.yml | 65 ++---- .../tasks/platform/redhat_rhel_kvm/main.yml | 12 +- .../platform/redhat_rhel_kvm/rhv-hooks.yml | 3 +- .../redhat_rhel_kvm/set-tuned-profile.yml | 4 + .../platform_defaults_redhat_ocp_virt.yml | 19 ++ 39 files changed, 1374 insertions(+), 84 deletions(-) create mode 100644 playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml create mode 100644 playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh create mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml diff --git a/playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml b/playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml new file mode 100644 index 000000000..285e60488 --- /dev/null +++ b/playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml @@ -0,0 +1,11 @@ +--- +- hosts: all + gather_facts: true + serial: 1 + vars: + sap_hypervisor_node_platform: redhat_ocp_virt + + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure diff --git a/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml new file mode 100644 index 000000000..a2747ac4f --- /dev/null +++ b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml @@ -0,0 +1,89 @@ +sap_hypervisor_node_preconfigure_cluster_config: + + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # detailed configuration for every worker that should be configured + # + workers: + - kubernetes_reserved_cpus: "0,1" # CPU cores reserved for + # kubernetes + + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index cc26e0917..628a6a86d 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -3,7 +3,191 @@ sap_hypervisor_node_preconfigure ======================= -This role will set and check the required settings and parameters for a hypervisor running VMs for SAP HANA. +This role will configure the following hypervisors in order to run SAP workloads: +* Redhat Openshift Virtualization (OCPV) +* Redhat Enterprise Virtualization (RHV) + +Platform: Redhat Openshift Virtualization +========================================= + +This role will configure a plain vanilla Openshift cluster so it can be used for SAP workloads. + +Requirements +------------ +A freshly installed Openshift cluster. +The worker nodes should have > 96GB of memory. +Storage is required, e.g. via NFS, Openshift Data Foundation or local storage. +This role can setup access to a Netapp Filer via Trident storage connector. +Point the `KUBECONFIG` environment variable to you `kubeconfig`. + + +Install the packages stated in `requirements.txt` on the host where the role runs. +The required packages are: +``` +httpd-tools +ansible-collection-kubernetes-core +``` + + +Make the role available in case you didn't install it already in an ansible roles directory, e.g. + +``` +mkdir -p ~/.ansible/roles/ +ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible/roles/ +``` + +Role Variables +-------------- +General variables are defined in sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +``` +# Install the trident NFS storage provider +sap_hypervisor_node_preconfigure_install_trident: False +# URL of the trident installer package to use +sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v23.01.0/trident-installer-23.01.0.tar.gz + +# should SRIOV be enabled for unsupported NICs +sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True + +# Amount of memory [GB] to be reserved for the hypervisor on hosts >= 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB +# Amount of memory [GB] to be reserved for the hypervisor on hosts < 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB + +# Should the check for the minimal amount of be ignored? Minimal amount is 96 GB +# If ignored, the amount of $hostmemory - $reserved is allocated with a lower bound of 0 in case $reserved > $hostmemory +sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False +``` + +The following variables are describing the nodes and networks to be used. It can make sense to have them in a seperate file, e.g. see `playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml` for an example. +``` +sap_hypervisor_node_preconfigure_cluster_config: + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # detailed configuration for every worker that should be configured + workers: + kubernetes_reserved_cpus: "0,1" # CPU cores reserved for + # kubernetes + + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov +``` + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +See `playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml` for this example: + +``` +--- +- hosts: all + gather_facts: true + serial: 1 + vars: + sap_hypervisor_node_platform: redhat_ocp_virt + + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure +``` + +Example Usage +------------- +Make sure to set the `KUBECONFIG` environment variable, e.g. +``` +export KUBECONFIG=~/.kubeconfig +``` +To invoke the example playbook with the example configuration using your localhost as ansible host use the following command line: +``` +ansible-playbook --connection=local -i localhost, playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml -e @s/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml +``` + + +Platform: RHEL KVM +=================== +set and check the required settings and parameters for a hypervisor running VMs for SAP HANA. Requirements ------------ @@ -48,18 +232,31 @@ Example Playbook Simple example that just sets the parameters. ``` +--- - hosts: all - roles: - - sap_hypervisor_node_preconfigure + gather_facts: true + serial: 1 + vars: + sap_hypervisor_node_platform: redhat_rhel_kvm + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure ``` Run in assert mode to verify that parameters have been set. ``` +--- - hosts: all - roles: - - sap_hypervisor_node_preconfigure + gather_facts: true + serial: 1 vars: - - sap_hypervisor_node_preconfigure_assert: yes + sap_hypervisor_node_platform: redhat_rhel_kvm + sap_hypervisor_node_preconfigure_assert: yes + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure ``` License ------- diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml new file mode 100644 index 000000000..2d8b3feab --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-cnv +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: kubevirt-hyperconverged-group + namespace: openshift-cnv +spec: + targetNamespaces: + - openshift-cnv +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: hco-operatorhub + namespace: openshift-cnv +spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: kubevirt-hyperconverged +# startingCSV: kubevirt-hyperconverged-operator.v4.10.0 +# channel: "stable" +--- +apiVersion: hco.kubevirt.io/v1beta1 +kind: HyperConverged +metadata: + name: kubevirt-hyperconverged + namespace: openshift-cnv +spec: diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh new file mode 100644 index 000000000..7732ba78e --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# in order to allow unsupported SRIOV nics such as Mellanox +oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' + + diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml new file mode 100644 index 000000000..9451b3401 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-sriov-network-operator +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: sriov-network-operators + namespace: openshift-sriov-network-operator +spec: + targetNamespaces: + - openshift-sriov-network-operator +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: sriov-network-operator-subscription + namespace: openshift-sriov-network-operator +spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: sriov-network-operator +# startingCSV: sriov-network-operator + channel: "stable" diff --git a/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml b/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml index 21c4a5b4e..911776897 100644 --- a/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml @@ -1,5 +1,4 @@ --- - - name: "Check if server is booted in BIOS or UEFI mode" ansible.builtin.stat: path: /sys/firmware/efi @@ -40,14 +39,14 @@ - name: "Set the grub.cfg location RHEL" ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_uefi_boot_dir: /boot/efi/EFI/redhat/grub.cfg - when: + when: - ansible_distribution == 'RedHat' - name: "Set the grub.cfg location SLES" ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_uefi_boot_dir: /boot/efi/EFI/BOOT/grub.cfg - when: - - ansible_distribution == 'SLES' or ansible_distribution == 'SLES_SAP' + when: + - ansible_distribution == 'SLES' or ansible_distribution == 'SLES_SAP' - name: "Run grub-mkconfig (UEFI mode)" ansible.builtin.command: "grub2-mkconfig -o {{ __sap_hypervisor_node_preconfigure_uefi_boot_dir }}" diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 0b4c2c801..47965cdc0 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -3,8 +3,15 @@ galaxy_info: namespace: community role_name: sap_hypervisor_node_preconfigure author: Nils Koenig - description: Provide the configuration of SAP-certified hypervisors + description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 min_ansible_version: 2.9 galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] + platforms: + - name: CentOS + versions: + 8 + - name: RHEL + versions: + 8 dependencies: [] diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 new file mode 100644 index 000000000..183bfb353 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 @@ -0,0 +1,18 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} +spec: + config: + ignition: + version: 3.2.0 + kernelArguments: + - intel_iommu=on + - iommu=pt + - default_hugepagesz=1GB + - hugepagesz=1GB + - hugepages={{ __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages }} + - tsx=on diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml new file mode 100644 index 000000000..488c3eaa9 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml @@ -0,0 +1,34 @@ +--- +- name: Get worker name + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_name: + "{{ __sap_hypervisor_node_preconfigure_register_worker['metadata']['labels']['kubernetes.io/hostname'] }}" + +- name: Get memory of worker node + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_memory_gb: + "{{ (__sap_hypervisor_node_preconfigure_register_worker['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" + +- name: Check if host has minimal amount of memory (96Gb) + ansible.builtin.assert: + that: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 96 + fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" + +# calculate memory to be allocated as hugepages +# if system < 512GB memory use 32GB as upper boundary, 64GB otherwise as upper boundary +- name: Calculate amount of hugepages to reserve (host memory < 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: + "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int < 512 + +- name: Calculate amount of hugepages to reserve (host memory >= 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: + "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 + +- name: "Include kargs for {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + ansible.builtin.include_tasks: kargs.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml new file mode 100644 index 000000000..82fec7ac3 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -0,0 +1,24 @@ +--- +- name: Include node network + ansible.builtin.include_tasks: node-network.yml + with_items: "{{ __sap_hypervisor_node_preconfigure_register_worker.networks }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker_network + index_var: __sap_hypervisor_node_preconfigure_register_worker_network_nr + when: __sap_hypervisor_node_preconfigure_register_worker.networks is defined + +- name: "Create MCP for {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + template: + path: "mcp.yml.j2" + state: present + +- name: Pause so cluster can process config + ansible.builtin.pause: + minutes: 1 + +# How to wait for node to be scheduleable? (NodeSchedulable) +- name: Wait for all k8s nodes to be ready + ansible.builtin.command: oc wait --for=condition=Ready nodes --all --timeout=3600s + register: __sap_hypervisor_node_preconfigure_register_nodes_ready + changed_when: __sap_hypervisor_node_preconfigure_register_nodes_ready.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml new file mode 100644 index 000000000..bbdbdfffd --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml @@ -0,0 +1,49 @@ +--- +- name: Create SAP bridge NodeNetworkConfigurationPolicy + kubernetes.core.k8s: + state: present + definition: + apiVersion: nmstate.io/v1 + kind: NodeNetworkConfigurationPolicy + metadata: + name: "sap-bridge-policy-{{ worker.name }}" + spec: + nodeSelector: + kubernetes.io/hostname: "{{ worker.name }}" + desiredState: + interfaces: + - name: sapbridge + description: "Linux bridge with {{ worker.sap_bridge_interface }} as physical port to access SAP network" + type: linux-bridge + state: up + ipv4: + enabled: false + bridge: + options: + stp: + enabled: false + port: + - name: "{{ worker.sap_bridge_interface }}" + + +- name: Create SAP bridge NetworkAttachmentDefinition + kubernetes.core.k8s: + state: present + definition: + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + kubernetes.io/hostname: "{{ worker.name }}" + machineconfiguration.openshift.io/role: "{{ worker.name }}" + namespace: "{{ vm_namespace }}" + name: sap-bridge-network-definition + annotations: + k8s.v1.cni.cncf.io/resourceName: bridge.network.kubevirt.io/sapbridge + spec: + config: '{ + "cniVersion": "0.3.1", + "name": "sap-bridge-network-definition", + "type": "cnv-bridge", + "bridge": "sapbridge", + "macspoofchk": true + }' diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml new file mode 100644 index 000000000..6dd050ea9 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml @@ -0,0 +1,21 @@ +--- +- name: "Download rhel 8.6 image" + kubernetes.core.k8s: + state: present + definition: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + namespace: openshift-virtualization-os-images + name: rhel-86 + annotations: + cdi.kubevirt.io/storage.bind.immediate.requested: 'true' + spec: + source: + registry: + url: 'docker://registry.redhat.io/rhel8/rhel-guest-image:8.6.0' + pullMethod: node + storage: + resources: + requests: + storage: 10Gi diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml new file mode 100644 index 000000000..25e88c1b2 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml @@ -0,0 +1,61 @@ +--- +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ + feature.node.kubernetes.io/cpu-feature-invtsc=true --overwrite=true" + register: __sap_hypervisor_node_preconfigure_register_label_node + changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 + +- name: Enable CPU Manager by patching MCP of "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + state: patched + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfigPool + metadata: + name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + labels: + custom-kubelet: "cpumanager-enabled" + +- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: "cpumanager-enabled" + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: "cpumanager-enabled" + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" + +- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: "cpumanager-enabled" + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: "cpumanager-enabled" + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" + register: __sap_hypervisor_node_preconfigure_register_label_node + changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml new file mode 100644 index 000000000..1a08c9306 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml @@ -0,0 +1,74 @@ +--- +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: openshift-cnv + +- name: Create CNV OperatorGroup kubevirt-hyperconverged-group + kubernetes.core.k8s: + state: present + + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: kubevirt-hyperconverged-group + namespace: openshift-cnv + spec: + targetNamespaces: + - openshift-cnv + +- name: Create CNV Subscription + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: hco-operatorhub + namespace: openshift-cnv + spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: kubevirt-hyperconverged + +- name: Wait + ansible.builtin.pause: + seconds: 60 + +- name: Get Install Plan Name + retries: 10 + delay: 10 + ansible.builtin.command: oc get subscriptions/hco-operatorhub --namespace openshift-cnv --output=jsonpath='{$.status.installplan.name}' + register: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name + until: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.stdout != "" + changed_when: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.stdout != "" + +- name: Wait for Install Plan to finish + ansible.builtin.command: "oc wait installplan \ + {{ __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.stdout }} --namespace openshift-cnv --for=condition='Installed' --timeout='5m'" + register: __sap_hypervisor_node_preconfigure_register_wait_for_installplan + changed_when: __sap_hypervisor_node_preconfigure_register_wait_for_installplan.rc != 0 + +- name: Wait + ansible.builtin.pause: + seconds: 300 + +- name: Create CNV HyperConverged + kubernetes.core.k8s: + state: present + definition: + apiVersion: hco.kubevirt.io/v1beta1 + kind: HyperConverged + metadata: + name: kubevirt-hyperconverged + namespace: openshift-cnv + spec: + +- name: Wait + ansible.builtin.pause: + seconds: 300 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml new file mode 100644 index 000000000..a961de89f --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml @@ -0,0 +1,89 @@ +--- +- name: Delete the nmstate operator namespace + kubernetes.core.k8s: + state: absent + definition: + apiVersion: v1 + kind: Namespace + metadata: + labels: + kubernetes.io/metadata.name: openshift-nmstate + name: openshift-nmstate + name: openshift-nmstate + spec: + finalizers: + - kubernetes + +- name: Pause to give operator a chance to uninstall + ansible.builtin.pause: + minutes: 2 + +- name: Create the nmstate operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + labels: + kubernetes.io/metadata.name: openshift-nmstate + name: openshift-nmstate + name: openshift-nmstate + spec: + finalizers: + - kubernetes + +- name: Create the OperatorGroup + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + annotations: + olm.providedAPIs: NMState.v1.nmstate.io + generateName: openshift-nmstate- + name: openshift-nmstate-tn6k8 + namespace: openshift-nmstate + spec: + targetNamespaces: + - openshift-nmstate + +- name: Pause to give operator a chance to install + ansible.builtin.pause: + minutes: 2 + +- name: Subscribe to the nmstate Operator + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + labels: + operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: "" + name: kubernetes-nmstate-operator + namespace: openshift-nmstate + spec: + channel: stable + installPlanApproval: Automatic + name: kubernetes-nmstate-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + +- name: Pause to give operator a chance to install + ansible.builtin.pause: + minutes: 5 + +- name: Create instance of the nmstate operator + kubernetes.core.k8s: + state: present + definition: + apiVersion: nmstate.io/v1 + kind: NMState + metadata: + name: nmstate + +- name: Pause to give instance a chance to come up + ansible.builtin.pause: + minutes: 5 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml new file mode 100644 index 000000000..1379b1152 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml @@ -0,0 +1,55 @@ +--- +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: openshift-sriov-network-operator + +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: sriov-network-operators + namespace: openshift-sriov-network-operator + spec: + targetNamespaces: + - openshift-sriov-network-operator +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: sriov-network-operator-subscription + namespace: openshift-sriov-network-operator + spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: sriov-network-operator + # startingCSV: sriov-network-operator + channel: "stable" + + +- name: Pause to give operator a chance to install + ansible.builtin.pause: + minutes: 3 + +- name: Copy patch to enable unsupported NICs + ansible.builtin.copy: + src: sriov-enabled-unsupported-nics.sh + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/sriov-enabled-unsupported-nics.sh" + mode: "0755" + when: sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics + +- name: Enable unsupported NICs + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/sriov-enabled-unsupported-nics.sh" + when: sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics + register: __sap_hypervisor_node_preconfigure_register_enable_unsupported_nics + changed_when: __sap_hypervisor_node_preconfigure_register_enable_unsupported_nics.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml new file mode 100644 index 000000000..9747b55ad --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml @@ -0,0 +1,48 @@ +--- +- name: Download trident + ansible.builtin.unarchive: + remote_src: true + src: "{{ sap_hypervisor_node_preconfigure_install_trident_url }}" + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/" + +- name: Uninstall trident + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/trident-installer/tridentctl uninstall -n trident" + ignore_errors: true + register: __sap_hypervisor_node_preconfigure_register_uninstall_trident + changed_when: __sap_hypervisor_node_preconfigure_register_uninstall_trident.rc != 0 + +- name: Install trident + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/trident-installer/tridentctl install -n trident" + register: __sap_hypervisor_node_preconfigure_register_install_trident + changed_when: __sap_hypervisor_node_preconfigure_register_install_trident.rc != 0 + +- name: Copy backend file + ansible.builtin.template: + src: "trident-backend.json.j2" + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/trident-backend.json" + mode: "0644" + +- name: Create trident backend + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}\ + /trident-installer/tridentctl -n trident create backend -f\ + {{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}\ + /trident-backend.json" + register: __sap_hypervisor_node_preconfigure_register_create_trident_backend + changed_when: __sap_hypervisor_node_preconfigure_register_create_trident_backend.rc != 0 + +- name: Create storage class + kubernetes.core.k8s: + state: present + definition: + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: nas + annotations: + storageclass.kubernetes.io/is-default-class: 'true' + provisioner: csi.trident.netapp.io + parameters: + backendType: "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}" + snapshots: "True" + provisioningType: "thin" + encryption: "false" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml new file mode 100644 index 000000000..e4555f803 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml @@ -0,0 +1,15 @@ +--- +- name: Create ~/bin + ansible.builtin.file: + path: ~/bin + state: directory + mode: "0755" + +- name: Get and extract virtctl +# become: yes + ansible.builtin.unarchive: + validate_certs: false + remote_src: true + src: "https://hyperconverged-cluster-cli-download-openshift-cnv.apps.\ + {{ sap_hypervisor_node_preconfigure_cluster_config.cluster_url }}/amd64/linux/virtctl.tar.gz" + dest: ~/bin diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml new file mode 100644 index 000000000..bd28ea55b --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml @@ -0,0 +1,11 @@ +--- +- name: Personalize template + ansible.builtin.template: + src: 99-kargs-worker.yml.j2 + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }}.yml.j2" + mode: "0644" + +- name: Enable hugepages + kubernetes.core.k8s: + state: present + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }}.yml.j2" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml new file mode 100644 index 000000000..57a52da24 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml @@ -0,0 +1,11 @@ +--- +- name: Label worker with invtsc flag + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: default + labels: + 'feature.node.kubernetes.io/cpu-feature-invtsc': enabled diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml new file mode 100644 index 000000000..9cc470a04 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -0,0 +1,61 @@ +--- +- name: Get a list of all nodes from any namespace + kubernetes.core.k8s_info: + kind: Node + register: __sap_hypervisor_node_preconfigure_register_node_list + +- name: Generate list with worker node names + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_node_name_list: + "{{ __sap_hypervisor_node_preconfigure_register_worker_node_name_list | \ + d([]) + [__sap_hypervisor_node_preconfigure_register_worker_node.name] }}" + with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker_node + +- name: Filter hosts + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_nodes: + "{{ __sap_hypervisor_node_preconfigure_register_nodes | \ + d([]) + [__sap_hypervisor_node_preconfigure_register_host] }}" + with_items: "{{ __sap_hypervisor_node_preconfigure_register_node_list['resources'] }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_host + when: __sap_hypervisor_node_preconfigure_register_host.metadata.name in __sap_hypervisor_node_preconfigure_register_worker_node_name_list + +- name: Assert that configured nodes are found + ansible.builtin.assert: + that: __sap_hypervisor_node_preconfigure_register_nodes is defined + fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster + success_msg: Configured nodes found + +# - meta: end_play + +- name: Include prepare + ansible.builtin.include_tasks: prepare.yml +- name: Include tuned virtual host + ansible.builtin.include_tasks: tuned-virtual-host.yml +- name: Include install CNV operator + ansible.builtin.include_tasks: install-cnv-operator.yml +- name: Include install sriov operator + ansible.builtin.include_tasks: install-sriov-operator.yml +- name: Include install nmstate operator + ansible.builtin.include_tasks: install-nmstate-operator.yml +- name: Include install virtctl + ansible.builtin.include_tasks: install-virtctl.yml +- name: Include setup worker node + ansible.builtin.include_tasks: setup-worker-node.yml + +# How to wait for node to be scheduleable? (NodeSchedulable) +- name: Wait for all k8s nodes to be ready + ansible.builtin.command: oc wait --for=condition=Ready nodes --all --timeout=3600s + register: __sap_hypervisor_node_preconfigure_register_nodes_ready + changed_when: __sap_hypervisor_node_preconfigure_register_nodes_ready.rc != 0 + +- name: Print nodes + ansible.builtin.debug: + var: __sap_hypervisor_node_preconfigure_register_nodes_ready.stdout_lines + +- name: Include Trident installation + ansible.builtin.include_tasks: install-trident.yml + when: sap_hypervisor_node_preconfigure_install_trident diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 new file mode 100644 index 000000000..1a39d0a06 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 @@ -0,0 +1,47 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + labels: + machineconfiguration.openshift.io/mco-built-in: "" + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} +spec: + configuration: + source: + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 00-worker + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-container-runtime + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-capabilities + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-seccomp-use-default + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-registries + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-ssh + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: + - worker + - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + nodeSelector: + matchLabels: + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + paused: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml new file mode 100644 index 000000000..088f86fe8 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -0,0 +1,99 @@ +--- +- name: Print network + ansible.builtin.debug: + var: __sap_hypervisor_node_preconfigure_register_worker_network + +- name: "Create NodeNetworkConfigurationPolicy\ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on\ + {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: nmstate.io/v1 + kind: NodeNetworkConfigurationPolicy + metadata: + name: "{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + spec: + nodeSelector: + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + desiredState: + interfaces: + - "{{ __sap_hypervisor_node_preconfigure_register_worker_network }}" + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' + +# XXX didn't work - why? +- name: "Create NetworkAttachmentDefinition {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + namespace: "{{ sap_hypervisor_node_preconfigure_cluster_config.vm_namespace }}" + name: "{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-network-definition" + annotations: + k8s.v1.cni.cncf.io/resourceName: "bridge.network.kubevirt.io/{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" + spec: + config: '{ + "cniVersion": "0.3.1", + "name": "sapbridge-network-definition", + "type": "cnv-bridge", + "bridge": "sapbridge", + "macspoofchk": true + }' + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' + +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }}\ + feature.node.kubernetes.io/network-sriov.capable=true --overwrite=true" + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' + register: __sap_hypervisor_node_preconfigure_register_label_node + changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 + +- name: "Create SRIOV NodeNetworkConfigurationPolicy\ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name.name }} on\ + {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: sriovnetwork.openshift.io/v1 + kind: SriovNetworkNodePolicy + metadata: + name: "iface-{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-sriov-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + namespace: openshift-sriov-network-operator + spec: + resourceName: "iface{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}sriov" + nodeSelector: + feature.node.kubernetes.io/network-sriov.capable: "true" + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + priority: 5 + mtu: 9000 + numVfs: 8 + nicSelector: + pfNames: ['{{ __sap_hypervisor_node_preconfigure_register_worker_network.interface }}#0-7'] + deviceType: vfio-pci + isRdma: false + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == "sriov" + +- name: "Create SriovNetwork Attachment Definition {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: sriovnetwork.openshift.io/v1 + kind: SriovNetwork + metadata: + name: "iface-{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-sriov" + namespace: openshift-sriov-network-operator + spec: + ipam: | + { + "type": "host-local", + "subnet": "192.168.1.0/24", + "rangeStart": "192.168.1.200", + "rangeEnd": "192.168.1.210" + } + networkNamespace: openshift-sriov-network-operator + resourceName: "iface{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}sriov" + spoofChk: "off" + trust: "on" + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == "sriov" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml new file mode 100644 index 000000000..dd879b22c --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml @@ -0,0 +1,9 @@ +--- +- name: Gather Facts + ansible.builtin.gather_facts: + +- name: Create Tempdir + ansible.builtin.tempfile: + state: directory + suffix: "_sap_hypervisor_node_preconfigure" + register: __sap_hypervisor_node_preconfigure_register_tmpdir diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml new file mode 100644 index 000000000..2dc78034f --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml @@ -0,0 +1,80 @@ +--- +- name: Include configure kargs + ansible.builtin.include_tasks: configure-kargs-per-node.yml + with_items: "{{ __sap_hypervisor_node_preconfigure_register_nodes }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker + index_var: __sap_hypervisor_node_preconfigure_register_worker_nr + +- name: Include configure worker + ansible.builtin.include_tasks: configure-worker-node.yml + with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker + index_var: __sap_hypervisor_node_preconfigure_register_worker_nr + +- name: Enable CPU Manager by patching MCP worker + kubernetes.core.k8s: + state: patched + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfigPool + metadata: + name: worker + labels: + custom-kubelet: cpumanager-enabled + + +- name: Create kubletconfig for cpumanager worker + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + kubernetes.core.k8s: + state: absent + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" + +- name: Create kubletconfig for cpumanager worker + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" + +- name: Create kubletconfig for cpumanager worker + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + machineconfiguration.openshift.io/role: worker + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh new file mode 100644 index 000000000..7732ba78e --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# in order to allow unsupported SRIOV nics such as Mellanox +oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' + + diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 new file mode 100644 index 000000000..183bfb353 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 @@ -0,0 +1,18 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} +spec: + config: + ignition: + version: 3.2.0 + kernelArguments: + - intel_iommu=on + - iommu=pt + - default_hugepagesz=1GB + - hugepagesz=1GB + - hugepages={{ __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages }} + - tsx=on diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 new file mode 100644 index 000000000..1a39d0a06 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 @@ -0,0 +1,47 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + labels: + machineconfiguration.openshift.io/mco-built-in: "" + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} +spec: + configuration: + source: + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 00-worker + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-container-runtime + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-capabilities + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-seccomp-use-default + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-registries + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-ssh + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: + - worker + - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + nodeSelector: + matchLabels: + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + paused: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 new file mode 100644 index 000000000..e422aab11 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 @@ -0,0 +1,18 @@ +{ + "nfsMountOptions": "nfsvers=3", + "defaults": { + "exportPolicy": "default" + }, + "debug":false, + "managementLIF":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.management }}", + "dataLIF":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.data }}", + "svm":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.svm }}", + "backendName": "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.backend }}", + "aggregate":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.aggregate }}", + "username":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.username }}", + "password":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.password }}", + "storageDriverName":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}", + "storagePrefix":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_prefix }}", + "version":1 +} + diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml new file mode 100644 index 000000000..e2dd4f483 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml @@ -0,0 +1,21 @@ +--- +- name: Set virtual-host for worker nodes + kubernetes.core.k8s: + state: present + definition: + apiVersion: tuned.openshift.io/v1 + kind: Tuned + metadata: + name: virtual-host + namespace: openshift-cluster-node-tuning-operator + spec: + profile: + - data: | + [main] + include=virtual-host + name: virtual-host + recommend: + - match: + - label: "node-role.kubernetes.io/worker" + priority: 10 + profile: virtual-host diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml index 52cd899ce..379ea44e4 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml @@ -4,11 +4,13 @@ - name: Get kernel command line ansible.builtin.command: cat /proc/cmdline register: __sap_hypervisor_node_preconfigure_kernelcmdline_assert + changed_when: __sap_hypervisor_node_preconfigure_kernelcmdline_assert.rc != 0 - name: "Assert - Kernel same page merging (KSM): Get status" - ansible.builtin.shell: systemctl status ksm + ansible.builtin.command: systemctl status ksm register: __sap_hypervisor_node_preconfigure_ksmstatus_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_ksmstatus_assert.rc != 0 - name: "Assert - Kernel same page merging (KSM): Check if stopped" ansible.builtin.assert: @@ -18,9 +20,10 @@ ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - name: "Assert - Kernel same page merging (KSM) Tuning Daemon: Get status" - ansible.builtin.shell: systemctl status ksmtuned + ansible.builtin.command: systemctl status ksmtuned register: __sap_hypervisor_node_preconfigure_ksmtunedstatus_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_ksmtunedstatus_assert.rc != 0 - name: "Assert - Kernel same page merging (KSM) Tuning Daemon: Check if stopped" ansible.builtin.assert: @@ -30,10 +33,12 @@ ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - name: Check CPU Stepping - ansible.builtin.shell: lscpu | awk '/Stepping/{print $2}' + ansible.builtin.shell: set -o pipefail && lscpu | awk '/Stepping/{print $2}' register: __sap_hypervisor_node_preconfigure_cpu_stepping_output_assert + changed_when: __sap_hypervisor_node_preconfigure_cpu_stepping_output_assert.rc != 0 -- set_fact: +- name: Register stepping as fact + ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_cpu_stepping_assert: "{{ __sap_hypervisor_node_preconfigure_cpu_stepping_output_assert.stdout }}" - name: Print CPU Stepping @@ -42,11 +47,13 @@ # skylake: - name: Assert - Check Intel Skylake CPU Platform + when: __sap_hypervisor_node_preconfigure_cpu_stepping_assert == "4" block: - name: Get ple_gap ansible.builtin.command: grep -E '^options\s+kvm_intel.*?ple_gap\s*=\s*0.*$' /etc/modprobe.d/kvm.conf register: __sap_hypervisor_node_preconfigure_skylake_plegap_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_skylake_plegap_assert.rc != 0 - name: Assert - Check if ple_gap=0 ansible.builtin.assert: @@ -61,9 +68,9 @@ fail_msg: "FAIL: spectre_v2=retpoline is not on Kernel command line" success_msg: "PASS: spectre_v2=retpoline is on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - when: __sap_hypervisor_node_preconfigure_cpu_stepping_assert == "4" - name: Assert - check sap_hypervisor_node_preconfigure_nx_huge_pages + when: sap_hypervisor_node_preconfigure_nx_huge_pages is defined block: - name: "Assert - Check kvm.nx_huge_pages is {{ sap_hypervisor_node_preconfigure_nx_huge_pages }}" ansible.builtin.assert: @@ -72,13 +79,13 @@ success_msg: "PASS: kvm.nx_huge_pages is {{ sap_hypervisor_node_preconfigure_nx_huge_pages }}" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - when: sap_hypervisor_node_preconfigure_nx_huge_pages is defined - - name: Assert - check seccomp_sanbox=0 block: - - command: grep -E '^seccomp_sandbox\s+=\s+0.*$' /etc/libvirt/qemu.conf + - name: Get seccomp setting + ansible.builtin.command: grep -E '^seccomp_sandbox\s+=\s+0.*$' /etc/libvirt/qemu.conf register: __sap_hypervisor_node_preconfigure_seccomp_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_seccomp_assert.rc != 0 - name: "Assert - Check seccomp_sanbox=0 is in /etc/libvirt/qemu.conf" ansible.builtin.assert: @@ -90,38 +97,37 @@ - name: Assert - check amount of 1G hugepages block: - name: Get amount of 1G hugepages - ansible.builtin.shell: hugeadm --pool-list | grep 1073741824 | awk '{print $3}' - register: __sap_hypervisor_node_preconfigure_1Ghugepages_assert + ansible.builtin.shell: set -o pipefail && hugeadm --pool-list | grep 1073741824 | awk '{print $3}' + register: __sap_hypervisor_node_preconfigure_1g_hugepages_assert + changed_when: __sap_hypervisor_node_preconfigure_1g_hugepages_assert.rc != 0 - name: "Check that at least {{ sap_hypervisor_node_preconfigure_reserved_ram }} GB are available for the hypervisor and the rest are 1G hugepages" ansible.builtin.assert: - that: "{{ ( ansible_memtotal_mb / 1024 )|int - sap_hypervisor_node_preconfigure_reserved_ram }} >= {{ __sap_hypervisor_node_preconfigure_1Ghugepages_assert.stdout }}" + that: "{{ (ansible_memtotal_mb / 1024) | int - sap_hypervisor_node_preconfigure_reserved_ram }} >= {{ __sap_hypervisor_node_preconfigure_1g_hugepages_assert.stdout }}" fail_msg: "FAIL: Not enough memory reserved for hypervisor" success_msg: "PASS: Enough memory reserved for hypervisor" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - name: Assert - check Kernel command line block: - - assert: + - name: Ensure iommu is enabled + ansible.builtin.assert: that: "'intel_iommu=on' in __sap_hypervisor_node_preconfigure_kernelcmdline_assert.stdout" fail_msg: "FAIL: intel_iommu=on not on Kernel command line" success_msg: "PASS: intel_iommu=on on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - - assert: + - name: Ensure iommu passthrough is enabled + ansible.builtin.assert: that: "'iommu=pt' in __sap_hypervisor_node_preconfigure_kernelcmdline_assert.stdout" fail_msg: "FAIL: iommu=pt not on Kernel command line" success_msg: "PASS: iommu=pt on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - - assert: + # XXX shouldn't tsx be on? + - name: Ensure tsx is off + ansible.builtin.assert: that: "'tsx=off' in __sap_hypervisor_node_preconfigure_kernelcmdline_assert.stdout" fail_msg: "FAIL: tsx=off not on Kernel command line" success_msg: "PASS: tsx=off on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - - -#- name: Trigger tuned profile sap-hana-kvm activation -# include_tasks: set-tuned-profile.yml -# -##### install hooks: HP, cpufreq diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml index 2abf6750e..4838f18fd 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml @@ -2,6 +2,7 @@ - name: Check file permissions ansible.builtin.command: "stat -c%a /usr/libexec/vdsm/hooks/before_vm_start/{{ item }}" register: __sap_hypervisor_node_preconfigure_register_file_permissions_assert + changed_when: __sap_hypervisor_node_preconfigure_register_file_permissions_assert.rc != 0 - name: Assert hook file permissions ansible.builtin.assert: @@ -14,16 +15,19 @@ ansible.builtin.file: path: /tmp/sap_hypervisor_node_preconfigure state: directory + mode: "0755" - name: Copy hook for checking ansible.builtin.copy: dest: "/tmp/sap_hypervisor_node_preconfigure/{{ item }}" src: "{{ item }}" + mode: "0755" - name: Diff hook ansible.builtin.command: "diff -uw /tmp/sap_hypervisor_node_preconfigure/{{ item }} /usr/libexec/vdsm/hooks/before_vm_start/{{ item }}" register: __sap_hypervisor_node_preconfigure_register_hook_diff_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_register_hook_diff_assert.rc != 0 - name: Assert hook content ansible.builtin.assert: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml index cb6508c2f..ab0d0c9b3 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml @@ -4,6 +4,7 @@ - name: Get tuned profile ansible.builtin.command: tuned-adm active register: __sap_hypervisor_node_preconfigure_tuned_profile_assert + changed_when: __sap_hypervisor_node_preconfigure_tuned_profile_assert.rc != 0 - name: Verify tuned profile ansible.builtin.assert: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml index e7ae07c18..b49399e4f 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml @@ -1,60 +1,29 @@ --- # tasks file for sap_hypervisor_node_preconfigure -- name: Test if kernel same page merging (KSM) exists - ansible.builtin.shell: systemctl cat ksm - register: ksm - ignore_errors: true - become: true - become_user: root - -- name: Test if kernel same page merging (KSM) tuning daemon exists - ansible.builtin.shell: systemctl cat ksmtuned - register: ksmtuned - ignore_errors: true - become: true - become_user: root - -- name: Stop kernel same page merging (KSM) - ansible.builtin.shell: systemctl stop ksm - when: ksm.rc == 0 - become: true - become_user: root - -- name: Disable kernel same page merging (KSM) - ansible.builtin.shell: systemctl disable ksm - when: ksm.rc == 0 - become: true - become_user: root - -- name: Stop Kernel Samepage Merging (KSM) Tuning Daemon - ansible.builtin.shell: systemctl stop ksmtuned - when: ksmtuned.rc == 0 - become: true - become_user: root - -- name: Disable Kernel Samepage Merging (KSM) Tuning Daemon - ansible.builtin.shell: systemctl disable ksmtuned - when: ksmtuned.rc == 0 - become: true - become_user: root +- name: Stop and disable kernel same page merging (KSM) + ansible.builtin.systemd: + name: ksm + state: stopped + enabled: false + +- name: Stop and disable kernel same page merging (KSM) tuning daemon + ansible.builtin.systemd: + name: ksmtuned + state: stopped + enabled: false - name: Check CPU Stepping - ansible.builtin.shell: lscpu | awk '/Stepping/{print $2}' + ansible.builtin.shell: set -o pipefail && lscpu | awk '/Stepping/{print $2}' register: cpu_stepping_output - become: true - become_user: root + changed_when: cpu_stepping_output.rc != 0 -- set_fact: +- name: Register CPU stepping as fact + ansible.builtin.set_fact: cpu_stepping: "{{ cpu_stepping_output.stdout }}" become: true become_user: root -- name: Print CPU Stepping - ansible.builtin.shell: echo "{{ cpu_stepping }}" - become: true - become_user: root - # skylake: - name: Set ple_gap=0 on Intel Skylake CPU Platform ansible.builtin.lineinfile: @@ -108,7 +77,7 @@ become: true become_user: root -- name: Trigger tuned profile sap-hana-kvm activation +- name: Include allocate hughepages at runtime ansible.builtin.include_tasks: allocate-hugepages-at-runtime.yml when: sap_hypervisor_node_preconfigure_reserve_hugepages == "runtime" @@ -123,7 +92,7 @@ with_items: - default_hugepagesz=1GB - hugepagesz=1GB - - hugepages={{ ( ansible_memtotal_mb / 1024 )|int - sap_hypervisor_node_preconfigure_reserved_ram }} + - hugepages={{ (ansible_memtotal_mb / 1024) | int - sap_hypervisor_node_preconfigure_reserved_ram }} notify: __sap_hypervisor_node_preconfigure_regenerate_grub2_conf_handler tags: grubconfig when: sap_hypervisor_node_preconfigure_reserve_hugepages == "static" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml index 213a45bca..a8c606aa1 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml @@ -13,13 +13,17 @@ assert_prefix: "assert-" when: sap_hypervisor_node_preconfigure_assert|d(false) -- include_tasks: '{{ assert_prefix }}installation.yml' +- name: Include "{{ assert_prefix }}installation.yml" + ansible.builtin.include_tasks: '{{ assert_prefix }}installation.yml' -- include_tasks: '{{ assert_prefix }}configuration.yml' +- name: Include "{{ assert_prefix }}configuration.yml" + ansible.builtin.include_tasks: '{{ assert_prefix }}configuration.yml' -- include_tasks: '{{ assert_prefix }}set-tuned-profile.yml' +- name: Include "{{ assert_prefix }}set-tuned-profile.yml" + ansible.builtin.include_tasks: '{{ assert_prefix }}set-tuned-profile.yml' -- include_tasks: "{{ assert_prefix }}rhv-hooks.yml" +- name: Include "{{ assert_prefix }}rhv-hooks.yml" + ansible.builtin.include_tasks: "{{ assert_prefix }}rhv-hooks.yml" loop: - "{{ role_path }}/tasks/platform/{{ sap_hypervisor_node_platform }}/50_hana" - "{{ role_path }}/tasks/platform/{{ sap_hypervisor_node_platform }}/50_iothread_pinning" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml index 045b55069..ee0d63a8d 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml @@ -3,6 +3,7 @@ ansible.builtin.file: path: /usr/libexec/vdsm/hooks/before_vm_start state: directory + mode: "0755" become: true become_user: root @@ -10,6 +11,6 @@ ansible.builtin.copy: dest: "/usr/libexec/vdsm/hooks/before_vm_start/{{ item }}" src: "{{ item }}" - mode: '0755' + mode: "0755" become: true become_user: root diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml index 415c4a194..91c3d7757 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml @@ -3,12 +3,14 @@ ansible.builtin.file: path: /usr/lib/tuned/sap-hana-kvm-host state: directory + mode: "0755" become: true become_user: root - name: Create sap-hana-kvm-host tuned profile ansible.builtin.copy: dest: "/usr/lib/tuned/sap-hana-kvm-host/tuned.conf" + mode: "0644" content: | # # tuned configuration @@ -36,5 +38,7 @@ - name: Activate tuned profile ansible.builtin.command: tuned-adm profile sap-hana-kvm-host + register: __sap_hypervisor_node_preconfigre_register_tuned_activation_output become: true become_user: root + changed_when: __sap_hypervisor_node_preconfigre_register_tuned_activation_output.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml new file mode 100644 index 000000000..18a34fec4 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -0,0 +1,19 @@ +--- +# vars file for sap_hypervisor_node_preconfigure +# + +# Install the trident NFS storage provider +sap_hypervisor_node_preconfigure_install_trident: False +# URL of the trident installer package to use +sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v23.01.0/trident-installer-23.01.0.tar.gz + +# should SRIOV be enabled for unsupported NICs +sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True + +# Amount of memory [GB] to be reserved for the hypervisor on hosts >= 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB +# Amount of memory [GB] to be reserved for the hypervisor on hosts < 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB + +# Should the check for the minimal amount of be ignored? Minimal amount is 96 GB +sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False