From b2d215ea5580233309fe27cb883ae6df98dc644d Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Thu, 23 Nov 2023 12:49:20 +0100 Subject: [PATCH 1/6] sriov: Adding and renaming sriov role. --- roles/sriov/.ansible-lint | 4 + roles/sriov/.github/CODEOWNERS | 1 + .../sriov/.github/workflows/publish-role.yml | 12 +++ roles/sriov/.gitignore | 7 ++ roles/sriov/.pre-commit-config.yaml | 19 +++++ roles/sriov/.yamllint | 17 +++++ roles/sriov/README.md | 54 ++++++++++++++ roles/sriov/defaults/main.yml | 35 +++++++++ roles/sriov/handlers/main.yml | 1 + roles/sriov/meta/main.yml | 58 +++++++++++++++ roles/sriov/molecule/default/Dockerfile.j2 | 14 ++++ roles/sriov/molecule/default/INSTALL.rst | 22 ++++++ roles/sriov/molecule/default/files/uname | 12 +++ roles/sriov/molecule/default/molecule.yml | 33 +++++++++ roles/sriov/molecule/default/playbook.yml | 10 +++ roles/sriov/molecule/default/prepare.yml | 17 +++++ roles/sriov/molecule/default/requirements.yml | 3 + roles/sriov/molecule/tests/test_default.py | 11 +++ roles/sriov/molecule/vagrant/INSTALL.rst | 23 ++++++ roles/sriov/molecule/vagrant/molecule.yml | 31 ++++++++ roles/sriov/molecule/vagrant/playbook.yml | 5 ++ roles/sriov/molecule/vagrant/prepare.yml | 9 +++ roles/sriov/tasks/all.yml | 4 + roles/sriov/tasks/config.yml | 40 ++++++++++ roles/sriov/tasks/main.yml | 2 + roles/sriov/tasks/mellanox.yml | 48 ++++++++++++ roles/sriov/tasks/mlxconfig.yml | 73 +++++++++++++++++++ roles/sriov/vars/main.yml | 2 + 28 files changed, 567 insertions(+) create mode 100644 roles/sriov/.ansible-lint create mode 100644 roles/sriov/.github/CODEOWNERS create mode 100644 roles/sriov/.github/workflows/publish-role.yml create mode 100644 roles/sriov/.gitignore create mode 100644 roles/sriov/.pre-commit-config.yaml create mode 100644 roles/sriov/.yamllint create mode 100644 roles/sriov/README.md create mode 100644 roles/sriov/defaults/main.yml create mode 100644 roles/sriov/handlers/main.yml create mode 100644 roles/sriov/meta/main.yml create mode 100644 roles/sriov/molecule/default/Dockerfile.j2 create mode 100644 roles/sriov/molecule/default/INSTALL.rst create mode 100644 roles/sriov/molecule/default/files/uname create mode 100644 roles/sriov/molecule/default/molecule.yml create mode 100644 roles/sriov/molecule/default/playbook.yml create mode 100644 roles/sriov/molecule/default/prepare.yml create mode 100644 roles/sriov/molecule/default/requirements.yml create mode 100644 roles/sriov/molecule/tests/test_default.py create mode 100644 roles/sriov/molecule/vagrant/INSTALL.rst create mode 100644 roles/sriov/molecule/vagrant/molecule.yml create mode 100644 roles/sriov/molecule/vagrant/playbook.yml create mode 100644 roles/sriov/molecule/vagrant/prepare.yml create mode 100644 roles/sriov/tasks/all.yml create mode 100644 roles/sriov/tasks/config.yml create mode 100644 roles/sriov/tasks/main.yml create mode 100644 roles/sriov/tasks/mellanox.yml create mode 100644 roles/sriov/tasks/mlxconfig.yml create mode 100644 roles/sriov/vars/main.yml diff --git a/roles/sriov/.ansible-lint b/roles/sriov/.ansible-lint new file mode 100644 index 0000000..e0635bb --- /dev/null +++ b/roles/sriov/.ansible-lint @@ -0,0 +1,4 @@ +skip_list: + # Can't skip linting on a multiline block, see: + # https://github.com/ansible/ansible-lint/issues/557 + - '204' diff --git a/roles/sriov/.github/CODEOWNERS b/roles/sriov/.github/CODEOWNERS new file mode 100644 index 0000000..24e3ab1 --- /dev/null +++ b/roles/sriov/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stackhpc/ansible diff --git a/roles/sriov/.github/workflows/publish-role.yml b/roles/sriov/.github/workflows/publish-role.yml new file mode 100644 index 0000000..51ecc35 --- /dev/null +++ b/roles/sriov/.github/workflows/publish-role.yml @@ -0,0 +1,12 @@ +--- +name: Publish Ansible Role +'on': + push: + tags: + - "v?[0-9]+.[0-9]+.[0-9]+" + workflow_dispatch: +jobs: + publish_role: + uses: stackhpc/.github/.github/workflows/publish-role.yml@main + secrets: + GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} diff --git a/roles/sriov/.gitignore b/roles/sriov/.gitignore new file mode 100644 index 0000000..5652df4 --- /dev/null +++ b/roles/sriov/.gitignore @@ -0,0 +1,7 @@ +__pycache__ +*.py[cod] +*$py.class +.molecule +.cache +*pytestdebug.log +roles/ diff --git a/roles/sriov/.pre-commit-config.yaml b/roles/sriov/.pre-commit-config.yaml new file mode 100644 index 0000000..7df9713 --- /dev/null +++ b/roles/sriov/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +--- + +# See http://pre-commit.com for more information +# See http://pre-commit.com/hooks.html for more hooks + +repos: + + - repo: https://github.com/pre-commit/pre-commit-hooks + sha: v2.2.3 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: detect-private-key + + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.4.4 + hooks: + - id: autopep8 diff --git a/roles/sriov/.yamllint b/roles/sriov/.yamllint new file mode 100644 index 0000000..a52f204 --- /dev/null +++ b/roles/sriov/.yamllint @@ -0,0 +1,17 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable + comments-indentation: disable + +ignore: | + roles + virtenv + venv diff --git a/roles/sriov/README.md b/roles/sriov/README.md new file mode 100644 index 0000000..600f4eb --- /dev/null +++ b/roles/sriov/README.md @@ -0,0 +1,54 @@ +stackhpc.sriov +============== + +[![Build Status](https://travis-ci.com/stackhpc/ansible-role-sriov.svg?branch=master)](https://travis-ci.com/stackhpc/ansible-role-sriov) + +Ansible role to enable SR-IOV on network devices. + +Requirements +------------ +None + +Role Variables +-------------- + +See `defaults/main.yml` + +Dependencies +------------ + +- `stackhpc.grubcmdline` + +Example Playbook +---------------- + +``` +- name: configure sr-iov + hosts: compute + vars: + sriov_devices: + - name: p4p1 + numvfs: 63 + - name: p3p1 + numvfs: 8 + # Don't add a udev rule to set numvfs. This can be useful if you use an alternative method + # to set the number of virtual functions e.g some custom scripts to enable VFLAG, but want + # to use the role to set firmware parameters. + on_boot_configuration_enabled: false + tasks: + - include_role: + name: sriov + handlers: + - name: reboot + include_tasks: tasks/reboot.yml +``` + +License +------- + +Apache2 + +Author Information +------------------ + +Will Szumski diff --git a/roles/sriov/defaults/main.yml b/roles/sriov/defaults/main.yml new file mode 100644 index 0000000..a8d702a --- /dev/null +++ b/roles/sriov/defaults/main.yml @@ -0,0 +1,35 @@ +--- +# List of NICs to configure +sriov_devices: [] + +sriov_workdir: "{{ ansible_facts.env.HOME}}/.sriov" + +sriov_mft_url: https://www.mellanox.com/downloads/MFT/mft-4.23.0-104-x86_64-rpm.tgz + +sriov_mft_tarball: "{{ sriov_mft_url | urlsplit('path') | basename }}" + +sriov_os_pkgs: + - gcc + - rpm-build + # -E- Could not find lspci, you may need to install "pciutils" package + - pciutils + # NOTE(wszumski): Assumes running latest kernel. We could include package version with + # output of uname -r, but the packages aren't always available. + - kernel-devel + - make + +sriov_restart_handler: "reboot" + +sriov_numvfs: 8 + +sriov_udev_rule_path: /etc/udev/rules.d/70-sriov.rules + +sriov_mellanox_vendor_ids: + - "0x15b3" + +# Flag to skip installing any software dependencies. This may be useful if you +# are providing them by some other means. +sriov_install_enabled: true + +# Set iommu=pt on the kernel cmdline +sriov_iommu_pt_enabled: true diff --git a/roles/sriov/handlers/main.yml b/roles/sriov/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/sriov/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/sriov/meta/main.yml b/roles/sriov/meta/main.yml new file mode 100644 index 0000000..f18e7ab --- /dev/null +++ b/roles/sriov/meta/main.yml @@ -0,0 +1,58 @@ +--- +galaxy_info: + + role_name: sriov + author: Will Szumski + description: Ansible role to enable SR-IOV + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + license: Apache + + min_ansible_version: 2.4 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + # github_branch: + + # + # platforms is a list of platforms, and each platform has a name and a list of versions. + # + + platforms: + - name: EL + versions: + - 7 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + +# List your role dependencies here, one per line. Be sure to remove the '[]' above, +# if you add dependencies to this list. + +# this is an example from github +# - src: https://github.com/chusiang/helloworld.ansible.role.git + # scm: git + # version: master + # name: ansible-role-helloworld + +# this is an example from ansible galaxy +# - src: geerlingguy.apache + # version: master + # name: ansible-role-geerlingguy-apache diff --git a/roles/sriov/molecule/default/Dockerfile.j2 b/roles/sriov/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/sriov/molecule/default/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/sriov/molecule/default/INSTALL.rst b/roles/sriov/molecule/default/INSTALL.rst new file mode 100644 index 0000000..6a44bde --- /dev/null +++ b/roles/sriov/molecule/default/INSTALL.rst @@ -0,0 +1,22 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* Docker Engine + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[docker]' diff --git a/roles/sriov/molecule/default/files/uname b/roles/sriov/molecule/default/files/uname new file mode 100644 index 0000000..4464137 --- /dev/null +++ b/roles/sriov/molecule/default/files/uname @@ -0,0 +1,12 @@ +#!/bin/bash + +# MFT compile depends on detection of kernel version and architecture + +if [ $1 = "-m" ]; then + echo "x86_64" +else + for f in /usr/src/kernels/*; do + echo $(basename $f) + break + done +fi diff --git a/roles/sriov/molecule/default/molecule.yml b/roles/sriov/molecule/default/molecule.yml new file mode 100644 index 0000000..037d97b --- /dev/null +++ b/roles/sriov/molecule/default/molecule.yml @@ -0,0 +1,33 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: centos7-docker + image: mplachter/docker-centos7-molecule + command: /usr/sbin/init + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true +provisioner: + name: ansible + ansible_args: + - --skip-tags=skip_when_testing + options: + become: true + lint: + name: ansible-lint + options: + R: true +scenario: + name: default +verifier: + name: testinfra + directory: ../tests + options: + verbose: true + lint: + name: flake8 diff --git a/roles/sriov/molecule/default/playbook.yml b/roles/sriov/molecule/default/playbook.yml new file mode 100644 index 0000000..bb49314 --- /dev/null +++ b/roles/sriov/molecule/default/playbook.yml @@ -0,0 +1,10 @@ +--- +- name: Converge + hosts: all + roles: + - role: stackhpc.sriov + handlers: + - name: Simulate reboot + listen: reboot + debug: + msg: "Simulating reboot" diff --git a/roles/sriov/molecule/default/prepare.yml b/roles/sriov/molecule/default/prepare.yml new file mode 100644 index 0000000..990f61d --- /dev/null +++ b/roles/sriov/molecule/default/prepare.yml @@ -0,0 +1,17 @@ +--- + +- hosts: all + tasks: + - name: Install kernel packages + package: + name: + - kernel + - kernel-devel + state: present + - name: replace uname to fix detection of kernel and architecture in docker + copy: + src: files/uname + dest: /bin/uname + owner: root + group: root + mode: 0755 diff --git a/roles/sriov/molecule/default/requirements.yml b/roles/sriov/molecule/default/requirements.yml new file mode 100644 index 0000000..2223bbd --- /dev/null +++ b/roles/sriov/molecule/default/requirements.yml @@ -0,0 +1,3 @@ +--- +- src: git+https://github.com/stackhpc/ansible-role-grubcmdline + name: stackhpc.grubcmdline diff --git a/roles/sriov/molecule/tests/test_default.py b/roles/sriov/molecule/tests/test_default.py new file mode 100644 index 0000000..046223e --- /dev/null +++ b/roles/sriov/molecule/tests/test_default.py @@ -0,0 +1,11 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_stub(host): + # TODO: add some tests + pass diff --git a/roles/sriov/molecule/vagrant/INSTALL.rst b/roles/sriov/molecule/vagrant/INSTALL.rst new file mode 100644 index 0000000..4f44b67 --- /dev/null +++ b/roles/sriov/molecule/vagrant/INSTALL.rst @@ -0,0 +1,23 @@ +******* +Vagrant driver installation guide +******* + +Requirements +============ + +* Vagrant +* Virtualbox, Parallels, VMware Fusion, VMware Workstation or VMware Desktop + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[vagrant]' diff --git a/roles/sriov/molecule/vagrant/molecule.yml b/roles/sriov/molecule/vagrant/molecule.yml new file mode 100644 index 0000000..486fd8b --- /dev/null +++ b/roles/sriov/molecule/vagrant/molecule.yml @@ -0,0 +1,31 @@ +--- +dependency: + name: galaxy +driver: + name: vagrant + provider: + name: virtualbox +lint: + name: yamllint +platforms: + - name: bento-centos-7-vagrant + box: bento/centos-7 + cpus: 1 + memory: 1024 +provisioner: + name: ansible + options: + become: true + lint: + name: ansible-lint + options: + R: true +scenario: + name: vagrant +verifier: + name: testinfra + directory: ../tests + options: + verbose: true + lint: + name: flake8 diff --git a/roles/sriov/molecule/vagrant/playbook.yml b/roles/sriov/molecule/vagrant/playbook.yml new file mode 100644 index 0000000..d0ac6c3 --- /dev/null +++ b/roles/sriov/molecule/vagrant/playbook.yml @@ -0,0 +1,5 @@ +--- +- name: Converge + hosts: all + roles: + - role: stackhpc.sriov diff --git a/roles/sriov/molecule/vagrant/prepare.yml b/roles/sriov/molecule/vagrant/prepare.yml new file mode 100644 index 0000000..4b18d48 --- /dev/null +++ b/roles/sriov/molecule/vagrant/prepare.yml @@ -0,0 +1,9 @@ +--- +- name: Prepare + hosts: all + gather_facts: false + tasks: + - name: Install python for Ansible + raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) + become: true + changed_when: false diff --git a/roles/sriov/tasks/all.yml b/roles/sriov/tasks/all.yml new file mode 100644 index 0000000..3bd0387 --- /dev/null +++ b/roles/sriov/tasks/all.yml @@ -0,0 +1,4 @@ +--- + +- include_tasks: config.yml +- include_tasks: mellanox.yml diff --git a/roles/sriov/tasks/config.yml b/roles/sriov/tasks/config.yml new file mode 100644 index 0000000..181739d --- /dev/null +++ b/roles/sriov/tasks/config.yml @@ -0,0 +1,40 @@ +--- +- name: Persist sriov_numvfs with udev rule + blockinfile: + path: "{{ sriov_udev_rule_path }}" + block: | + {% for device in sriov_devices %} + {% if device.on_boot_configuration_enabled | default(true) | bool %} + SUBSYSTEM=="net", ACTION=="add", KERNEL=="{{ device.name }}", RUN+="/usr/bin/sh -c 'echo {{ device.numvfs | default(sriov_numvfs) }} > /sys/class/net/{{ device.name }}/device/sriov_numvfs'" + {% endif %} + {% endfor %} + marker_begin: BEGIN SRIOV + mode: 0644 + owner: root + group: root + create: true + become: true + notify: "{{ sriov_restart_handler }}" + +- name: Add iommu to kernel command line (Intel) + include_role: + name: stackhpc.grubcmdline + tags: skip_when_testing + vars: + kernel_cmdline: + - intel_iommu=on + kernel_cmdline_remove: + - ^intel_iommu= + kernel_restart_handler: "{{ sriov_restart_handler }}" + when: "'Intel' in ansible_facts.processor.0" + +- name: Set iommu=pt + include_role: + name: stackhpc.grubcmdline + tags: skip_when_testing + vars: + kernel_cmdline: + - iommu=pt + kernel_cmdline_remove: + - ^iommu= + kernel_restart_handler: "{{ sriov_restart_handler }}" diff --git a/roles/sriov/tasks/main.yml b/roles/sriov/tasks/main.yml new file mode 100644 index 0000000..2f454ab --- /dev/null +++ b/roles/sriov/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ sriov_action | default('all') }}.yml" diff --git a/roles/sriov/tasks/mellanox.yml b/roles/sriov/tasks/mellanox.yml new file mode 100644 index 0000000..394c3ff --- /dev/null +++ b/roles/sriov/tasks/mellanox.yml @@ -0,0 +1,48 @@ +--- + +- name: Determine list of pci addresses + command: grep PCI_SLOT_NAME /sys/class/net/{{ item }}/device/uevent + become: true + with_items: "{{ sriov_devices | map(attribute='name') | list }}" + register: network_devices + changed_when: false + +- name: Set default value for sriov_pci_addrs + set_fact: + sriov_pci_addrs: [] + +- name: set_fact containing list of pci_address + set_fact: + sriov_pci_addrs: "{{ sriov_pci_addrs + [pci_addr] }}" + vars: + pci_addr: "{{ item.stdout | regex_replace('^PCI_SLOT_NAME=', '') }}" + with_items: "{{ network_devices.results }}" + +- name: "Display sriov_pci_addrs value" + debug: + var: sriov_pci_addrs + +- name: Determine list of vendor ids + slurp: + src: /sys/class/net/{{ item }}/device/vendor + become: true + with_items: "{{ sriov_devices | map(attribute='name') | list }}" + register: vendors + changed_when: false + +- name: Set default value for sriov_pci_addrs + set_fact: + device_meta: "{{ sriov_devices | map(attribute='name') + | zip(vendors.results | map(attribute='content') | map ('b64decode') | map('trim'), sriov_pci_addrs, sriov_devices | map(attribute='numvfs', default=sriov_numvfs)) + | list }}" + +- debug: + var: device_meta + +- include_tasks: mlxconfig.yml + # e.g item=(u'enp3s0f1', u'0x15b3', u'0000:03:00.1', 31) + vars: + pci_addr: "{{ item[2] }}" + sriov_numvfs: "{{ item[3] }}" + loop: "{{ device_meta }}" + when: item[1] in sriov_mellanox_vendor_ids diff --git a/roles/sriov/tasks/mlxconfig.yml b/roles/sriov/tasks/mlxconfig.yml new file mode 100644 index 0000000..dd0efb9 --- /dev/null +++ b/roles/sriov/tasks/mlxconfig.yml @@ -0,0 +1,73 @@ +--- + +- name: Install MFT dependencies + yum: + name: "{{ sriov_os_pkgs }}" + become: true + when: sriov_install_enabled + +- name: Make working directory + file: + state: directory + path: "{{ sriov_workdir }}" + when: sriov_install_enabled + +- name: Download MFT + vars: + # Strip the file extension + no_ext: "{{ (sriov_mft_tarball | splitext)[0] }}" + unarchive: + src: "{{ sriov_mft_url }}" + dest: "{{ sriov_workdir }}" + remote_src: true + creates: "{{ sriov_workdir }}/{{ no_ext }}" + when: sriov_install_enabled + +- name: Determine if we can run mst_status + command: mst status + become: true + failed_when: false + changed_when: false + register: mst_status_result + +- name: Install MFT + command: "{{ sriov_workdir }}/{{ no_ext }}/install.sh" + vars: + # Strip the file extension + no_ext: "{{ (sriov_mft_tarball | splitext)[0] }}" + become: true + register: mft_install_result + when: + - sriov_install_enabled + - mst_status_result.rc != 0 + +- name: "Run: mst start" + command: mst start + become: true + tags: + - skip_ansible_lint + - skip_when_testing + when: mft_install_result.changed + +- name: Check current settings + command: mlxconfig -d {{ pci_addr }} q + register: mlxconfig_result + become: true + changed_when: false + +- name: Enable SR-IOV with mlxconfig + command: mlxconfig -y -d {{ pci_addr }} set SRIOV_EN=1 + become: true + vars: + sriov_enabled: "{{ mlxconfig_result.stdout | regex_search('^.*SRIOV_EN.*True.*$', multiline=True) }}" + when: not sriov_enabled + notify: "{{ sriov_restart_handler }}" + +- name: Set numvfs with mlxconfig + command: mlxconfig -y -d {{ pci_addr }} set NUM_OF_VFS={{ sriov_numvfs }} + become: true + vars: + regex: '^.*NUM_OF_VFS.*?{{ sriov_numvfs }}.*$' + num_of_vfs: "{{ mlxconfig_result.stdout | regex_search(regex, multiline=True) }}" + when: not num_of_vfs + notify: "{{ sriov_restart_handler }}" diff --git a/roles/sriov/vars/main.yml b/roles/sriov/vars/main.yml new file mode 100644 index 0000000..4a8359d --- /dev/null +++ b/roles/sriov/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for stackhpc.sriov From 625ecd09e5244df4be5aeef2975188f34151c8ab Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Thu, 23 Nov 2023 12:56:34 +0100 Subject: [PATCH 2/6] sriov: removing unnecesarry files --- roles/sriov/.ansible-lint | 4 ---- roles/sriov/.github/CODEOWNERS | 1 - .../sriov/.github/workflows/publish-role.yml | 12 ------------ roles/sriov/.gitignore | 7 ------- roles/sriov/.pre-commit-config.yaml | 19 ------------------- roles/sriov/.yamllint | 17 ----------------- 6 files changed, 60 deletions(-) delete mode 100644 roles/sriov/.ansible-lint delete mode 100644 roles/sriov/.github/CODEOWNERS delete mode 100644 roles/sriov/.github/workflows/publish-role.yml delete mode 100644 roles/sriov/.gitignore delete mode 100644 roles/sriov/.pre-commit-config.yaml delete mode 100644 roles/sriov/.yamllint diff --git a/roles/sriov/.ansible-lint b/roles/sriov/.ansible-lint deleted file mode 100644 index e0635bb..0000000 --- a/roles/sriov/.ansible-lint +++ /dev/null @@ -1,4 +0,0 @@ -skip_list: - # Can't skip linting on a multiline block, see: - # https://github.com/ansible/ansible-lint/issues/557 - - '204' diff --git a/roles/sriov/.github/CODEOWNERS b/roles/sriov/.github/CODEOWNERS deleted file mode 100644 index 24e3ab1..0000000 --- a/roles/sriov/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @stackhpc/ansible diff --git a/roles/sriov/.github/workflows/publish-role.yml b/roles/sriov/.github/workflows/publish-role.yml deleted file mode 100644 index 51ecc35..0000000 --- a/roles/sriov/.github/workflows/publish-role.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: Publish Ansible Role -'on': - push: - tags: - - "v?[0-9]+.[0-9]+.[0-9]+" - workflow_dispatch: -jobs: - publish_role: - uses: stackhpc/.github/.github/workflows/publish-role.yml@main - secrets: - GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} diff --git a/roles/sriov/.gitignore b/roles/sriov/.gitignore deleted file mode 100644 index 5652df4..0000000 --- a/roles/sriov/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -__pycache__ -*.py[cod] -*$py.class -.molecule -.cache -*pytestdebug.log -roles/ diff --git a/roles/sriov/.pre-commit-config.yaml b/roles/sriov/.pre-commit-config.yaml deleted file mode 100644 index 7df9713..0000000 --- a/roles/sriov/.pre-commit-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -# See http://pre-commit.com for more information -# See http://pre-commit.com/hooks.html for more hooks - -repos: - - - repo: https://github.com/pre-commit/pre-commit-hooks - sha: v2.2.3 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: detect-private-key - - - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.4.4 - hooks: - - id: autopep8 diff --git a/roles/sriov/.yamllint b/roles/sriov/.yamllint deleted file mode 100644 index a52f204..0000000 --- a/roles/sriov/.yamllint +++ /dev/null @@ -1,17 +0,0 @@ -extends: default - -rules: - braces: - max-spaces-inside: 1 - level: error - brackets: - max-spaces-inside: 1 - level: error - line-length: disable - truthy: disable - comments-indentation: disable - -ignore: | - roles - virtenv - venv From d9019586d4b2f3bcf3049e9149a7abe68e84cc32 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Thu, 23 Nov 2023 13:31:19 +0100 Subject: [PATCH 3/6] lint: fixing lint violations --- roles/grubcmdline/tasks/main.yml | 2 +- roles/iommu/tasks/main.yml | 2 +- roles/sriov/defaults/main.yml | 5 ++-- roles/sriov/meta/main.yml | 4 ++-- roles/sriov/molecule/default/playbook.yml | 6 ++--- roles/sriov/molecule/default/prepare.yml | 12 +++++----- roles/sriov/molecule/vagrant/playbook.yml | 2 +- roles/sriov/molecule/vagrant/prepare.yml | 2 +- roles/sriov/tasks/all.yml | 6 +++-- roles/sriov/tasks/config.yml | 18 +++++++------- roles/sriov/tasks/main.yml | 3 ++- roles/sriov/tasks/mellanox.yml | 29 +++++++++++------------ roles/sriov/tasks/mlxconfig.yml | 24 +++++++++++-------- 13 files changed, 60 insertions(+), 55 deletions(-) diff --git a/roles/grubcmdline/tasks/main.yml b/roles/grubcmdline/tasks/main.yml index 8a329e7..f2f12d8 100644 --- a/roles/grubcmdline/tasks/main.yml +++ b/roles/grubcmdline/tasks/main.yml @@ -83,7 +83,7 @@ - ansible_facts.os_family == "RedHat" - ansible_facts.distribution_version is version('8.0', '<=') -- name: Generate new grub config (Ubuntu/Debian) # noqa: no-changed-when +- name: Generate new grub config (Ubuntu/Debian) # noqa: no-changed-when ansible.builtin.command: /usr/sbin/update-grub become: true when: ansible_facts.os_family == "Debian" diff --git a/roles/iommu/tasks/main.yml b/roles/iommu/tasks/main.yml index 03c8688..4ca9872 100644 --- a/roles/iommu/tasks/main.yml +++ b/roles/iommu/tasks/main.yml @@ -3,7 +3,7 @@ ansible.builtin.include_role: name: stackhpc.linux.grubcmdline vars: - kernel_cmdline: # noqa: var-naming[no-role-prefix] + kernel_cmdline: # noqa: var-naming[no-role-prefix] - intel_iommu=on kernel_cmdline_remove: # noqa: var-naming[no-role-prefix] - ^intel_iommu= diff --git a/roles/sriov/defaults/main.yml b/roles/sriov/defaults/main.yml index a8d702a..17376ce 100644 --- a/roles/sriov/defaults/main.yml +++ b/roles/sriov/defaults/main.yml @@ -1,8 +1,7 @@ --- # List of NICs to configure sriov_devices: [] - -sriov_workdir: "{{ ansible_facts.env.HOME}}/.sriov" +sriov_workdir: "{{ ansible_facts.env.HOME }}/.sriov" sriov_mft_url: https://www.mellanox.com/downloads/MFT/mft-4.23.0-104-x86_64-rpm.tgz @@ -18,7 +17,7 @@ sriov_os_pkgs: - kernel-devel - make -sriov_restart_handler: "reboot" +sriov_restart_handler: reboot sriov_numvfs: 8 diff --git a/roles/sriov/meta/main.yml b/roles/sriov/meta/main.yml index f18e7ab..152cfee 100644 --- a/roles/sriov/meta/main.yml +++ b/roles/sriov/meta/main.yml @@ -11,7 +11,7 @@ galaxy_info: license: Apache - min_ansible_version: 2.4 + min_ansible_version: "2.4" # If this a Container Enabled role, provide the minimum Ansible Container version. # min_ansible_container_version: @@ -31,7 +31,7 @@ galaxy_info: platforms: - name: EL versions: - - 7 + - all galaxy_tags: [] # List tags for your role here, one per line. A tag is a keyword that describes diff --git a/roles/sriov/molecule/default/playbook.yml b/roles/sriov/molecule/default/playbook.yml index bb49314..8392fa6 100644 --- a/roles/sriov/molecule/default/playbook.yml +++ b/roles/sriov/molecule/default/playbook.yml @@ -2,9 +2,9 @@ - name: Converge hosts: all roles: - - role: stackhpc.sriov + - role: sriov handlers: - name: Simulate reboot listen: reboot - debug: - msg: "Simulating reboot" + ansible.builtin.debug: + msg: Simulating reboot diff --git a/roles/sriov/molecule/default/prepare.yml b/roles/sriov/molecule/default/prepare.yml index 990f61d..b66fbef 100644 --- a/roles/sriov/molecule/default/prepare.yml +++ b/roles/sriov/molecule/default/prepare.yml @@ -1,17 +1,17 @@ --- - -- hosts: all +- name: Install kernel packages and replace uname + hosts: all tasks: - name: Install kernel packages - package: + ansible.builtin.package: name: - kernel - kernel-devel state: present - - name: replace uname to fix detection of kernel and architecture in docker - copy: + - name: Replace uname to fix detection of kernel and architecture in docker + ansible.builtin.copy: src: files/uname dest: /bin/uname owner: root group: root - mode: 0755 + mode: "0755" diff --git a/roles/sriov/molecule/vagrant/playbook.yml b/roles/sriov/molecule/vagrant/playbook.yml index d0ac6c3..618c1b0 100644 --- a/roles/sriov/molecule/vagrant/playbook.yml +++ b/roles/sriov/molecule/vagrant/playbook.yml @@ -2,4 +2,4 @@ - name: Converge hosts: all roles: - - role: stackhpc.sriov + - role: sriov diff --git a/roles/sriov/molecule/vagrant/prepare.yml b/roles/sriov/molecule/vagrant/prepare.yml index 4b18d48..cd7e21a 100644 --- a/roles/sriov/molecule/vagrant/prepare.yml +++ b/roles/sriov/molecule/vagrant/prepare.yml @@ -4,6 +4,6 @@ gather_facts: false tasks: - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) + ansible.builtin.raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) become: true changed_when: false diff --git a/roles/sriov/tasks/all.yml b/roles/sriov/tasks/all.yml index 3bd0387..d3cd8fe 100644 --- a/roles/sriov/tasks/all.yml +++ b/roles/sriov/tasks/all.yml @@ -1,4 +1,6 @@ --- +- name: Include config tasks + ansible.builtin.include_tasks: config.yml -- include_tasks: config.yml -- include_tasks: mellanox.yml +- name: Include mellanox tasks + ansible.builtin.include_tasks: mellanox.yml diff --git a/roles/sriov/tasks/config.yml b/roles/sriov/tasks/config.yml index 181739d..f6539a0 100644 --- a/roles/sriov/tasks/config.yml +++ b/roles/sriov/tasks/config.yml @@ -1,15 +1,15 @@ --- - name: Persist sriov_numvfs with udev rule - blockinfile: + ansible.builtin.blockinfile: path: "{{ sriov_udev_rule_path }}" block: | - {% for device in sriov_devices %} + {% for device in sriov_devices %} {% if device.on_boot_configuration_enabled | default(true) | bool %} SUBSYSTEM=="net", ACTION=="add", KERNEL=="{{ device.name }}", RUN+="/usr/bin/sh -c 'echo {{ device.numvfs | default(sriov_numvfs) }} > /sys/class/net/{{ device.name }}/device/sriov_numvfs'" {% endif %} {% endfor %} marker_begin: BEGIN SRIOV - mode: 0644 + mode: "0644" owner: root group: root create: true @@ -17,24 +17,24 @@ notify: "{{ sriov_restart_handler }}" - name: Add iommu to kernel command line (Intel) - include_role: + ansible.builtin.include_role: name: stackhpc.grubcmdline tags: skip_when_testing vars: - kernel_cmdline: + kernel_cmdline: # noqa var-naming[no-role-prefix] - intel_iommu=on - kernel_cmdline_remove: + kernel_cmdline_remove: # noqa var-naming[no-role-prefix] - ^intel_iommu= kernel_restart_handler: "{{ sriov_restart_handler }}" when: "'Intel' in ansible_facts.processor.0" - name: Set iommu=pt - include_role: + ansible.builtin.include_role: name: stackhpc.grubcmdline tags: skip_when_testing vars: - kernel_cmdline: + kernel_cmdline: # noqa var-naming[no-role-prefix] - iommu=pt - kernel_cmdline_remove: + kernel_cmdline_remove: # noqa var-naming[no-role-prefix] - ^iommu= kernel_restart_handler: "{{ sriov_restart_handler }}" diff --git a/roles/sriov/tasks/main.yml b/roles/sriov/tasks/main.yml index 2f454ab..4115556 100644 --- a/roles/sriov/tasks/main.yml +++ b/roles/sriov/tasks/main.yml @@ -1,2 +1,3 @@ --- -- include: "{{ sriov_action | default('all') }}.yml" +- name: Include action tasks + ansible.builtin._include: "{{ sriov_action | default('all') }}.yml" diff --git a/roles/sriov/tasks/mellanox.yml b/roles/sriov/tasks/mellanox.yml index 394c3ff..91a9bb7 100644 --- a/roles/sriov/tasks/mellanox.yml +++ b/roles/sriov/tasks/mellanox.yml @@ -1,29 +1,28 @@ --- - - name: Determine list of pci addresses - command: grep PCI_SLOT_NAME /sys/class/net/{{ item }}/device/uevent + ansible.builtin.command: grep PCI_SLOT_NAME /sys/class/net/{{ item }}/device/uevent become: true with_items: "{{ sriov_devices | map(attribute='name') | list }}" register: network_devices changed_when: false - name: Set default value for sriov_pci_addrs - set_fact: + ansible.builtin.set_fact: sriov_pci_addrs: [] -- name: set_fact containing list of pci_address - set_fact: +- name: Add set_fact containing list of pci_address + ansible.builtin.set_fact: sriov_pci_addrs: "{{ sriov_pci_addrs + [pci_addr] }}" vars: pci_addr: "{{ item.stdout | regex_replace('^PCI_SLOT_NAME=', '') }}" with_items: "{{ network_devices.results }}" -- name: "Display sriov_pci_addrs value" - debug: +- name: Display sriov_pci_addrs value + ansible.builtin.debug: var: sriov_pci_addrs - name: Determine list of vendor ids - slurp: + ansible.builtin.slurp: src: /sys/class/net/{{ item }}/device/vendor become: true with_items: "{{ sriov_devices | map(attribute='name') | list }}" @@ -31,16 +30,16 @@ changed_when: false - name: Set default value for sriov_pci_addrs - set_fact: - device_meta: "{{ sriov_devices | map(attribute='name') - | zip(vendors.results | map(attribute='content') | map ('b64decode') | map('trim'), sriov_pci_addrs, sriov_devices | map(attribute='numvfs', default=sriov_numvfs)) - | list }}" + ansible.builtin.set_fact: + device_meta: "{{ sriov_devices | map(attribute='name') | zip(vendors.results | map(attribute='content') | map('b64decode') | map('trim'), sriov_pci_addrs, sriov_devices + | map(attribute='numvfs', default=sriov_numvfs)) | list }}" -- debug: +- name: Display device_meta variable + ansible.builtin.debug: var: device_meta -- include_tasks: mlxconfig.yml - # e.g item=(u'enp3s0f1', u'0x15b3', u'0000:03:00.1', 31) +- name: Include tasks in mlxconfig.yml + ansible.builtin.include_tasks: mlxconfig.yml vars: pci_addr: "{{ item[2] }}" sriov_numvfs: "{{ item[3] }}" diff --git a/roles/sriov/tasks/mlxconfig.yml b/roles/sriov/tasks/mlxconfig.yml index dd0efb9..c97d73d 100644 --- a/roles/sriov/tasks/mlxconfig.yml +++ b/roles/sriov/tasks/mlxconfig.yml @@ -1,22 +1,22 @@ --- - - name: Install MFT dependencies - yum: + ansible.builtin.yum: name: "{{ sriov_os_pkgs }}" become: true when: sriov_install_enabled - name: Make working directory - file: + ansible.builtin.file: state: directory path: "{{ sriov_workdir }}" + mode: "0700" when: sriov_install_enabled - name: Download MFT vars: # Strip the file extension no_ext: "{{ (sriov_mft_tarball | splitext)[0] }}" - unarchive: + ansible.builtin.unarchive: src: "{{ sriov_mft_url }}" dest: "{{ sriov_workdir }}" remote_src: true @@ -24,19 +24,21 @@ when: sriov_install_enabled - name: Determine if we can run mst_status - command: mst status + ansible.builtin.command: mst status become: true failed_when: false changed_when: false register: mst_status_result - name: Install MFT - command: "{{ sriov_workdir }}/{{ no_ext }}/install.sh" + ansible.builtin.command: "{{ sriov_workdir }}/{{ no_ext }}/install.sh" vars: # Strip the file extension no_ext: "{{ (sriov_mft_tarball | splitext)[0] }}" become: true register: mft_install_result + changed_when: mft_install_result.rc == 0 + when: - sriov_install_enabled - mst_status_result.rc != 0 @@ -50,24 +52,26 @@ when: mft_install_result.changed - name: Check current settings - command: mlxconfig -d {{ pci_addr }} q + ansible.builtin.command: mlxconfig -d {{ pci_addr }} q register: mlxconfig_result become: true changed_when: false - name: Enable SR-IOV with mlxconfig - command: mlxconfig -y -d {{ pci_addr }} set SRIOV_EN=1 + ansible.builtin.command: mlxconfig -y -d {{ pci_addr }} set SRIOV_EN=1 become: true vars: sriov_enabled: "{{ mlxconfig_result.stdout | regex_search('^.*SRIOV_EN.*True.*$', multiline=True) }}" when: not sriov_enabled notify: "{{ sriov_restart_handler }}" + changed_when: true - name: Set numvfs with mlxconfig - command: mlxconfig -y -d {{ pci_addr }} set NUM_OF_VFS={{ sriov_numvfs }} + ansible.builtin.command: mlxconfig -y -d {{ pci_addr }} set NUM_OF_VFS={{ sriov_numvfs }} become: true vars: - regex: '^.*NUM_OF_VFS.*?{{ sriov_numvfs }}.*$' + regex: ^.*NUM_OF_VFS.*?{{ sriov_numvfs }}.*$ num_of_vfs: "{{ mlxconfig_result.stdout | regex_search(regex, multiline=True) }}" when: not num_of_vfs notify: "{{ sriov_restart_handler }}" + changed_when: true From 41580994b32a4ba908d8b9f80b8f0de606e00bfb Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Thu, 23 Nov 2023 13:35:04 +0100 Subject: [PATCH 4/6] sanity: fixing shebang --- roles/sriov/molecule/default/files/uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sriov/molecule/default/files/uname b/roles/sriov/molecule/default/files/uname index 4464137..b4aa130 100644 --- a/roles/sriov/molecule/default/files/uname +++ b/roles/sriov/molecule/default/files/uname @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # MFT compile depends on detection of kernel version and architecture From 7b3734f3d762a2fe4c7287ba7443d7e1d66c3015 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Thu, 23 Nov 2023 14:14:24 +0100 Subject: [PATCH 5/6] sriov: Removing meta/main.yml --- roles/sriov/meta/main.yml | 58 --------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 roles/sriov/meta/main.yml diff --git a/roles/sriov/meta/main.yml b/roles/sriov/meta/main.yml deleted file mode 100644 index 152cfee..0000000 --- a/roles/sriov/meta/main.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -galaxy_info: - - role_name: sriov - author: Will Szumski - description: Ansible role to enable SR-IOV - - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - # issue_tracker_url: http://example.com/issue/tracker - - license: Apache - - min_ansible_version: "2.4" - - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - # Optionally specify the branch Galaxy will use when accessing the GitHub - # repo for this role. During role install, if no tags are available, - # Galaxy will use this branch. During import Galaxy will access files on - # this branch. If Travis integration is configured, only notifications for this - # branch will be accepted. Otherwise, in all cases, the repo's default branch - # (usually master) will be used. - # github_branch: - - # - # platforms is a list of platforms, and each platform has a name and a list of versions. - # - - platforms: - - name: EL - versions: - - all - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. - -dependencies: [] - -# List your role dependencies here, one per line. Be sure to remove the '[]' above, -# if you add dependencies to this list. - -# this is an example from github -# - src: https://github.com/chusiang/helloworld.ansible.role.git - # scm: git - # version: master - # name: ansible-role-helloworld - -# this is an example from ansible galaxy -# - src: geerlingguy.apache - # version: master - # name: ansible-role-geerlingguy-apache From cdcd3b60548bd4356217df6de96c0b99523c0e40 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Fri, 24 Nov 2023 13:27:56 +0100 Subject: [PATCH 6/6] sriov: fixing conditional, removing unnecessary empty line --- roles/sriov/tasks/mlxconfig.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/sriov/tasks/mlxconfig.yml b/roles/sriov/tasks/mlxconfig.yml index c97d73d..32d2581 100644 --- a/roles/sriov/tasks/mlxconfig.yml +++ b/roles/sriov/tasks/mlxconfig.yml @@ -37,8 +37,7 @@ no_ext: "{{ (sriov_mft_tarball | splitext)[0] }}" become: true register: mft_install_result - changed_when: mft_install_result.rc == 0 - + changed_when: true when: - sriov_install_enabled - mst_status_result.rc != 0