Fix kernel cmdline args we add not being propogated to newly installe…

…d kernels. (oamg#1193)

On some upgrades, any kernel commandline args that we were adding were added to the default kernel
but once the user installed a new kernel, those args were not propogated to the new kernel.  This
was happening on S390x and on RHEL7=>8 upgrades.

To fix this, we add the kernel commandline args to both the default kernel and to the defaults for
all kernels.

On S390x and upgrades to RHEL9 or greater, this is done by placing the kernel cmdline arguments into
the /etc/kernel/cmdline file.

On upgrades to RHEL <= 8 for all architectures other than S390x, this is done by having
grub2-editenv modify the /boot/grub2/grubenv file.

Jira: RHEL-26840, OAMG-10424
(cherry picked from commit 2b27cfb)

repos/system_upgrade/common/actors/kernelcmdlineconfig/actor.py

@@ -29,4 +29,5 @@ def process(self):
 
         if ff.firmware == 'bios' and os.path.ismount('/boot/efi'):
             configs = ['/boot/grub2/grub.cfg', '/boot/efi/EFI/redhat/grub.cfg']
-        kernelcmdlineconfig.modify_kernel_args_in_boot_cfg(configs)
+
+        kernelcmdlineconfig.entrypoint(configs)

repos/system_upgrade/common/actors/kernelcmdlineconfig/libraries/kernelcmdlineconfig.py

@@ -1,9 +1,27 @@
+import re
+
+from leapp import reporting
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries import stdlib
-from leapp.libraries.common.config import architecture
+from leapp.libraries.common.config import architecture, version
 from leapp.libraries.stdlib import api
 from leapp.models import InstalledTargetKernelInfo, KernelCmdlineArg, TargetKernelCmdlineArgTasks
 
+KERNEL_CMDLINE_FILE = "/etc/kernel/cmdline"
+
+
+class ReadOfKernelArgsError(Exception):
+    """
+    Failed to retrieve the kernel command line arguments
+    """
+
+
+def use_cmdline_file():
+    if (architecture.matches_architecture(architecture.ARCH_S390X) or
+            version.matches_target_version('>= 9.0')):
+        return True
+    return False
+
 
 def run_grubby_cmd(cmd):
     try:
@@ -15,6 +33,9 @@ def run_grubby_cmd(cmd):
             stdlib.run(['/usr/sbin/zipl'])
 
     except (OSError, stdlib.CalledProcessError) as e:
+        # In most cases we don't raise StopActorExecutionError in post-upgrade
+        # actors.
+        #
         raise StopActorExecutionError(
             "Failed to append extra arguments to kernel command line.",
             details={"details": str(e)})
@@ -31,22 +52,36 @@ def format_kernelarg_msgs_for_grubby_cmd(kernelarg_msgs):
     return ' '.join(kernel_args)
 
 
-def modify_kernel_args_in_boot_cfg(configs_to_modify_explicitly=None):
-    kernel_info = next(api.consume(InstalledTargetKernelInfo), None)
-    if not kernel_info:
-        return
+def set_default_kernel_args(kernel_args):
+    if use_cmdline_file():
+        # Put kernel_args into /etc/kernel/cmdline
+        with open(KERNEL_CMDLINE_FILE, 'w') as f:
+            f.write(kernel_args)
+    else:
+        # Use grub2-editenv to put the kernel args into /boot/grub2/grubenv
+        stdlib.run(['grub2-editenv', '-', 'set', 'kernelopts={}'.format(kernel_args)])
 
-    # Collect desired kernelopt modifications
+
+def retrieve_arguments_to_modify():
+    """
+    Retrieve the arguments other actors would like to add or remove from the kernel cmdline.
+    """
     kernelargs_msgs_to_add = list(api.consume(KernelCmdlineArg))
     kernelargs_msgs_to_remove = []
+
     for target_kernel_arg_task in api.consume(TargetKernelCmdlineArgTasks):
         kernelargs_msgs_to_add.extend(target_kernel_arg_task.to_add)
         kernelargs_msgs_to_remove.extend(target_kernel_arg_task.to_remove)
 
-    if not kernelargs_msgs_to_add and not kernelargs_msgs_to_remove:
-        return  # There is no work to do
+    return kernelargs_msgs_to_add, kernelargs_msgs_to_remove
+
 
-    grubby_modify_kernelargs_cmd = ['grubby', '--update-kernel={0}'.format(kernel_info.kernel_img_path)]
+def modify_args_for_default_kernel(kernel_info,
+                                   kernelargs_msgs_to_add,
+                                   kernelargs_msgs_to_remove,
+                                   configs_to_modify_explicitly=None):
+    grubby_modify_kernelargs_cmd = ['grubby',
+                                    '--update-kernel={0}'.format(kernel_info.kernel_img_path)]
 
     if kernelargs_msgs_to_add:
         grubby_modify_kernelargs_cmd += [
@@ -64,3 +99,116 @@ def modify_kernel_args_in_boot_cfg(configs_to_modify_explicitly=None):
             run_grubby_cmd(cmd)
     else:
         run_grubby_cmd(grubby_modify_kernelargs_cmd)
+
+
+def _extract_grubby_value(record):
+    data = record.split('=', 1)[1]
+    matches = re.match(r'^([\'"]?)(.*)\1$', data)
+    return matches.group(2)
+
+
+def retrieve_args_for_default_kernel(kernel_info):
+    # Copy the args for the default kernel to all kernels.
+    kernel_args = None
+    kernel_root = None
+    cmd = ['grubby', '--info', kernel_info.kernel_img_path]
+    output = stdlib.run(cmd, split=False)
+    for record in output['stdout'].splitlines():
+        # This could be done with one regex but it's cleaner to parse it as
+        # structured data.
+        if record.startswith('args='):
+            temp_kernel_args = _extract_grubby_value(record)
+
+            if kernel_args:
+                api.current_logger().warning('Grubby output is malformed:'
+                                             ' `args=` is listed more than once.')
+                if kernel_args != temp_kernel_args:
+                    raise ReadOfKernelArgsError('Grubby listed `args=` multiple'
+                                                ' times with different values.')
+            kernel_args = _extract_grubby_value(record)
+        elif record.startswith('root='):
+            api.current_logger().warning('Grubby output is malformed:'
+                                         ' `root=` is listed more than once.')
+            if kernel_root:
+                raise ReadOfKernelArgsError('Grubby listed `root=` multiple'
+                                            ' times with different values')
+            kernel_root = _extract_grubby_value(record)
+
+    if not kernel_args or not kernel_root:
+        raise ReadOfKernelArgsError(
+            'Failed to retrieve kernel command line to save for future installed'
+            ' kernels: root={}, args={}'.format(kernel_root, kernel_args)
+        )
+
+    return kernel_root, kernel_args
+
+
+def modify_kernel_args_in_boot_cfg(configs_to_modify_explicitly=None):
+    kernel_info = next(api.consume(InstalledTargetKernelInfo), None)
+    if not kernel_info:
+        return
+
+    # Collect desired kernelopt modifications
+    kernelargs_msgs_to_add, kernelargs_msgs_to_remove = retrieve_arguments_to_modify()
+    if not kernelargs_msgs_to_add and not kernelargs_msgs_to_remove:
+        # Nothing to do
+        return
+
+    # Modify the kernel cmdline for the default kernel
+    modify_args_for_default_kernel(kernel_info,
+                                   kernelargs_msgs_to_add,
+                                   kernelargs_msgs_to_remove,
+                                   configs_to_modify_explicitly)
+
+    # Copy kernel params from the default kernel to all the kernels
+    kernel_root, kernel_args = retrieve_args_for_default_kernel(kernel_info)
+    complete_kernel_args = 'root={} {}'.format(kernel_root, kernel_args)
+    set_default_kernel_args(complete_kernel_args)
+
+
+def entrypoint(configs=None):
+    try:
+        modify_kernel_args_in_boot_cfg(configs)
+    except ReadOfKernelArgsError as e:
+        api.current_logger().error(str(e))
+
+        if use_cmdline_file():
+            report_hint = reporting.Hints(
+                'After the system has been rebooted into the new version of RHEL, you'
+                ' should take the kernel cmdline arguments from /proc/cmdline (Everything'
+                ' except the BOOT_IMAGE entry and initrd entries) and copy them into'
+                ' /etc/kernel/cmdline before installing any new kernels.'
+            )
+        else:
+            report_hint = reporting.Hints(
+                'After the system has been rebooted into the new version of RHEL, you'
+                ' should take the kernel cmdline arguments from /proc/cmdline (Everything'
+                ' except the BOOT_IMAGE entry and initrd entries) and then use the'
+                ' grub2-editenv command to make them the default kernel args.  For example,'
+                ' if /proc/cmdline contains:\n\n'
+                '    BOOT_IMAGE=(hd0,msdos1)/vmlinuz-4.18.0-425.3.1.el8.x86_64'
+                ' root=/dev/mapper/rhel_ibm--root ro console=tty0'
+                ' console=ttyS0,115200 rd_NO_PLYMOUTH\n\n'
+                ' then run the following grub2-editenv command:\n\n'
+                '    # grub2-editenv - set "kernelopts=root=/dev/mapper/rhel_ibm--root'
+                ' ro console=tty0 console=ttyS0,115200 rd_NO_PLYMOUTH"'
+            )
+
+        reporting.create_report([
+            reporting.Title('Could not set the kernel arguments for future kernels'),
+            reporting.Summary(
+                'During the upgrade we needed to modify the kernel command line arguments.'
+                ' We were able to change the arguments for the default kernel but we were'
+                ' not able to set the arguments as the default for kernels installed in'
+                ' the future.'
+            ),
+            report_hint,
+            reporting.Severity(reporting.Severity.HIGH),
+            reporting.Groups([
+                reporting.Groups.BOOT,
+                reporting.Groups.KERNEL,
+                reporting.Groups.POST,
+            ]),
+            reporting.RelatedResource('file', '/etc/kernel/cmdline'),
+            reporting.RelatedResource('file', '/proc/cmdline'),
+        ])

repos/system_upgrade/common/actors/kernelcmdlineconfig/tests/test_kernelcmdlineconfig.py

@@ -1,7 +1,10 @@
+from __future__ import division
+
 from collections import namedtuple
 
 import pytest
 
+from leapp.exceptions import StopActorExecutionError
 from leapp.libraries import stdlib
 from leapp.libraries.actor import kernelcmdlineconfig
 from leapp.libraries.common.config import architecture
@@ -11,14 +14,44 @@
 
 TARGET_KERNEL_NEVRA = 'kernel-core-1.2.3-4.x86_64.el8.x64_64'
 
+# pylint: disable=E501
+SAMPLE_KERNEL_ARGS = ('ro rootflags=subvol=root'
+                      ' resume=/dev/mapper/luks-2c0df999-81ec-4a35-a1f9-b93afee8c6ad'
+                      ' rd.luks.uuid=luks-90a6412f-c588-46ca-9118-5aca35943d25'
+                      ' rd.luks.uuid=luks-2c0df999-81ec-4a35-a1f9-b93afee8c6ad rhgb quiet'
+                      )
+SAMPLE_KERNEL_ROOT = 'UUID=1aa15850-2685-418d-95a6-f7266a2de83a'
+TEMPLATE_GRUBBY_INFO_OUTPUT = """index=0
+kernel="/boot/vmlinuz-6.5.13-100.fc37.x86_64"
+args="{0}"
+root="{1}"
+initrd="/boot/initramfs-6.5.13-100.fc37.x86_64.img"
+title="Fedora Linux (6.5.13-100.fc37.x86_64) 37 (Thirty Seven)"
+id="a3018267cdd8451db7c77bb3e5b1403d-6.5.13-100.fc37.x86_64"
+"""  # noqa: E501
+SAMPLE_GRUBBY_INFO_OUTPUT = TEMPLATE_GRUBBY_INFO_OUTPUT.format(SAMPLE_KERNEL_ARGS, SAMPLE_KERNEL_ROOT)
+# pylint: enable=E501
+
 
 class MockedRun(object):
-    def __init__(self):
+    def __init__(self, outputs=None):
+        """
+        Mock stdlib.run().
+
+        If outputs is given, it is a dictionary mapping a cmd to output as stdout.
+        """
         self.commands = []
+        self.outputs = outputs or {}
 
     def __call__(self, cmd, *args, **kwargs):
         self.commands.append(cmd)
-        return {}
+        return {
+            "stdout": self.outputs.get(" ".join(cmd), ""),
+            "stderr": "",
+            "signal": None,
+            "exit_code": 0,
+            "pid": 1234,
+        }
 
 
 @pytest.mark.parametrize(
@@ -50,7 +83,7 @@ def __call__(self, cmd, *args, **kwargs):
         ),
     ]
 )
-def test_kernelcmdline_config_valid_msgs(monkeypatch, msgs, expected_grubby_kernelopt_args):
+def test_kernelcmdline_config_valid_msgs(monkeypatch, tmpdir, msgs, expected_grubby_kernelopt_args):
     kernel_img_path = '/boot/vmlinuz-X'
     kernel_info = InstalledTargetKernelInfo(pkg_nevra=TARGET_KERNEL_NEVRA,
                                             uname_r='',
@@ -61,18 +94,28 @@ def test_kernelcmdline_config_valid_msgs(monkeypatch, msgs, expected_grubby_kern
     grubby_base_cmd = ['grubby', '--update-kernel={}'.format(kernel_img_path)]
     expected_grubby_cmd = grubby_base_cmd + expected_grubby_kernelopt_args
 
-    mocked_run = MockedRun()
+    mocked_run = MockedRun(
+        outputs={" ".join(("grubby", "--info", kernel_img_path)): SAMPLE_GRUBBY_INFO_OUTPUT}
+    )
     monkeypatch.setattr(stdlib, 'run', mocked_run)
-    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(architecture.ARCH_X86_64, msgs=msgs))
+    monkeypatch.setattr(api, 'current_actor',
+                        CurrentActorMocked(architecture.ARCH_X86_64,
+                                           dst_ver="8.1",
+                                           msgs=msgs)
+                        )
     kernelcmdlineconfig.modify_kernel_args_in_boot_cfg()
-    assert mocked_run.commands and len(mocked_run.commands) == 1
-    assert expected_grubby_cmd == mocked_run.commands.pop()
+    assert mocked_run.commands and len(mocked_run.commands) == 3
+    assert expected_grubby_cmd == mocked_run.commands.pop(0)
 
-    mocked_run = MockedRun()
+    mocked_run = MockedRun(
+        outputs={" ".join(("grubby", "--info", kernel_img_path)): SAMPLE_GRUBBY_INFO_OUTPUT}
+    )
     monkeypatch.setattr(stdlib, 'run', mocked_run)
     monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(architecture.ARCH_S390X, msgs=msgs))
+    monkeypatch.setattr(kernelcmdlineconfig, 'KERNEL_CMDLINE_FILE', str(tmpdir / 'cmdline'))
+
     kernelcmdlineconfig.modify_kernel_args_in_boot_cfg()
-    assert mocked_run.commands and len(mocked_run.commands) == 2
+    assert mocked_run.commands and len(mocked_run.commands) == 3
     assert expected_grubby_cmd == mocked_run.commands.pop(0)
     assert ['/usr/sbin/zipl'] == mocked_run.commands.pop(0)
 
@@ -86,9 +129,17 @@ def test_kernelcmdline_explicit_configs(monkeypatch):
                                             initramfs_path='/boot/initramfs-X')
     msgs = [kernel_info, TargetKernelCmdlineArgTasks(to_remove=[KernelCmdlineArg(key='key1', value='value1')])]
 
-    mocked_run = MockedRun()
+    grubby_cmd_info = ["grubby", "--info", kernel_img_path]
+    mocked_run = MockedRun(
+        outputs={" ".join(grubby_cmd_info): SAMPLE_GRUBBY_INFO_OUTPUT}
+    )
     monkeypatch.setattr(stdlib, 'run', mocked_run)
-    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(architecture.ARCH_X86_64, msgs=msgs))
+    monkeypatch.setattr(api, 'current_actor',
+                        CurrentActorMocked(architecture.ARCH_X86_64,
+                                           dst_ver="8.1",
+                                           msgs=msgs
+                                           )
+                        )
 
     configs = ['/boot/grub2/grub.cfg', '/boot/efi/EFI/redhat/grub.cfg']
     kernelcmdlineconfig.modify_kernel_args_in_boot_cfg(configs_to_modify_explicitly=configs)
@@ -97,19 +148,27 @@ def test_kernelcmdline_explicit_configs(monkeypatch):
                                  '--remove-args', 'key1=value1']
     expected_cmds = [
         grubby_cmd_without_config + ['-c', '/boot/grub2/grub.cfg'],
-        grubby_cmd_without_config + ['-c', '/boot/efi/EFI/redhat/grub.cfg']
+        grubby_cmd_without_config + ['-c', '/boot/efi/EFI/redhat/grub.cfg'],
+        grubby_cmd_info,
+        ["grub2-editenv", "-", "set", "kernelopts=root={} {}".format(
+            SAMPLE_KERNEL_ROOT, SAMPLE_KERNEL_ARGS)],
     ]
 
     assert mocked_run.commands == expected_cmds
 
 
 def test_kernelcmdline_config_no_args(monkeypatch):
+    kernel_img_path = '/boot/vmlinuz-X'
     kernel_info = InstalledTargetKernelInfo(pkg_nevra=TARGET_KERNEL_NEVRA,
                                             uname_r='',
-                                            kernel_img_path='/boot/vmlinuz-X',
+                                            kernel_img_path=kernel_img_path,
                                             initramfs_path='/boot/initramfs-X')
 
-    mocked_run = MockedRun()
+    mocked_run = MockedRun(
+        outputs={" ".join(("grubby", "--info", kernel_img_path)):
+                 TEMPLATE_GRUBBY_INFO_OUTPUT.format("", "")
+                 }
+    )
     monkeypatch.setattr(stdlib, 'run', mocked_run)
     monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(architecture.ARCH_S390X, msgs=[kernel_info]))
     kernelcmdlineconfig.modify_kernel_args_in_boot_cfg()
@@ -122,3 +181,32 @@ def test_kernelcmdline_config_no_version(monkeypatch):
     monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(architecture.ARCH_S390X))
     kernelcmdlineconfig.modify_kernel_args_in_boot_cfg()
     assert not mocked_run.commands
+
+
+def test_kernelcmdline_config_malformed_args(monkeypatch):
+    kernel_img_path = '/boot/vmlinuz-X'
+    kernel_info = InstalledTargetKernelInfo(pkg_nevra=TARGET_KERNEL_NEVRA,
+                                            uname_r='',
+                                            kernel_img_path=kernel_img_path,
+                                            initramfs_path='/boot/initramfs-X')
+
+    # For this test, we need to check we get the proper error if grubby --info
+    # doesn't output any args information at all.
+    grubby_info_output = "\n".join(line for line in SAMPLE_GRUBBY_INFO_OUTPUT.splitlines()
+                                   if not line.startswith("args="))
+    mocked_run = MockedRun(
+        outputs={" ".join(("grubby", "--info", kernel_img_path)): grubby_info_output,
+                 }
+    )
+    msgs = [kernel_info,
+            TargetKernelCmdlineArgTasks(to_remove=[
+                KernelCmdlineArg(key='key1', value='value1')])
+            ]
+    monkeypatch.setattr(stdlib, 'run', mocked_run)
+    monkeypatch.setattr(api, 'current_actor',
+                        CurrentActorMocked(architecture.ARCH_S390X, msgs=msgs))
+
+    with pytest.raises(kernelcmdlineconfig.ReadOfKernelArgsError,
+                       match="Failed to retrieve kernel command line to save for future"
+                       " installed kernels."):
+        kernelcmdlineconfig.modify_kernel_args_in_boot_cfg()