diff --git a/release-process/distro-configs/ubuntu18-lts.json b/release-process/distro-configs/ubuntu18-lts.json index dee11a17352..20126ae9077 100644 --- a/release-process/distro-configs/ubuntu18-lts.json +++ b/release-process/distro-configs/ubuntu18-lts.json @@ -4,10 +4,11 @@ "ami_name_pattern": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-*", "user": "ubuntu", "setup_commands": [ + "cloud-init status --wait", "sudo apt-get update -y", "sudo apt-get dist-upgrade -f -y" ], "install_build_deps": "sudo apt-get install -y rpm ccache cmake g++-multilib pkg-config zlib1g-dev git python-dev libacl1-dev ninja-build manpages-dev capnproto libcapnp-dev gdb python3-pexpect", - "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl libdbus-glib-1-2 libreoffice", + "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl tar bzip2 libdbus-glib-1-2 libreoffice", "exclude_tests": ["x86/pkeys.*"] } diff --git a/release-process/distro-configs/ubuntu20-lts.json b/release-process/distro-configs/ubuntu20-lts.json index ef7fd27033f..a22b04ad043 100644 --- a/release-process/distro-configs/ubuntu20-lts.json +++ b/release-process/distro-configs/ubuntu20-lts.json @@ -4,10 +4,11 @@ "ami_name_pattern": "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-*", "user": "ubuntu", "setup_commands": [ + "cloud-init status --wait", "sudo apt-get update -y", "sudo apt-get dist-upgrade -f -y" ], "install_build_deps": "sudo apt-get install -y rpm ccache cmake g++-multilib pkg-config zlib1g-dev git python-dev libacl1-dev ninja-build manpages-dev capnproto libcapnp-dev gdb python3-pexpect", - "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl libdbus-glib-1-2 libreoffice", + "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl tar bzip2 libdbus-glib-1-2 libreoffice", "exclude_tests": ["x86/pkeys.*"] } diff --git a/release-process/distro-configs/ubuntu22-lts.json b/release-process/distro-configs/ubuntu22-lts.json index 26f05673be9..5a8136e25ef 100644 --- a/release-process/distro-configs/ubuntu22-lts.json +++ b/release-process/distro-configs/ubuntu22-lts.json @@ -4,9 +4,10 @@ "ami_name_pattern": "ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-*", "user": "ubuntu", "setup_commands": [ + "cloud-init status --wait", "sudo apt-get update -y", "sudo apt-get dist-upgrade -f -y" ], "install_build_deps": "sudo apt-get install -y rpm ccache cmake g++-multilib pkg-config zlib1g-dev git python-dev-is-python3 libacl1-dev ninja-build manpages-dev capnproto libcapnp-dev gdb python3-pexpect", - "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl libdbus-glib-1-2 libreoffice" + "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl tar bzip2 libdbus-glib-1-2 libreoffice" } diff --git a/release-process/distro-configs/ubuntu23.04.json b/release-process/distro-configs/ubuntu23.04.json index 68a0386ccdc..9c778df9d1f 100644 --- a/release-process/distro-configs/ubuntu23.04.json +++ b/release-process/distro-configs/ubuntu23.04.json @@ -4,9 +4,10 @@ "ami_name_pattern": "ubuntu/images/hvm-ssd/ubuntu-lunar-23.04-*", "user": "ubuntu", "setup_commands": [ + "cloud-init status --wait", "sudo apt-get update -y", "sudo apt-get dist-upgrade -f -y" ], "install_build_deps": "sudo apt-get install -y rpm ccache cmake g++-multilib pkg-config zlib1g-dev git python-dev-is-python3 libacl1-dev ninja-build manpages-dev capnproto libcapnp-dev gdb python3-pexpect", - "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl libdbus-glib-1-2 libreoffice" + "install_test_deps": "sudo apt-get install -y tightvncserver xtightvncviewer curl tar bzip2 libdbus-glib-1-2 libreoffice" } diff --git a/release-process/rr-testing.sh b/release-process/rr-testing.sh index c5280709880..7e12df7ade6 100644 --- a/release-process/rr-testing.sh +++ b/release-process/rr-testing.sh @@ -1,11 +1,24 @@ # Bash script to build rr and run tests. # -# Requires variables to be set: +# Requires variables and functions to be set. See test-system.py. # $git_revision : git revision to check out, build and test # $build_dist : 1 if we should build dist packages, 0 otherwise +# $test_firefox : 1 to run firefox tests, 0 to skip +# $ctest_options : options to pass to ctest, e.g to exclude certain tests +# setup_commands : function to setup environment, e.g. 'apt update' +# install_build_deps : function to install dependencies required to build rr +# install_test_deps : function to install dependencies required by tests + +set -x # echo commands +set -e # default to exiting on error" uname -a +setup_commands +install_build_deps + +install_test_deps & # job %1 + # Free up space before we (re)start rm -rf ~/rr || true @@ -21,7 +34,7 @@ ninja # Test deps are installed in parallel with our build. # Make sure that install has finished before running tests -wait_for_test_deps +wait %1 # Enable perf events for rr echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid @@ -69,7 +82,7 @@ function xvnc-runner { CMD=$1 EXPECT=$2 echo PASSED: $CMD } -if [[ $TEST_FIREFOX == 1 ]]; then +if [[ $test_firefox == 1 ]]; then rm -rf /tmp/firefox /tmp/firefox-profile || true mkdir /tmp/firefox-profile ( cd /tmp; curl -L 'https://download.mozilla.org/?product=firefox-latest&os=linux64&lang=en-US' | tar -jxf - ) diff --git a/release-process/test-system.py b/release-process/test-system.py index d8e51425053..a70818984da 100755 --- a/release-process/test-system.py +++ b/release-process/test-system.py @@ -126,25 +126,32 @@ def terminate(self): with pathlib.Path(__file__).with_name('rr-testing.sh').open('rb') as f: rr_testing_script = f.read() +def config_script_function(config_key): + lines = [] + entry = distro_config.get(config_key) + if isinstance(entry, str): + lines = [entry] + elif isinstance(entry, list): + lines = entry + elif entry is not None: + raise ValueError('Invalid config entry %s: %s' % (config_key, entry)) + return ('function %s {\n%s\n}' % (config_key, '\n'.join(lines))) + vm = Ec2Vm(args.machine_type, args.architecture, distro_config, args.keypair_pem_file) success = False try: vm.wait_for_ssh() exclude_tests = distro_config['exclude_tests'] if 'exclude_tests' in distro_config else [] - full_script = b'\n'.join( - [b"set -x # echo commands", - b"set -e # default to exiting on error"] + - list(map(lambda c: c.encode('utf-8'), distro_config['setup_commands'])) + + full_script = '\n'.join( [ - distro_config['install_build_deps'].encode('utf-8'), - b'%s &'%(distro_config['install_test_deps'].encode('utf-8')), - b'function wait_for_test_deps {\nwait %1\n}', - b'git_revision=%s'%args.git_revision.encode('utf-8'), - b'build_dist=%d'%(1 if args.dist_files_dir is not None else 0), - b'TEST_FIREFOX=%d'%(1 if args.architecture == 'x86_64' else 0), - b'ctest_options="%s"'%b' '.join(map(lambda r: b'-E %s'%r.encode('utf-8'), exclude_tests)), - rr_testing_script - ]) + config_script_function('setup_commands'), + config_script_function('install_build_deps'), + config_script_function('install_test_deps'), + 'git_revision=%s'%args.git_revision, + 'build_dist=%d'%(1 if args.dist_files_dir is not None else 0), + 'test_firefox=%d'%(1 if args.architecture == 'x86_64' else 0), + 'ctest_options="%s"'%' '.join('-E %s'%r for r in exclude_tests), + ]).encode('utf-8') + b'\n' + rr_testing_script vm.ssh(['/bin/bash', '-s'], full_script) if args.dist_files_dir is not None: vm.scp_from(['-r'], '/tmp/dist', args.dist_files_dir)