From 1c95581dbf927e55c05142989f96f543765c4f96 Mon Sep 17 00:00:00 2001 From: William Douglas Date: Sun, 17 Nov 2024 13:11:48 -0800 Subject: [PATCH] Add --single-step support for update Allow users to update by stepping through each release between their current version and the latest. This option is primarily for cases where update is failing due to memory or disk space running out when updating normally. Signed-off-by: William Douglas --- docs/swupd.1.rst | 2 + src/cmds/update.c | 37 ++- src/swupd.h | 2 +- src/swupd_lib/version.c | 141 +++++++++- swupd.zsh | 1 + .../update-no-disk-space.bats | 14 +- .../signature/version-sig-check.bats | 2 + .../functional/update/update-single-step.bats | 262 ++++++++++++++++++ 8 files changed, 430 insertions(+), 31 deletions(-) create mode 100755 test/functional/update/update-single-step.bats diff --git a/docs/swupd.1.rst b/docs/swupd.1.rst index a697a6d2e..1e7e3f55a 100644 --- a/docs/swupd.1.rst +++ b/docs/swupd.1.rst @@ -201,6 +201,8 @@ update --download Do not perform an update, instead download all resources needed to perform the update, and exit. +--single-step Update by to latest by first going to each release prior + --update-search-file-index Update the index used by search-file to speed up searches. Don't enable this if you have download or space restrictions. diff --git a/src/cmds/update.c b/src/cmds/update.c index 68a5e0573..9b119d929 100644 --- a/src/cmds/update.c +++ b/src/cmds/update.c @@ -39,11 +39,13 @@ #define FLAG_DOWNLOAD_ONLY 2000 #define FLAG_UPDATE_SEARCH_FILE_INDEX 2001 #define FLAG_UPDATE_3RD_PARTY 2002 +#define FLAG_SINGLE_STEP 2003 static int requested_version = -1; static bool download_only = false; static bool update_search_file_index = false; static bool keepcache = false; +static bool cmdline_option_single_step = false; static char swupd_binary[LINE_MAX] = { 0 }; #ifdef THIRDPARTY @@ -142,11 +144,11 @@ int add_included_manifests(struct manifest *mom, struct list **subs) return 0; } -static enum swupd_code check_versions(int *current_version, int *server_version, int req_version, char *path_prefix) +static enum swupd_code check_versions(int *current_version, int *server_version, int req_version, char *path_prefix, bool single_step) { int ret; - ret = read_versions(current_version, server_version, path_prefix); + ret = read_versions(current_version, server_version, path_prefix, single_step); if (ret != SWUPD_OK) { return ret; } @@ -156,8 +158,12 @@ static enum swupd_code check_versions(int *current_version, int *server_version, } if (req_version != -1) { if (req_version < *current_version) { - error("Requested version for update (%d) must be greater than current version (%d)\n", - req_version, *current_version); + // This error message isn't valid if the + // system has gone through a format bump + if (!on_new_format()) { + error("Requested version for update (%d) must be greater than current version (%d)\n", + req_version, *current_version); + } return SWUPD_INVALID_OPTION; } if (req_version < *server_version) { @@ -285,7 +291,7 @@ enum swupd_code execute_update_extra(extra_proc_fn_t post_update_fn, extra_proc_ /* get versions */ timelist_timer_start(globals.global_times, "Get versions"); - ret = check_versions(¤t_version, &server_version, requested_version, globals.path_prefix); + ret = check_versions(¤t_version, &server_version, requested_version, globals.path_prefix, cmdline_option_single_step); if (ret != SWUPD_OK) { goto clean_curl; } @@ -448,7 +454,17 @@ enum swupd_code execute_update_extra(extra_proc_fn_t post_update_fn, extra_proc_ progress_next_step("run_postupdate_scripts", PROGRESS_UNDEFINED); /* Determine if another update is needed so the scripts block */ - int new_current_version = get_current_version(globals.path_prefix); + int new_current_version, new_server_version; + int versions_check = check_versions(&new_current_version, &new_server_version, requested_version, globals.path_prefix, false); + if (versions_check != SWUPD_OK) { + // update was fine, but we aren't seeing the new server version + // for some reason so don't set re_update on single_step + new_server_version = new_current_version; + } + + if (cmdline_option_single_step && new_server_version > new_current_version) { + re_update = true; + } if (on_new_format() && (requested_version == -1 || (requested_version > new_current_version))) { re_update = true; } @@ -560,6 +576,7 @@ static const struct option prog_opts[] = { #ifdef THIRDPARTY { "3rd-party", no_argument, 0, FLAG_UPDATE_3RD_PARTY }, #endif + { "single-step", no_argument, 0, FLAG_SINGLE_STEP }, }; static void print_help(void) @@ -579,6 +596,7 @@ static void print_help(void) #ifdef THIRDPARTY print(" --3rd-party Also update content from 3rd-party repositories\n"); #endif + print(" --single-step Update to latest by first going to each release prior\n"); print("\n"); } @@ -617,6 +635,9 @@ static bool parse_opt(int opt, char *optarg) cmdline_option_3rd_party = optarg_to_bool(optarg); return true; #endif + case FLAG_SINGLE_STEP: + cmdline_option_single_step = optarg_to_bool(optarg); + return true; default: return false; } @@ -697,7 +718,7 @@ enum swupd_code update_main(int argc, char **argv) ret = check_update(); #ifdef THIRDPARTY - if (cmdline_option_3rd_party) { + if (cmdline_option_3rd_party && !cmdline_option_single_step) { progress_finish_steps(ret); info("\nChecking update status of content from 3rd-party repositories\n\n"); ret = third_party_execute_check_update(); @@ -707,7 +728,7 @@ enum swupd_code update_main(int argc, char **argv) ret = execute_update(); #ifdef THIRDPARTY - if (cmdline_option_3rd_party) { + if (cmdline_option_3rd_party && !cmdline_option_single_step) { if (ret == SWUPD_OK) { progress_finish_steps(ret); info("\nUpdating content from 3rd-party repositories\n\n"); diff --git a/src/swupd.h b/src/swupd.h index ac036f19a..9fc0eaa28 100644 --- a/src/swupd.h +++ b/src/swupd.h @@ -167,7 +167,7 @@ extern enum swupd_code walk_tree(struct manifest *, const char *, bool, const re extern int get_latest_version(char *v_url); extern int get_int_from_url(const char *url); -extern enum swupd_code read_versions(int *current_version, int *server_version, char *path_prefix); +extern enum swupd_code read_versions(int *current_version, int *server_version, char *path_prefix, bool single_step); extern int get_current_version(char *path_prefix); extern bool get_distribution_string(char *path_prefix, char *dist); extern int get_current_format(void); diff --git a/src/swupd_lib/version.c b/src/swupd_lib/version.c index 875528966..900b036f5 100644 --- a/src/swupd_lib/version.c +++ b/src/swupd_lib/version.c @@ -35,7 +35,7 @@ int get_int_from_url(const char *url) { int err, value; - char tmp_string[MAX_VERSION_STR_SIZE+1]; + char tmp_string[MAX_VERSION_STR_SIZE + 1]; struct curl_file_data tmp_data = { MAX_VERSION_STR_SIZE, 0, tmp_string @@ -147,11 +147,11 @@ static int verify_signature(char *url, struct curl_file_data *tmp_version) return ret; } -static int get_version_from_url(char *url) +static int get_version_from_url_inmemory(char *url) { int ret = 0; int err = 0; - char version_str[MAX_VERSION_STR_SIZE+1]; + char version_str[MAX_VERSION_STR_SIZE + 1]; int sig_verified = 0; /* struct for version data */ @@ -188,49 +188,154 @@ static int get_version_from_url(char *url) } return -SWUPD_ERROR_SIGNATURE_VERIFICATION; } + return ret; +} + +static int get_version_from_file(char *filename, int current_version) +{ + FILE *infile; + char *line = NULL; + // start as -EINVAL to report cases of an empty file + int next_version = -EINVAL; + int tmp_version = 0; + int err = 0; + size_t len = 0; + ssize_t read; + + infile = fopen(filename, "r"); + if (infile == NULL) { + return -EINVAL; + } + + while ((read = getline(&line, &len, infile)) != -1) { + err = str_to_int(line, &tmp_version); + if (err) { + break; + } + if (current_version < 0) { + next_version = tmp_version; + break; + } else if (tmp_version <= current_version) { + // We did get a valid version so report it + // but it isn't usable for update + if (next_version < 0) { + next_version = tmp_version; + } + break; + } + next_version = tmp_version; + } + + (void)fclose(infile); + + if (line) { + FREE(line); + } + + if (err) { + return err; + } + return next_version; +} + +static int get_version_from_url(char *url, char *filename, int current_version) +{ + int ret = 0; + char *sig_filename; + char *sig_url; + char *download_file; + char *download_sig; + + string_or_die(&sig_filename, "%s.sig", filename); + string_or_die(&sig_url, "%s.sig", url); + download_file = statedir_get_download_file(filename); + download_sig = statedir_get_download_file(sig_filename); + ret = swupd_curl_get_file(url, download_file); + if (ret) { + goto out; + } + + /* Sig check */ + if (globals.sigcheck && globals.sigcheck_latest) { + ret = swupd_curl_get_file(sig_url, download_sig); + if (ret) { + ret = -SWUPD_ERROR_SIGNATURE_VERIFICATION; + error("Signature for latest file (%s) is missing\n", url); + goto out; + } + if (!signature_verify(download_file, download_sig, SIGNATURE_IGNORE_EXPIRATION)) { + ret = -SWUPD_ERROR_SIGNATURE_VERIFICATION; + error("Signature verification failed for URL: %s\n", url); + goto out; + } + } else { + warn_nosigcheck(url); + } + + ret = get_version_from_file(download_file, current_version); + +out: + FREE(sig_filename); + FREE(sig_url); + FREE(download_file); + FREE(download_sig); return ret; } /* this function attempts to download the latest server version string file from - * the preferred server to a memory buffer, returning either a negative integer - * error code or >= 0 representing the server version + * the preferred server, returning either a negative integer error code or >= 0 + * representing the server version * * if v_url is non-NULL the version at v_url is fetched. If v_url is NULL the * global version_url is used and the cached version may be used instead of - * attempting to download the version string again. If v_url is the empty string - * the global version_url is used and the cached version is ignored. */ -int get_latest_version(char *v_url) + * attempting to download the version string again (unless current_version is + * negative, indicating single-step and the version needs to be recalculated). + * If v_url is the empty string the global version_url is used and the cached + * version is ignored. + * + * if current_version >= 0 it is used to indicate the next version rather than + * the latest version should be returned. */ +static int get_version(char *v_url, int current_version) { + char *filename = "latest"; char *url = NULL; int ret = 0; static int cached_version = -1; - if (cached_version > 0 && v_url == NULL) { + if (cached_version > 0 && v_url == NULL && current_version < 0) { return cached_version; } + // TODO: why allow "" when we do NULL test above? + // maybe consolidate. if (v_url == NULL || str_cmp(v_url, "") == 0) { v_url = globals.version_url; } - string_or_die(&url, "%s/version/format%s/latest", v_url, globals.format_string); - ret = get_version_from_url(url); + string_or_die(&url, "%s/version/format%s/%s", v_url, globals.format_string, filename); + ret = get_version_from_url(url, filename, current_version); FREE(url); cached_version = ret; return ret; } +int get_latest_version(char *v_url) +{ + return get_version(v_url, -1); +} + /* gets the latest version of the update content regardless of what format we * are currently in */ int version_get_absolute_latest(void) { + char *filename = "latest_version"; char *url = NULL; int ret; - string_or_die(&url, "%s/version/latest_version", globals.version_url); - ret = get_version_from_url(url); + string_or_die(&url, "%s/version/%s", globals.version_url, filename); + ret = get_version_from_url_inmemory(url); FREE(url); return ret; @@ -329,15 +434,21 @@ bool get_distribution_string(char *path_prefix, char *dist) return true; } -enum swupd_code read_versions(int *current_version, int *server_version, char *path_prefix) +enum swupd_code read_versions(int *current_version, int *server_version, char *path_prefix, bool single_step) { *current_version = get_current_version(path_prefix); - *server_version = get_latest_version(""); if (*current_version < 0) { error("Unable to determine current OS version\n"); return SWUPD_CURRENT_VERSION_UNKNOWN; } + + if (single_step) { + *server_version = get_version("", *current_version); + } else { + *server_version = get_version("", -1); + } + if (*server_version == -SWUPD_ERROR_SIGNATURE_VERIFICATION) { error("Unable to determine the server version as signature verification failed\n"); return SWUPD_SIGNATURE_VERIFICATION_FAILED; diff --git a/swupd.zsh b/swupd.zsh index 5ced94213..a04778e1f 100644 --- a/swupd.zsh +++ b/swupd.zsh @@ -199,6 +199,7 @@ local -a update=( '(-)'{-s,--status}'[Show current OS version and latest version available on server. Equivalent to "swupd check-update"]' '(help -s --status -k --keepcache)'{-k,--keepcache}'[Do not delete the swupd state directory content after updating the system]' '(help -s --status)--download[Download all content, but do not actually install the update]' + '(help -s --status)--single-step[Update to latest by first going to each release prior]' ) local update_official=( '(help -s --status)--update-search-file-index[Update the index used by search-file to speed up searches]' diff --git a/test/functional/only_in_ci_system/update-no-disk-space.bats b/test/functional/only_in_ci_system/update-no-disk-space.bats index 2cdfe7b75..8d7e4c9eb 100755 --- a/test/functional/only_in_ci_system/update-no-disk-space.bats +++ b/test/functional/only_in_ci_system/update-no-disk-space.bats @@ -30,23 +30,23 @@ test_setup() { } -@test "UPD045: Updating a system with no disk space left (downloading the current MoM)" { +@test "UPD045: Updating a system with no disk space left (fails to get the server version)" { # When updating a system and we run out of disk space while downloading the - # MoMs we should not retry the download since it will fail for sure + # server version we should not retry the download since it will fail for sure # fill up all the space in the disk sudo dd if=/dev/zero of="$TEST_NAME"/testfs/dummy >& /dev/null || print "Using all space left in disk" run sudo sh -c "timeout 30 $SWUPD update $SWUPD_OPTS" - assert_status_is "$SWUPD_COULDNT_LOAD_MOM" + assert_status_is "$SWUPD_SERVER_CONNECTION_ERROR" expected_output=$(cat <<-EOM Update started - Preparing to update from 10 to 20 - Error: Curl - Error downloading to local file - 'file://$ABS_TEST_DIR/web-dir/10/Manifest.MoM.tar' - Error: Curl - Check free space for $ABS_TEST_DIR/testfs/state? - Error: Failed to retrieve 10 MoM manifest + Error: Curl - Cannot close file '$ABS_CACHE_DIR/download/latest' after writing - 'No space left on device' + Error: Curl - Error downloading to local file - 'file://$ABS_TEST_DIR/web-dir/version/formatstaging/latest' + Error: Curl - Check free space for $ABS_STATE_DIR? + Error: Unable to determine the server version Update failed EOM ) diff --git a/test/functional/signature/version-sig-check.bats b/test/functional/signature/version-sig-check.bats index b25dfa604..586ebfde2 100755 --- a/test/functional/signature/version-sig-check.bats +++ b/test/functional/signature/version-sig-check.bats @@ -92,6 +92,7 @@ test_setup() { No extra files need to be downloaded Installing files... Update was applied + Warning: THE SIGNATURE OF file://$ABS_TEST_DIR/web-dir/version/formatstaging/latest WILL NOT BE VERIFIED Calling post-update helper scripts Update successful - System updated from version 10 to version 20 EOM @@ -127,6 +128,7 @@ test_setup() { No extra files need to be downloaded Installing files... Update was applied + Warning: THE SIGNATURE OF file://$ABS_TEST_DIR/web-dir/version/formatstaging/latest WILL NOT BE VERIFIED Calling post-update helper scripts Update successful - System updated from version 10 to version 20 EOM diff --git a/test/functional/update/update-single-step.bats b/test/functional/update/update-single-step.bats new file mode 100755 index 000000000..afd40a31d --- /dev/null +++ b/test/functional/update/update-single-step.bats @@ -0,0 +1,262 @@ +#!/usr/bin/env bats + +# Author: William Douglas +# Email: william.douglas@intel.com + +load "../testlib" + +test_setup() { + + create_test_environment -r "$TEST_NAME" 10 1 + create_version -r "$TEST_NAME" 20 10 1 + create_version -r "$TEST_NAME" 30 20 1 + +} + +test_teardown() { + + destroy_test_environment "$TEST_NAME" + +} + +@test "UPD077: Run update with --single-step old latest file format" { + + # Run update with --single-step where the latest file format only + # contains a single line of 30 + + run sudo sh -c "$SWUPD update $SWUPD_OPTS_NO_FMT --single-step" + + assert_status_is "$SWUPD_OK" + expected_output=$(cat <<-EOM + Update started + Preparing to update from 10 to 30 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 10 to version 30: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 10 to version 30 + EOM + ) + assert_is_output "$expected_output" + +} + +@test "UPD078: Run update with --single-step multiline latest file format" { + + # Run update with --single-step where the latest file format contains + # "30\n20\n10" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n20" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n10" + sign_version "$WEB_DIR/version/format1/latest" + + run sudo sh -c "$SWUPD update $SWUPD_OPTS_NO_FMT --single-step" + + assert_status_is "$SWUPD_OK" + expected_output=$(cat <<-EOM + Update started + Preparing to update from 10 to 20 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 10 to version 20: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 10 to version 20 + Update started + Preparing to update from 20 to 30 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 20 to version 30: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 20 to version 30 + EOM + ) + assert_is_output "$expected_output" + +} + +@test "UPD079: Run update with --single-step multiline latest missing current version" { + + # Run update with --single-step where the latest file format contains + # "30\n20" + # (the starting version is missing) + # Test may be reasonable to delete but for now verify process robustness + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n20" + sign_version "$WEB_DIR/version/format1/latest" + + run sudo sh -c "$SWUPD update $SWUPD_OPTS_NO_FMT --single-step" + + assert_status_is "$SWUPD_OK" + expected_output=$(cat <<-EOM + Update started + Preparing to update from 10 to 20 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 10 to version 20: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 10 to version 20 + Update started + Preparing to update from 20 to 30 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 20 to version 30: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 20 to version 30 + EOM + ) + assert_is_output "$expected_output" + +} + +@test "UPD080: Run update with --single-step multiline latest with target version" { + + # Run update with --single-step where the latest file format contains + # "40\n30\n20\n10" + # And a target version of 30 + create_version -r "$TEST_NAME" 40 30 1 + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n30" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n20" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n10" + sign_version "$WEB_DIR/version/format1/latest" + + run sudo sh -c "$SWUPD update $SWUPD_OPTS_NO_FMT --single-step -V 30" + + assert_status_is "$SWUPD_OK" + expected_output=$(cat <<-EOM + Update started + Preparing to update from 10 to 20 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 10 to version 20: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 10 to version 20 + Update started + Preparing to update from 20 to 30 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 20 to version 30: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 20 to version 30 + EOM + ) + assert_is_output "$expected_output" + +} + +@test "UPD081: Run update with normally but with a multiline target version" { + + # Run update without --single-step where the latest file format contains + # "50\n40\n30\n20\n10" + # 50 is chosen because at the time the test was written handling + # of a latest file with "50\n40\n30\n20\n10" was failing case due to + # file size but it should not fail going forward. + create_version -r "$TEST_NAME" 40 30 1 + create_version -r "$TEST_NAME" 50 40 1 + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n40" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n30" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n20" + write_to_protected_file -a "$WEB_DIR/version/format1/latest" "\n10" + sign_version "$WEB_DIR/version/format1/latest" + + run sudo sh -c "$SWUPD update $SWUPD_OPTS_NO_FMT" + + assert_status_is "$SWUPD_OK" + expected_output=$(cat <<-EOM + Update started + Preparing to update from 10 to 50 + Downloading packs for: + - os-core + Finishing packs extraction... + Statistics for going from version 10 to version 50: + changed bundles : 1 + new bundles : 0 + deleted bundles : 0 + changed files : 2 + new files : 0 + deleted files : 0 + Validate downloaded files + No extra files need to be downloaded + Installing files... + Update was applied + Calling post-update helper scripts + Update successful - System updated from version 10 to version 50 + EOM + ) + assert_is_output "$expected_output" + +}