From 87e564eeaa569adde786c4c39a2626ca3d687436 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 12:00:32 +0200 Subject: [PATCH 01/14] deltas: Add _ostree_get_relative_static_delta_index_path() This gets the subpath for a delta index file, which is of the form "delta-indexes/$commit.index", that contains all the deltas going to the particular commit. --- src/libostree/ostree-core-private.h | 3 +++ src/libostree/ostree-core.c | 30 ++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 5d9d948ad7..89d95ad9c6 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char *from, const char *to, guint i); +char * +_ostree_get_relative_static_delta_index_path (const char *to); + static inline char * _ostree_get_commitpartial_path (const char *checksum) { return g_strconcat ("state/", checksum, ".commitpartial", NULL); diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 29528fa51f..f822101a36 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char *checksum, return g_string_free (path, FALSE); } -char * -_ostree_get_relative_static_delta_path (const char *from, - const char *to, - const char *target) +static GString * +static_delta_path_base (const char *dir, + const char *from, + const char *to) { guint8 csum_to[OSTREE_SHA256_DIGEST_LEN]; char to_b64[44]; guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN]; - GString *ret = g_string_new ("deltas/"); + GString *ret = g_string_new (dir); ostree_checksum_inplace_to_bytes (to, csum_to); ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); @@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from, g_string_append_c (ret, '/'); g_string_append (ret, to_b64 + 2); + return ret; +} + +char * +_ostree_get_relative_static_delta_path (const char *from, + const char *to, + const char *target) +{ + GString *ret = static_delta_path_base ("deltas/", from, to); + if (target != NULL) { g_string_append_c (ret, '/'); @@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char *from, return _ostree_get_relative_static_delta_path (from, to, partstr); } +char * +_ostree_get_relative_static_delta_index_path (const char *to) +{ + GString *ret = static_delta_path_base ("delta-indexes/", NULL, to); + + g_string_append (ret, ".index"); + + return g_string_free (ret, FALSE); +} + gboolean _ostree_parse_delta_name (const char *delta_name, char **out_from, From 8e1f199dd4d035b5ff501aa37c6f1229ac3c0e61 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 12:03:40 +0200 Subject: [PATCH 02/14] deltas: Add ostree_repo_list_static_delta_indexes() function This lists all the available delta indexes. --- Makefile-libostree.am | 6 +- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 7 +- src/libostree/ostree-repo-static-delta-core.c | 87 +++++++++++++++++++ src/libostree/ostree-repo.h | 6 ++ 5 files changed, 101 insertions(+), 6 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index bdc47122f4..eeb0b6c60f 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -184,9 +184,9 @@ libostree_1_la_SOURCES += \ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym -#if BUILDOPT_IS_DEVEL_BUILD -#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym -#endif +if BUILDOPT_IS_DEVEL_BUILD +symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= EXTRA_DIST += \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index c1d6b35ef9..ea37823bf3 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -412,6 +412,7 @@ OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE ostree_repo_list_objects ostree_repo_list_commit_objects_starting_with ostree_repo_list_static_delta_names +ostree_repo_list_static_delta_indexes OstreeStaticDeltaGenerateOpt ostree_repo_static_delta_generate ostree_repo_static_delta_execute_offline_with_signature diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 8e8d9c8191..1350341cef 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -17,9 +17,10 @@ Boston, MA 02111-1307, USA. ***/ -/* Copy the bits below and uncomment the include in Makefile-libostree.am - when adding a symbol. -*/ +LIBOSTREE_2020.8 { +global: + ostree_repo_list_static_delta_indexes; +} LIBOSTREE_2020.7; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index e263e928b0..e3432aa5ac 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -168,6 +168,93 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, return TRUE; } +/** + * ostree_repo_list_static_delta_indexes: + * @self: Repo + * @out_indexes: (out) (element-type utf8) (transfer container): String name of delta indexes (checksum) + * @cancellable: Cancellable + * @error: Error + * + * This function synchronously enumerates all static delta indexes in the + * repository, returning its result in @out_indexes. + * + * Since: 2020.7 + */ +gboolean +ostree_repo_list_static_delta_indexes (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) ret_indexes = g_ptr_array_new_with_free_func (g_free); + + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + gboolean exists; + if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "delta-indexes", &dfd_iter, + &exists, error)) + return FALSE; + if (!exists) + { + /* Note early return */ + ot_transfer_out_value (out_indexes, &ret_indexes); + return TRUE; + } + + while (TRUE) + { + g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, }; + struct dirent *dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) + break; + if (dent->d_type != DT_DIR) + continue; + if (strlen (dent->d_name) != 2) + continue; + + if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE, + &sub_dfd_iter, error)) + return FALSE; + + while (TRUE) + { + struct dirent *sub_dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent, + cancellable, error)) + return FALSE; + if (sub_dent == NULL) + break; + if (sub_dent->d_type != DT_REG) + continue; + + const char *name1 = dent->d_name; + const char *name2 = sub_dent->d_name; + + /* base64 len is 43, but 2 chars are in the parent dir name */ + if (strlen (name2) != 41 + strlen(".index") || + !g_str_has_suffix (name2, ".index")) + continue; + + g_autoptr(GString) out = g_string_new (name1); + g_string_append_len (out, name2, 41); + + char checksum[OSTREE_SHA256_STRING_LEN+1]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; + + ostree_checksum_b64_inplace_to_bytes (out->str, csum); + ostree_checksum_inplace_from_bytes (csum, checksum); + + g_ptr_array_add (ret_indexes, g_strdup (checksum)); + } + } + + ot_transfer_out_value (out_indexes, &ret_indexes); + return TRUE; +} + gboolean _ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo, GVariant *checksum_array, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d52fc9c3a9..a3c5dc7799 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1046,6 +1046,12 @@ gboolean ostree_repo_list_static_delta_names (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_list_static_delta_indexes (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error); + /** * OstreeStaticDeltaGenerateOpt: * @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space From effde3d513de1570488ddf13dac43c6413a44c68 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 12:05:36 +0200 Subject: [PATCH 03/14] deltas: Update delta indexes when updating summary When we update the summary file (and its list of deltas) we also update all delta indexes. The index format is a single `a{sv}` variant identical to the metadata-part of the summary with (currently) only the `ostree.static-deltas` key. Since we expect most delta indexes to change rarely, we avoid unnecessary writes when reindexing. New indexes are compared to existing ones and only the changed ones are written to disk. This avoids unnecessary write load and mtime changes on the repo server. --- src/libostree/ostree-repo-static-delta-core.c | 165 ++++++++++++++++++ .../ostree-repo-static-delta-private.h | 5 + src/libostree/ostree-repo.c | 3 + 3 files changed, 173 insertions(+) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index e3432aa5ac..c58e368387 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -1205,3 +1205,168 @@ ostree_repo_static_delta_verify_signature (OstreeRepo *self, return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error); } + +static void +null_or_ptr_array_unref (GPtrArray *array) +{ + if (array != NULL) + g_ptr_array_unref (array); +} + +static gboolean +file_has_content (OstreeRepo *repo, + const char *subpath, + GBytes *data, + GCancellable *cancellable) +{ + struct stat stbuf; + glnx_autofd int existing_fd = -1; + + if (!glnx_fstatat (repo->repo_dir_fd, subpath, &stbuf, 0, NULL)) + return FALSE; + + if (stbuf.st_size != g_bytes_get_size (data)) + return FALSE; + + if (!glnx_openat_rdonly (repo->repo_dir_fd, subpath, TRUE, &existing_fd, NULL)) + return FALSE; + + g_autoptr(GBytes) existing_data = glnx_fd_readall_bytes (existing_fd, cancellable, NULL); + if (existing_data == NULL) + return FALSE; + + return g_bytes_equal (existing_data, data); +} + +gboolean +_ostree_repo_static_delta_reindex (OstreeRepo *repo, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) all_deltas = NULL; + g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */ + + deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref); + + if (opt_to_commit == NULL) + { + g_autoptr(GPtrArray) old_indexes = NULL; + + /* To ensure all old index files either is regenerated, or + * removed, we initialize all existing indexes to NULL in the + * hashtable. */ + if (!ostree_repo_list_static_delta_indexes (repo, &old_indexes, cancellable, error)) + return FALSE; + + for (int i = 0; i < old_indexes->len; i++) + { + const char *old_index = g_ptr_array_index (old_indexes, i); + g_hash_table_insert (deltas_to_commit_ht, g_strdup (old_index), NULL); + } + } + else + { + if (!ostree_validate_checksum_string (opt_to_commit, error)) + return FALSE; + + /* We ensure the specific old index either is regenerated, or removed */ + g_hash_table_insert (deltas_to_commit_ht, g_strdup (opt_to_commit), NULL); + } + + if (!ostree_repo_list_static_delta_names (repo, &all_deltas, cancellable, error)) + return FALSE; + + for (int i = 0; i < all_deltas->len; i++) + { + const char *delta_name = g_ptr_array_index (all_deltas, i); + g_autofree char *from = NULL; + g_autofree char *to = NULL; + GPtrArray *deltas_to_commit = NULL; + + if (!_ostree_parse_delta_name (delta_name, &from, &to, error)) + return FALSE; + + if (opt_to_commit != NULL && strcmp (to, opt_to_commit) != 0) + continue; + + deltas_to_commit = g_hash_table_lookup (deltas_to_commit_ht, to); + if (deltas_to_commit == NULL) + { + deltas_to_commit = g_ptr_array_new_with_free_func (g_free); + g_hash_table_insert (deltas_to_commit_ht, g_steal_pointer (&to), deltas_to_commit); + } + + g_ptr_array_add (deltas_to_commit, g_steal_pointer (&from)); + } + + GLNX_HASH_TABLE_FOREACH_KV (deltas_to_commit_ht, const char*, to, GPtrArray*, froms) + { + g_autofree char *index_path = _ostree_get_relative_static_delta_index_path (to); + + if (froms == NULL) + { + /* No index to this checksum seen, delete if it exists */ + + g_debug ("Removing delta index for %s", to); + if (!ot_ensure_unlinked_at (repo->repo_dir_fd, index_path, error)) + return FALSE; + } + else + { + g_auto(GVariantDict) index_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_autoptr(GVariant) index_variant = NULL; + g_autoptr(GBytes) index = NULL; + + /* We sort on from here so that the index file is reproducible */ + g_ptr_array_sort (froms, (GCompareFunc)g_strcmp0); + + g_variant_dict_init (&deltas_builder, NULL); + + for (int i = 0; i < froms->len; i++) + { + const char *from = g_ptr_array_index (froms, i); + g_autofree char *delta_name = NULL; + GVariant *digest; + + digest = _ostree_repo_static_delta_superblock_digest (repo, from, to, cancellable, error); + if (digest == NULL) + return FALSE; + + if (from != NULL) + delta_name = g_strconcat (from, "-", to, NULL); + else + delta_name = g_strdup (to); + + g_variant_dict_insert_value (&deltas_builder, delta_name, digest); + } + + /* The toplevel of the index is an a{sv} for extensibility, and we use same key name (and format) as when + * storing deltas in the summary. */ + g_variant_dict_init (&index_builder, NULL); + + g_variant_dict_insert_value (&index_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); + + index_variant = g_variant_ref_sink (g_variant_dict_end (&index_builder)); + index = g_variant_get_data_as_bytes (index_variant); + + g_autofree char *index_dirname = g_path_get_dirname (index_path); + if (!glnx_shutil_mkdir_p_at (repo->repo_dir_fd, index_dirname, DEFAULT_DIRECTORY_MODE, cancellable, error)) + return FALSE; + + /* delta indexes are generally small and static, so reading it back and comparing is cheap, and it will + lower the write load (and particular sync-load) on the disk during reindexing (i.e. summary updates), */ + if (file_has_content (repo, index_path, index, cancellable)) + continue; + + g_debug ("Updating delta index for %s", to); + if (!glnx_file_replace_contents_at (repo->repo_dir_fd, index_path, + g_bytes_get_data (index, NULL), g_bytes_get_size (index), + 0, cancellable, error)) + return FALSE; + } + } + + return TRUE; +} diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 5a2e687922..d6c706da3b 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -227,6 +227,11 @@ _ostree_repo_static_delta_delete (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); +gboolean +_ostree_repo_static_delta_reindex (OstreeRepo *repo, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error); /* Used for static deltas which due to a historical mistake are * inconsistent endian. diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index ba3e877f4f..3a331c903a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5927,6 +5927,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_ref_sink (summary); } + if (!_ostree_repo_static_delta_reindex (self, NULL, cancellable, error)) + return FALSE; + if (!_ostree_repo_file_replace_contents (self, self->repo_dir_fd, "summary", From 024ef1d756b2580ab99d0f69f451444be59c9ecc Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 12:26:17 +0200 Subject: [PATCH 04/14] deltas: Add and document no-deltas-in-summary config option By default this is FALSE to keep existing clients working. --- man/ostree.repo-config.xml | 14 ++++++++++ src/libostree/ostree-repo.c | 54 +++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index 7a01fc0172..e4984430b9 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -249,6 +249,20 @@ Boston, MA 02111-1307, USA. costly). + + + no-deltas-in-summary + Boolean value controlling whether OSTree should skip + putting an index of available deltas in the summary file. Defaults to false. + + + Since 2020.7 OSTree can use delta indexes outside the summary file, + making the summary file smaller (especially for larger repositories). However + by default we still create the index in the summary file to make older clients + work. If you know all clients will be 2020.7 later you can enable this to + save network bandwidth. + + diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 3a331c903a..9d9146b3a5 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5749,6 +5749,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self, * commits from working. */ g_autoptr(OstreeRepoAutoLock) lock = NULL; + gboolean no_deltas_in_summary = FALSE; + lock = _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, error); if (!lock) @@ -5781,35 +5783,41 @@ ostree_repo_regenerate_summary (OstreeRepo *self, } } - { - g_autoptr(GPtrArray) delta_names = NULL; - g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; + if (!ot_keyfile_get_boolean_with_default (self->config, "core", + "no-deltas-in-summary", FALSE, + &no_deltas_in_summary, error)) + return FALSE; - if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) - return FALSE; + if (!no_deltas_in_summary) + { + g_autoptr(GPtrArray) delta_names = NULL; + g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; - g_variant_dict_init (&deltas_builder, NULL); - for (guint i = 0; i < delta_names->len; i++) - { - g_autofree char *from = NULL; - g_autofree char *to = NULL; - GVariant *digest; + if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) + return FALSE; - if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error)) - return FALSE; + g_variant_dict_init (&deltas_builder, NULL); + for (guint i = 0; i < delta_names->len; i++) + { + g_autofree char *from = NULL; + g_autofree char *to = NULL; + GVariant *digest; - digest = _ostree_repo_static_delta_superblock_digest (self, - (from && from[0]) ? from : NULL, - to, cancellable, error); - if (digest == NULL) - return FALSE; + if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error)) + return FALSE; - g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); - } + digest = _ostree_repo_static_delta_superblock_digest (self, + (from && from[0]) ? from : NULL, + to, cancellable, error); + if (digest == NULL) + return FALSE; - if (delta_names->len > 0) - g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); - } + g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); + } + + if (delta_names->len > 0) + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); + } { g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_LAST_MODIFIED, From c304703e1d88265f1fd76744b1abaa2ddf415a9e Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 12 Oct 2020 15:57:53 +0200 Subject: [PATCH 05/14] deltas: Make ostree_repo_static_delta_reindex() public It is useful to be able to trigger this without having to regenerate the summary. For example, if you are not using summaries, or ar generating the summaries yourself. --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo-static-delta-core.c | 26 ++++++++++++++++--- src/libostree/ostree-repo.c | 2 +- src/libostree/ostree-repo.h | 17 ++++++++++++ 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index ea37823bf3..81dc889082 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -413,6 +413,7 @@ ostree_repo_list_objects ostree_repo_list_commit_objects_starting_with ostree_repo_list_static_delta_names ostree_repo_list_static_delta_indexes +ostree_repo_static_delta_reindex OstreeStaticDeltaGenerateOpt ostree_repo_static_delta_generate ostree_repo_static_delta_execute_offline_with_signature diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 1350341cef..82d6a9b6f1 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,6 +20,7 @@ LIBOSTREE_2020.8 { global: ostree_repo_list_static_delta_indexes; + ostree_repo_static_delta_reindex; } LIBOSTREE_2020.7; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index c58e368387..670a10a515 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -1238,11 +1238,29 @@ file_has_content (OstreeRepo *repo, return g_bytes_equal (existing_data, data); } +/** + * ostree_repo_static_delta_reindex: + * @repo: Repo + * @flags: Flags affecting the indexing operation + * @opt_to_commit: ASCII SHA256 checksum of target commit, or %NULL to index all targets + * @cancellable: Cancellable + * @error: Error + * + * The delta index for a particular commit lists all the existing deltas that can be used + * when downloading that commit. This operation regenerates these indexes, either for + * a particular commit (if @opt_to_commit is non-%NULL), or for all commits that + * are reachable by an existing delta (if @opt_to_commit is %NULL). + * + * This is normally called automatically when the summary is updated in ostree_repo_regenerate_summary(). + * + * Locking: shared + */ gboolean -_ostree_repo_static_delta_reindex (OstreeRepo *repo, - const char *opt_to_commit, - GCancellable *cancellable, - GError **error) +ostree_repo_static_delta_reindex (OstreeRepo *repo, + OstreeStaticDeltaIndexFlags flags, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error) { g_autoptr(GPtrArray) all_deltas = NULL; g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 9d9146b3a5..c22a6666c6 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5935,7 +5935,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_ref_sink (summary); } - if (!_ostree_repo_static_delta_reindex (self, NULL, cancellable, error)) + if (!ostree_repo_static_delta_reindex (self, 0, NULL, cancellable, error)) return FALSE; if (!_ostree_repo_file_replace_contents (self, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index a3c5dc7799..6201e7b3c9 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1074,6 +1074,23 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo *self, GCancellable *cancellable, GError **error); +/** + * OstreeStaticDeltaIndexFlags: + * @OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE: No special flags + * + * Flags controlling static delta index generation. + */ +typedef enum { + OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0, +} OstreeStaticDeltaIndexFlags; + +_OSTREE_PUBLIC +gboolean ostree_repo_static_delta_reindex (OstreeRepo *repo, + OstreeStaticDeltaIndexFlags flags, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, GFile *dir_or_file, From 625606a7eccffd246b359a12215f1b5bad22b702 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 12:28:17 +0200 Subject: [PATCH 06/14] deltas: Add CLI ops to list and reindex delta-indexes --- src/ostree/ot-builtin-static-delta.c | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 3e0af5bd96..ff31b574e8 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -53,6 +53,8 @@ BUILTINPROTO(delete); BUILTINPROTO(generate); BUILTINPROTO(apply_offline); BUILTINPROTO(verify); +BUILTINPROTO(indexes); +BUILTINPROTO(reindex); #undef BUILTINPROTO @@ -75,6 +77,12 @@ static OstreeCommand static_delta_subcommands[] = { { "verify", OSTREE_BUILTIN_FLAG_NONE, ot_static_delta_builtin_verify, "Verify static delta signatures" }, + { "indexes", OSTREE_BUILTIN_FLAG_NONE, + ot_static_delta_builtin_indexes, + "List static delta indexes" }, + { "reindex", OSTREE_BUILTIN_FLAG_NONE, + ot_static_delta_builtin_reindex, + "Regenerate static delta indexes" }, { NULL, 0, NULL, NULL } }; @@ -126,6 +134,15 @@ static GOptionEntry verify_options[] = { { NULL } }; +static GOptionEntry indexes_options[] = { + { NULL } +}; + +static GOptionEntry reindex_options[] = { + { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Only update delta index to revision REV", "REV" }, + { NULL } +}; + static void static_delta_usage (char **argv, gboolean is_error) @@ -176,6 +193,46 @@ ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *in return TRUE; } +static gboolean +ot_static_delta_builtin_indexes (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr(OstreeRepo) repo = NULL; + g_autoptr(GOptionContext) context = g_option_context_new (""); + if (!ostree_option_context_parse (context, indexes_options, &argc, &argv, + invocation, &repo, cancellable, error)) + return FALSE; + + g_autoptr(GPtrArray) indexes = NULL; + if (!ostree_repo_list_static_delta_indexes (repo, &indexes, cancellable, error)) + return FALSE; + + if (indexes->len == 0) + g_print ("(No static deltas indexes)\n"); + else + { + for (guint i = 0; i < indexes->len; i++) + g_print ("%s\n", (char*)indexes->pdata[i]); + } + + return TRUE; +} + +static gboolean +ot_static_delta_builtin_reindex (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr(GOptionContext) context = g_option_context_new (""); + + g_autoptr(OstreeRepo) repo = NULL; + if (!ostree_option_context_parse (context, reindex_options, &argc, &argv, invocation, &repo, cancellable, error)) + return FALSE; + + if (!ostree_repo_static_delta_reindex (repo, 0, opt_to_rev, cancellable, error)) + return FALSE; + + return TRUE; +} + + static gboolean ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { From df7f07fc6c68717ba486e3869f508a28a857341b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 14:52:49 +0200 Subject: [PATCH 07/14] deltas: Use delta indexes when pulling If there is no delta index in the summary, try to fetch the delta index for the commit we're going to and use that to find the delta (if any). --- src/libostree/ostree-repo-pull-private.h | 4 +- src/libostree/ostree-repo-pull.c | 314 ++++++++++++++++++----- 2 files changed, 252 insertions(+), 66 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index a7ea85cd7e..d1c5fd148e 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -78,7 +78,8 @@ typedef struct { char *summary_sig_etag; guint64 summary_sig_last_modified; /* seconds since the epoch */ GVariant *summary; - GHashTable *summary_deltas_checksums; + GHashTable *summary_deltas_checksums; /* Filled from summary and delta indexes */ + gboolean summary_has_deltas; /* True if the summary existed and had a delta index */ GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ GHashTable *verified_commits; /* Set of commits that have been verified */ GHashTable *signapi_verified_commits; /* Map of commits that have been signapi verified */ @@ -93,6 +94,7 @@ typedef struct { GHashTable *requested_fallback_content; /* Maps checksum to itself */ GHashTable *pending_fetch_metadata; /* Map */ GHashTable *pending_fetch_content; /* Map */ + GHashTable *pending_fetch_delta_indexes; /* Set */ GHashTable *pending_fetch_delta_superblocks; /* Set */ GHashTable *pending_fetch_deltaparts; /* Set */ guint n_outstanding_metadata_fetches; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index bb14313603..70cbeca229 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -104,6 +104,14 @@ typedef struct { guint n_retries_remaining; } FetchDeltaSuperData; +typedef struct { + OtPullData *pull_data; + char *from_revision; + char *to_revision; + OstreeCollectionRef *requested_ref; /* (nullable) */ + guint n_retries_remaining; +} FetchDeltaIndexData; + static void variant_or_null_unref (gpointer data) { @@ -116,6 +124,8 @@ static void start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch); static void start_fetch_delta_superblock (OtPullData *pull_data, FetchDeltaSuperData *fetch_data); +static void start_fetch_delta_index (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data); static gboolean fetcher_queue_is_full (OtPullData *pull_data); static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, @@ -135,6 +145,8 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da static void enqueue_one_object_request_s (OtPullData *pull_data, FetchObjectData *fetch_data); +static void enqueue_one_static_delta_index_request_s (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data); static void enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data, FetchDeltaSuperData *fetch_data); static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data, @@ -149,6 +161,11 @@ static gboolean scan_one_metadata_object (OtPullData *pull_data, GCancellable *cancellable, GError **error); static void scan_object_queue_data_free (ScanObjectQueueData *scan_data); +static gboolean initiate_delta_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + const char *delta_from_revision, + GError **error); static gboolean update_progress (gpointer user_data) @@ -287,6 +304,7 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); g_queue_clear (&pull_data->scan_object_queue); g_hash_table_remove_all (pull_data->pending_fetch_metadata); + g_hash_table_remove_all (pull_data->pending_fetch_delta_indexes); g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks); g_hash_table_remove_all (pull_data->pending_fetch_deltaparts); g_hash_table_remove_all (pull_data->pending_fetch_content); @@ -320,6 +338,16 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_variant_unref (objname); } + /* Next, process delta index requests */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_indexes); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + FetchDeltaIndexData *fetch = key; + g_hash_table_iter_steal (&hiter); + start_fetch_delta_index (pull_data, g_steal_pointer (&fetch)); + } + /* Next, process delta superblock requests */ g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks); while (!fetcher_queue_is_full (pull_data) && @@ -2469,6 +2497,16 @@ fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data) g_free (fetch_data); } +static void +fetch_delta_index_data_free (FetchDeltaIndexData *fetch_data) +{ + g_free (fetch_data->from_revision); + g_free (fetch_data->to_revision); + if (fetch_data->requested_ref) + ostree_collection_ref_free (fetch_data->requested_ref); + g_free (fetch_data); +} + static void set_required_deltas_error (GError **error, const char *from_revision, @@ -2629,6 +2667,146 @@ validate_variant_is_csum (GVariant *csum, return ostree_validate_structureof_csum_v (csum, error); } +static gboolean +collect_available_deltas_for_pull (OtPullData *pull_data, + GVariant *deltas, + GError **error) +{ + gsize n; + + n = deltas ? g_variant_n_children (deltas) : 0; + for (gsize i = 0; i < n; i++) + { + const char *delta; + g_autoptr(GVariant) csum_v = NULL; + g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); + + g_variant_get_child (ref, 0, "&s", &delta); + g_variant_get_child (ref, 1, "v", &csum_v); + + if (!validate_variant_is_csum (csum_v, error)) + return FALSE; + + guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); + memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32); + g_hash_table_insert (pull_data->summary_deltas_checksums, + g_strdup (delta), + csum_data); + } + + return TRUE; +} + +static void +on_delta_index_fetched (GObject *src, + GAsyncResult *res, + gpointer data) + +{ + FetchDeltaIndexData *fetch_data = data; + OtPullData *pull_data = fetch_data->pull_data; + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + g_autoptr(GBytes) delta_index_data = NULL; + const char *from_revision = fetch_data->from_revision; + const char *to_revision = fetch_data->to_revision; + + if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src, + res, + &delta_index_data, + NULL, NULL, NULL, + error)) + { + if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + goto out; + g_clear_error (&local_error); + + /* below call to initiate_delta_request() will fail finding the delta and fall back to commit */ + } + else + { + g_autoptr(GVariant) delta_index = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, delta_index_data, FALSE)); + g_autoptr(GVariant) deltas = g_variant_lookup_value (delta_index, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); + + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; + } + + if (!initiate_delta_request (pull_data, + fetch_data->requested_ref, + to_revision, + from_revision, + &local_error)) + goto out; + + out: + g_assert (pull_data->n_outstanding_metadata_fetches > 0); + pull_data->n_outstanding_metadata_fetches--; + + if (local_error == NULL) + pull_data->n_fetched_metadata++; + + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); + + g_clear_pointer (&fetch_data, fetch_delta_index_data_free); +} + +static void +start_fetch_delta_index (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data) +{ + g_autofree char *delta_name = + _ostree_get_relative_static_delta_index_path (fetch_data->to_revision); + _ostree_fetcher_request_to_membuf (pull_data->fetcher, + pull_data->content_mirrorlist, + delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, + OSTREE_MAX_METADATA_SIZE, + 0, pull_data->cancellable, + on_delta_index_fetched, + g_steal_pointer (&fetch_data)); + pull_data->n_outstanding_metadata_fetches++; + pull_data->n_requested_metadata++; +} + +static void +enqueue_one_static_delta_index_request_s (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data) +{ + if (fetcher_queue_is_full (pull_data)) + { + g_debug ("queuing fetch of static delta index to %s", + fetch_data->to_revision); + + g_hash_table_add (pull_data->pending_fetch_delta_indexes, + g_steal_pointer (&fetch_data)); + } + else + { + start_fetch_delta_index (pull_data, g_steal_pointer (&fetch_data)); + } +} + +/* Start a request for a static delta index */ +static void +enqueue_one_static_delta_index_request (OtPullData *pull_data, + const char *to_revision, + const char *from_revision, + const OstreeCollectionRef *ref) +{ + FetchDeltaIndexData *fdata = g_new0(FetchDeltaIndexData, 1); + fdata->pull_data = pull_data; + fdata->from_revision = g_strdup (from_revision); + fdata->to_revision = g_strdup (to_revision); + fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + fdata->n_retries_remaining = pull_data->n_network_retries; + + enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fdata)); +} + static gboolean _ostree_repo_verify_summary (OstreeRepo *self, const char *name, @@ -3259,6 +3437,65 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, return TRUE; } +static gboolean +initiate_delta_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + const char *delta_from_revision, + GError **error) +{ + DeltaSearchResult deltares; + + /* Look for a delta to @to_revision in the summary data */ + if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares, + pull_data->cancellable, error)) + return FALSE; + + switch (deltares.result) + { + case DELTA_SEARCH_RESULT_NO_MATCH: + { + if (pull_data->require_static_deltas) /* No deltas found; are they required? */ + { + set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); + return FALSE; + } + else /* No deltas, fall back to object fetches. */ + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + } + break; + case DELTA_SEARCH_RESULT_FROM: + enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); + break; + case DELTA_SEARCH_RESULT_SCRATCH: + { + /* If a from-scratch delta is available, we don’t want to use it if + * the ref already exists locally, since we are likely only a few + * commits out of date; so doing an object pull is likely more + * bandwidth efficient. */ + if (delta_from_revision != NULL) + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + else + enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); + } + break; + case DELTA_SEARCH_RESULT_UNCHANGED: + { + /* If we already have the commit, here things get a little special; we've historically + * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas + * path, we don't, under the assumption the user wants as little network traffic as + * possible. + */ + if (pull_data->require_static_deltas) + break; + else + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + } + } + + return TRUE; +} + /* * initiate_request: * @ref: Optional ref name and collection ID @@ -3301,53 +3538,14 @@ initiate_request (OtPullData *pull_data, /* If we have a summary, we can use the newer logic */ if (pull_data->summary) { - DeltaSearchResult deltares; - - /* Look for a delta to @to_revision in the summary data */ - if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares, - pull_data->cancellable, error)) - return FALSE; - - switch (deltares.result) + if (!pull_data->summary_has_deltas) { - case DELTA_SEARCH_RESULT_NO_MATCH: - { - if (pull_data->require_static_deltas) /* No deltas found; are they required? */ - { - set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); - return FALSE; - } - else /* No deltas, fall back to object fetches. */ - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - } - break; - case DELTA_SEARCH_RESULT_FROM: - enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); - break; - case DELTA_SEARCH_RESULT_SCRATCH: - { - /* If a from-scratch delta is available, we don’t want to use it if - * the ref already exists locally, since we are likely only a few - * commits out of date; so doing an object pull is likely more - * bandwidth efficient. */ - if (delta_from_revision != NULL) - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - else - enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); - } - break; - case DELTA_SEARCH_RESULT_UNCHANGED: - { - /* If we already have the commit, here things get a little special; we've historically - * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas - * path, we don't, under the assumption the user wants as little network traffic as - * possible. - */ - if (pull_data->require_static_deltas) - break; - else - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - } + enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref); + } + else + { + if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error)) + return FALSE; } } else if (ref != NULL) @@ -3657,6 +3855,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, (GDestroyNotify)fetch_object_data_free); + pull_data->pending_fetch_delta_indexes = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_index_data_free, NULL); pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); @@ -4314,25 +4513,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, } deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); - n = deltas ? g_variant_n_children (deltas) : 0; - for (i = 0; i < n; i++) - { - const char *delta; - g_autoptr(GVariant) csum_v = NULL; - g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); - - g_variant_get_child (ref, 0, "&s", &delta); - g_variant_get_child (ref, 1, "v", &csum_v); - - if (!validate_variant_is_csum (csum_v, error)) - goto out; - - guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); - memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32); - g_hash_table_insert (pull_data->summary_deltas_checksums, - g_strdup (delta), - csum_data); - } + pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0; + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; } if (pull_data->summary && @@ -4900,6 +5083,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_delta_indexes, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref); g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); From e8a7485458fe744afb168e64a2e1422c578ac38b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Sep 2020 16:00:08 +0200 Subject: [PATCH 08/14] deltas: Add tests for delta indexes This tests generation of the index as well as using it when pulling --- tests/test-delta.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 557447bc73..bfdec593ff 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -28,7 +28,7 @@ skip_without_user_xattrs bindatafiles="bash true ostree" morebindatafiles="false ls" -echo '1..12' +echo '1..13' mkdir repo ostree_repo_init repo --mode=archive @@ -90,6 +90,11 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1 ${CMD_PREFIX} ostree --repo=repo prune ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta reindex +${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount +assert_file_has_content indexcount "^1$" +${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1 + permuteDirectory 1 files ${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files @@ -119,6 +124,12 @@ ${CMD_PREFIX} ostree --repo=repo static-delta generate --max-bsdiff-size=10000 - ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev}-${newrev} || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta reindex +${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount +assert_file_has_content indexcount "^2$" +${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${newrev} || exit 1 + if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --empty 2>>err.txt; then assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded" fi @@ -249,6 +260,41 @@ ${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null echo 'ok pull empty delta part' +rm -rf repo/delta-indexes +${CMD_PREFIX} ostree --repo=repo summary -u +${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt +assert_file_has_content summary.txt "ostree\.static\-deltas" + +${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary true +${CMD_PREFIX} ostree --repo=repo summary -u + +${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt +assert_not_file_has_content summary.txt "ostree\.static\-deltas" + +rm -rf repo2 +mkdir repo2 && ostree_repo_init repo2 --mode=bare-user +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} + +rm -rf repo/delta-indexes +if ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} &> no-delta.txt; then + assert_not_reached "failing pull with --require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content no-delta.txt "Static deltas required, but none found for" + +${CMD_PREFIX} ostree --repo=repo static-delta reindex +${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} + +${CMD_PREFIX} ostree --repo=repo2 fsck +${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null + +${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary false +${CMD_PREFIX} ostree --repo=repo summary -u + +${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt +assert_file_has_content summary.txt "ostree\.static\-deltas" + +echo 'ok pull delta part with delta index' + # Make a new branch to test "rebase deltas" echo otherbranch-content > files/otherbranch-content ${CMD_PREFIX} ostree --repo=repo commit -b otherbranch --tree=dir=files From 0984ff8471cb758f66e8958813670c8ee99e7358 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 9 Oct 2020 10:15:42 +0200 Subject: [PATCH 09/14] deltas: Take a shared repo lock while reindexing deltas This ensures we're not racing with a prune operation that can be removing the delta indexes we're relying on. --- src/libostree/ostree-repo-static-delta-core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 670a10a515..5370d1520a 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -1265,6 +1265,12 @@ ostree_repo_static_delta_reindex (OstreeRepo *repo, g_autoptr(GPtrArray) all_deltas = NULL; g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */ + /* Protect against parallel prune operation */ + g_autoptr(OstreeRepoAutoLock) lock = + _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_SHARED, cancellable, error); + if (!lock) + return FALSE; + deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref); if (opt_to_commit == NULL) From 6c8e6539e2487c5d30a18c9b89dd8d6cf4455bb7 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 9 Oct 2020 10:55:52 +0200 Subject: [PATCH 10/14] deltas: Set `indexed-deltas` key in the config and summary Clients can use these during pull and avoid downloading the summary if needed, or use the indexed-deltas instead of relying on the ones in the summary which may be left out. --- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo-static-delta-core.c | 15 +++++++++++++++ src/libostree/ostree-repo.c | 3 +++ 3 files changed, 19 insertions(+) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a48feca407..cbbe69717d 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -57,6 +57,7 @@ G_BEGIN_DECLS #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map" #define OSTREE_SUMMARY_MODE "ostree.summary.mode" #define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits" +#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas" #define _OSTREE_PAYLOAD_LINK_PREFIX "../" #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 5370d1520a..c253672489 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -1264,6 +1264,7 @@ ostree_repo_static_delta_reindex (OstreeRepo *repo, { g_autoptr(GPtrArray) all_deltas = NULL; g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */ + gboolean opt_indexed_deltas; /* Protect against parallel prune operation */ g_autoptr(OstreeRepoAutoLock) lock = @@ -1271,6 +1272,20 @@ ostree_repo_static_delta_reindex (OstreeRepo *repo, if (!lock) return FALSE; + /* Enusre that the "indexed-deltas" option is set on the config, so we know this when pulling */ + if (!ot_keyfile_get_boolean_with_default (repo->config, "core", + "indexed-deltas", FALSE, + &opt_indexed_deltas, error)) + return FALSE; + + if (!opt_indexed_deltas) + { + g_autoptr(GKeyFile) config = ostree_repo_copy_config (repo); + g_key_file_set_boolean (config, "core", "indexed-deltas", TRUE); + if (!ostree_repo_write_config (repo, config, error)) + return FALSE; + } + deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref); if (opt_to_commit == NULL) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index c22a6666c6..82f8db4436 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5842,6 +5842,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_new_boolean (tombstone_commits)); } + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_INDEXED_DELTAS, + g_variant_new_boolean (TRUE)); + /* Add refs which have a collection specified, which could be in refs/mirrors, * refs/heads, and/or refs/remotes. */ { From 125ed2b199153768f92049e48eb2cd40010f87ca Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 23 Oct 2020 13:05:25 +0200 Subject: [PATCH 11/14] pull: Only download summary if we need it for the pull operation If we have a commit id for all the refs we're pulling, and if we don't need the summary to list all the refs when mirroring then the only reason to download the summary is for the list of deltas. With the new "indexed-deltas" property in the config file (and mirrored to the summary file) we can detect when we don't need the summary for deltas and completely avoid downloading it then. --- src/libostree/ostree-repo-pull-private.h | 1 + src/libostree/ostree-repo-pull.c | 914 ++++++++++++----------- 2 files changed, 480 insertions(+), 435 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index d1c5fd148e..a827557aed 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -80,6 +80,7 @@ typedef struct { GVariant *summary; GHashTable *summary_deltas_checksums; /* Filled from summary and delta indexes */ gboolean summary_has_deltas; /* True if the summary existed and had a delta index */ + gboolean has_indexed_deltas; GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ GHashTable *verified_commits; /* Set of commits that have been verified */ GHashTable *signapi_verified_commits; /* Map of commits that have been signapi verified */ diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 70cbeca229..81956d3d64 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3535,18 +3535,17 @@ initiate_request (OtPullData *pull_data, return FALSE; } - /* If we have a summary, we can use the newer logic */ - if (pull_data->summary) + /* If we have a summary or delta index, we can use the newer logic. + * We prefer the index as it might have more deltas than the summary + * (i.e. leave some deltas out of summary to make it smaller). */ + if (pull_data->has_indexed_deltas) { - if (!pull_data->summary_has_deltas) - { - enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref); - } - else - { - if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error)) - return FALSE; - } + enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref); + } + else if (pull_data->summary_has_deltas) + { + if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error)) + return FALSE; } else if (ref != NULL) { @@ -3590,6 +3589,20 @@ initiate_request (OtPullData *pull_data, return TRUE; } +static gboolean +all_requested_refs_have_commit (GHashTable *requested_refs /* (element-type OstreeCollectionRef utf8) */) +{ + GLNX_HASH_TABLE_FOREACH_KV (requested_refs, const OstreeCollectionRef*, ref, + const char*, override_commitid) + { + /* Note: "" override means whatever is latest */ + if (override_commitid == NULL || *override_commitid == 0) + return FALSE; + } + + return TRUE; +} + /* ------------------------------------------------------------------------------------------ * Below is the libsoup-invariant API; these should match * the stub functions in the #else clause @@ -3695,9 +3708,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean opt_ref_keyring_map_set = FALSE; gboolean disable_sign_verify = FALSE; gboolean disable_sign_verify_summary = FALSE; + gboolean need_summary = FALSE; const char *main_collection_id = NULL; const char *url_override = NULL; gboolean inherit_transaction = FALSE; + gboolean require_summary_for_mirror = FALSE; g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ gsize i; g_autofree char **opt_localcache_repos = NULL; @@ -3709,6 +3724,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, */ const char *the_ref_to_fetch = NULL; OstreeRepoTransactionStats tstats = { 0, }; + gboolean remote_mode_loaded = FALSE; /* Default */ pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE; @@ -4132,408 +4148,509 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->is_commit_only) pull_data->disable_static_deltas = TRUE; - pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + /* Compute the set of collection-refs (and optional commit id) to fetch */ - { - g_autoptr(GBytes) bytes_sig = NULL; - gboolean summary_sig_not_modified = FALSE; - g_autofree char *summary_sig_etag = NULL; - guint64 summary_sig_last_modified = 0; - gsize n; - g_autoptr(GVariant) refs = NULL; - g_autoptr(GVariant) deltas = NULL; - g_autoptr(GVariant) additional_metadata = NULL; - gboolean summary_from_cache = FALSE; - gboolean remote_mode_loaded = FALSE; - gboolean tombstone_commits = FALSE; - - if (summary_sig_bytes_v) - { - /* Must both be specified */ - g_assert (summary_bytes_v); - - bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v); - bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v); + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches) + { + require_summary_for_mirror = TRUE; + } + else if (opt_collection_refs_set) + { + const gchar *collection_id, *ref_name, *checksum; - if (!bytes_sig || !bytes_summary) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "summary-bytes or summary-sig-bytes set to invalid value"); + while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum)) + { + if (!ostree_validate_rev (ref_name, error)) goto out; - } + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (collection_id, ref_name), + (*checksum != '\0') ? g_strdup (checksum) : NULL); + } + } + else if (refs_to_fetch != NULL) + { + char **strviter = refs_to_fetch; + char **commitid_strviter = override_commit_ids ?: NULL; - g_debug ("Loaded %s summary from options", remote_name_or_baseurl); - } + while (*strviter) + { + const char *branch = *strviter; - if (!bytes_sig) - { - g_autofree char *summary_sig_if_none_match = NULL; - guint64 summary_sig_if_modified_since = 0; - - /* Load the summary.sig from the network, but send its ETag and - * Last-Modified from the on-disk cache (if it exists) to reduce the - * download size if nothing’s changed. */ - _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig", - &summary_sig_if_none_match, &summary_sig_if_modified_since); - - g_clear_pointer (&summary_sig_etag, g_free); - summary_sig_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - summary_sig_if_none_match, summary_sig_if_modified_since, - pull_data->n_network_retries, - &bytes_sig, - &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) + if (ostree_validate_checksum_string (branch, NULL)) + { + char *key = g_strdup (branch); + g_hash_table_add (commits_to_fetch, key); + } + else + { + if (!ostree_validate_rev (branch, error)) + goto out; + char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (NULL, branch), commitid); + } + + strviter++; + if (commitid_strviter) + commitid_strviter++; + } + } + else + { + char **branches_iter; + + branches_iter = configured_branches; + + if (!(branches_iter && *branches_iter)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No configured branches for remote %s", remote_name_or_baseurl); goto out; + } + for (;branches_iter && *branches_iter; branches_iter++) + { + const char *branch = *branches_iter; - /* The server returned HTTP status 304 Not Modified, so we’re clear to - * load summary.sig from the cache. Also load summary, since - * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ - if (summary_sig_not_modified) - { - g_clear_pointer (&bytes_sig, g_bytes_unref); - g_clear_pointer (&bytes_summary, g_bytes_unref); - if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig", - &bytes_sig, - cancellable, error)) - goto out; + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (NULL, branch), NULL); + } + } - if (!bytes_summary && - !pull_data->remote_repo_local && - !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, - &bytes_summary, - cancellable, error)) - goto out; - } - } + /* Deltas are necessary when mirroring or resolving a requested ref to a commit. + * We try to avoid loading the potentially large summary if it is not needed. */ + need_summary = require_summary_for_mirror || !all_requested_refs_have_commit (requested_refs_to_fetch) || summary_sig_bytes_v != NULL; - if (bytes_sig && - !bytes_summary && - !pull_data->remote_repo_local && - !_ostree_repo_load_cache_summary_if_same_sig (self, - remote_name_or_baseurl, - bytes_sig, - &bytes_summary, - cancellable, - error)) - goto out; + /* If we don't have indexed deltas, we need the summary for deltas, so check + * the config file for support. + * NOTE: Avoid download if we don't need deltas */ + if (!need_summary && !pull_data->disable_static_deltas) + { + if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) + goto out; - if (bytes_summary && !summary_bytes_v) - { - g_debug ("Loaded %s summary from cache", remote_name_or_baseurl); - summary_from_cache = TRUE; - } + /* Check if remote has delta indexes outside summary */ + if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "indexed-deltas", FALSE, + &pull_data->has_indexed_deltas, error)) + goto out; - if (!pull_data->summary && !bytes_summary) - { - g_autofree char *summary_if_none_match = NULL; - guint64 summary_if_modified_since = 0; + if (!pull_data->has_indexed_deltas) + need_summary = TRUE; + } + + pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL, - &summary_if_none_match, &summary_if_modified_since); + if (need_summary) + { + g_autoptr(GBytes) bytes_sig = NULL; + gboolean summary_sig_not_modified = FALSE; + g_autofree char *summary_sig_etag = NULL; + guint64 summary_sig_last_modified = 0; + gsize n; + g_autoptr(GVariant) refs = NULL; + g_autoptr(GVariant) deltas = NULL; + g_autoptr(GVariant) additional_metadata = NULL; + gboolean summary_from_cache = FALSE; + gboolean tombstone_commits = FALSE; - g_clear_pointer (&summary_etag, g_free); - summary_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - summary_if_none_match, summary_if_modified_since, - pull_data->n_network_retries, + if (summary_sig_bytes_v) + { + /* Must both be specified */ + g_assert (summary_bytes_v); + + bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v); + bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v); + + if (!bytes_sig || !bytes_summary) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "summary-bytes or summary-sig-bytes set to invalid value"); + goto out; + } + + g_debug ("Loaded %s summary from options", remote_name_or_baseurl); + } + + if (!bytes_sig) + { + g_autofree char *summary_sig_if_none_match = NULL; + guint64 summary_sig_if_modified_since = 0; + + /* Load the summary.sig from the network, but send its ETag and + * Last-Modified from the on-disk cache (if it exists) to reduce the + * download size if nothing’s changed. */ + _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig", + &summary_sig_if_none_match, &summary_sig_if_modified_since); + + g_clear_pointer (&summary_sig_etag, g_free); + summary_sig_last_modified = 0; + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + summary_sig_if_none_match, summary_sig_if_modified_since, + pull_data->n_network_retries, + &bytes_sig, + &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) + goto out; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary.sig from the cache. Also load summary, since + * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ + if (summary_sig_not_modified) + { + g_clear_pointer (&bytes_sig, g_bytes_unref); + g_clear_pointer (&bytes_summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig", + &bytes_sig, + cancellable, error)) + goto out; + + if (!bytes_summary && + !pull_data->remote_repo_local && + !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, &bytes_summary, - &summary_not_modified, &summary_etag, &summary_last_modified, - OSTREE_MAX_METADATA_SIZE, cancellable, error)) - goto out; + goto out; + } + } - /* The server returned HTTP status 304 Not Modified, so we’re clear to - * load summary from the cache. */ - if (summary_not_modified) - { - g_clear_pointer (&bytes_summary, g_bytes_unref); - if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, - &bytes_summary, - cancellable, error)) - goto out; - } - } + if (bytes_sig && + !bytes_summary && + !pull_data->remote_repo_local && + !_ostree_repo_load_cache_summary_if_same_sig (self, + remote_name_or_baseurl, + bytes_sig, + &bytes_summary, + cancellable, + error)) + goto out; + + if (bytes_summary && !summary_bytes_v) + { + g_debug ("Loaded %s summary from cache", remote_name_or_baseurl); + summary_from_cache = TRUE; + } + + if (!pull_data->summary && !bytes_summary) + { + g_autofree char *summary_if_none_match = NULL; + guint64 summary_if_modified_since = 0; + + _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL, + &summary_if_none_match, &summary_if_modified_since); + + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; + + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + summary_if_none_match, summary_if_modified_since, + pull_data->n_network_retries, + &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) + goto out; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary from the cache. */ + if (summary_not_modified) + { + g_clear_pointer (&bytes_summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, + &bytes_summary, + cancellable, error)) + goto out; + } + } #ifndef OSTREE_DISABLE_GPGME - if (!bytes_summary && pull_data->gpg_verify_summary) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)"); - goto out; - } + if (!bytes_summary && pull_data->gpg_verify_summary) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)"); + goto out; + } #endif /* OSTREE_DISABLE_GPGME */ - if (!bytes_summary && pull_data->require_static_deltas) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Fetch configured to require static deltas, but no summary found"); - goto out; - } + if (!bytes_summary && require_summary_for_mirror) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Fetching all refs was requested in mirror mode, but remote repository does not have a summary"); + goto out; + } #ifndef OSTREE_DISABLE_GPGME - if (!bytes_sig && pull_data->gpg_verify_summary) - { - g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, - "GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)"); - goto out; - } + if (!bytes_sig && pull_data->gpg_verify_summary) + { + g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, + "GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)"); + goto out; + } - if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) - { - g_autoptr(OstreeGpgVerifyResult) result = NULL; - g_autoptr(GError) temp_error = NULL; + if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) + { + g_autoptr(OstreeGpgVerifyResult) result = NULL; + g_autoptr(GError) temp_error = NULL; - result = ostree_repo_verify_summary (self, pull_data->remote_name, - bytes_summary, bytes_sig, - cancellable, &temp_error); - if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error)) - { - if (summary_from_cache) - { - /* The cached summary doesn't match, fetch a new one and verify again. - * Don’t set the cache headers in the HTTP request, to force a - * full download. */ - if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Remote %s cached summary invalid and " - "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", - pull_data->remote_name); + result = ostree_repo_verify_summary (self, pull_data->remote_name, + bytes_summary, bytes_sig, + cancellable, &temp_error); + if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error)) + { + if (summary_from_cache) + { + /* The cached summary doesn't match, fetch a new one and verify again. + * Don’t set the cache headers in the HTTP request, to force a + * full download. */ + if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Remote %s cached summary invalid and " + "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", + pull_data->remote_name); + goto out; + } + else + g_debug ("Remote %s cached summary invalid, pulling new version", + pull_data->remote_name); + + summary_from_cache = FALSE; + g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, /* no cache headers */ + pull_data->n_network_retries, + &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; - } - else - g_debug ("Remote %s cached summary invalid, pulling new version", - pull_data->remote_name); - - summary_from_cache = FALSE; - g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); - g_clear_pointer (&summary_etag, g_free); - summary_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary", - OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - NULL, 0, /* no cache headers */ - pull_data->n_network_retries, - &bytes_summary, - &summary_not_modified, &summary_etag, &summary_last_modified, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) - goto out; - g_autoptr(OstreeGpgVerifyResult) retry = - ostree_repo_verify_summary (self, pull_data->remote_name, - bytes_summary, bytes_sig, - cancellable, error); - if (!ostree_gpg_verify_result_require_valid_signature (retry, error)) + g_autoptr(OstreeGpgVerifyResult) retry = + ostree_repo_verify_summary (self, pull_data->remote_name, + bytes_summary, bytes_sig, + cancellable, error); + if (!ostree_gpg_verify_result_require_valid_signature (retry, error)) + goto out; + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); goto out; - } - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } - } + } + } + } #endif /* OSTREE_DISABLE_GPGME */ - if (pull_data->signapi_summary_verifiers) - { - if (!bytes_sig && pull_data->signapi_summary_verifiers) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)"); - goto out; - } - if (bytes_summary && bytes_sig) - { - g_autoptr(GVariant) signatures = NULL; - g_autoptr(GError) temp_error = NULL; - - signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, - bytes_sig, FALSE); - - - g_assert (pull_data->signapi_summary_verifiers); - if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error)) - { - if (summary_from_cache) - { - /* The cached summary doesn't match, fetch a new one and verify again. - * Don’t set the cache headers in the HTTP request, to force a - * full download. */ - if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Remote %s cached summary invalid and " - "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", - pull_data->remote_name); + if (pull_data->signapi_summary_verifiers) + { + if (!bytes_sig && pull_data->signapi_summary_verifiers) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)"); + goto out; + } + if (bytes_summary && bytes_sig) + { + g_autoptr(GVariant) signatures = NULL; + g_autoptr(GError) temp_error = NULL; + + signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, + bytes_sig, FALSE); + + g_assert (pull_data->signapi_summary_verifiers); + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error)) + { + if (summary_from_cache) + { + /* The cached summary doesn't match, fetch a new one and verify again. + * Don’t set the cache headers in the HTTP request, to force a + * full download. */ + if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Remote %s cached summary invalid and " + "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", + pull_data->remote_name); + goto out; + } + else + g_debug ("Remote %s cached summary invalid, pulling new version", + pull_data->remote_name); + + summary_from_cache = FALSE; + g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, /* no cache headers */ + pull_data->n_network_retries, + &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; - } - else - g_debug ("Remote %s cached summary invalid, pulling new version", - pull_data->remote_name); - - summary_from_cache = FALSE; - g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); - g_clear_pointer (&summary_etag, g_free); - summary_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary", - OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - NULL, 0, /* no cache headers */ - pull_data->n_network_retries, - &bytes_summary, - &summary_not_modified, &summary_etag, &summary_last_modified, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) - goto out; - if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error)) + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error)) goto out; - } - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } - } - } + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + goto out; + } + } + } + } - if (bytes_summary) - { - pull_data->summary_data = g_bytes_ref (bytes_summary); - pull_data->summary_etag = g_strdup (summary_etag); - pull_data->summary_last_modified = summary_last_modified; - pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); + if (bytes_summary) + { + pull_data->summary_data = g_bytes_ref (bytes_summary); + pull_data->summary_etag = g_strdup (summary_etag); + pull_data->summary_last_modified = summary_last_modified; + pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); - if (!g_variant_is_normal_form (pull_data->summary)) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not normal form"); - goto out; - } - if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Doesn't match variant type '%s'", - (char *)OSTREE_SUMMARY_GVARIANT_FORMAT); + if (!g_variant_is_normal_form (pull_data->summary)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not normal form"); + goto out; + } + if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Doesn't match variant type '%s'", + (char *)OSTREE_SUMMARY_GVARIANT_FORMAT); + goto out; + } + + if (bytes_sig) + { + pull_data->summary_data_sig = g_bytes_ref (bytes_sig); + pull_data->summary_sig_etag = g_strdup (summary_sig_etag); + pull_data->summary_sig_last_modified = summary_sig_last_modified; + } + } + + if (!summary_from_cache && bytes_summary && bytes_sig) + { + if (!pull_data->remote_repo_local && + !_ostree_repo_cache_summary (self, + remote_name_or_baseurl, + bytes_summary, + summary_etag, summary_last_modified, + bytes_sig, + summary_sig_etag, summary_sig_last_modified, + cancellable, + error)) goto out; - } + } - if (bytes_sig) - { - pull_data->summary_data_sig = g_bytes_ref (bytes_sig); - pull_data->summary_sig_etag = g_strdup (summary_sig_etag); - pull_data->summary_sig_last_modified = summary_sig_last_modified; - } - } + if (pull_data->summary) + { + additional_metadata = g_variant_get_child_value (pull_data->summary, 1); - if (!summary_from_cache && bytes_summary && bytes_sig) - { - if (!pull_data->remote_repo_local && - !_ostree_repo_cache_summary (self, - remote_name_or_baseurl, - bytes_summary, - summary_etag, summary_last_modified, - bytes_sig, - summary_sig_etag, summary_sig_last_modified, - cancellable, - error)) - goto out; - } + if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) + main_collection_id = NULL; + else if (!ostree_validate_collection_id (main_collection_id, error)) + goto out; - if (pull_data->summary) - { - additional_metadata = g_variant_get_child_value (pull_data->summary, 1); + refs = g_variant_get_child_value (pull_data->summary, 0); + for (i = 0, n = g_variant_n_children (refs); i < n; i++) + { + const char *refname; + g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i); - if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) - main_collection_id = NULL; - else if (!ostree_validate_collection_id (main_collection_id, error)) - goto out; + g_variant_get_child (ref, 0, "&s", &refname); - refs = g_variant_get_child_value (pull_data->summary, 0); - for (i = 0, n = g_variant_n_children (refs); i < n; i++) - { - const char *refname; - g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i); + if (!ostree_validate_rev (refname, error)) + goto out; - g_variant_get_child (ref, 0, "&s", &refname); + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (main_collection_id, refname), NULL); + } + } - if (!ostree_validate_rev (refname, error)) - goto out; + g_autoptr(GVariant) collection_map = NULL; + collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); + if (collection_map != NULL) + { + GVariantIter collection_map_iter; + const char *collection_id; + g_autoptr(GVariant) collection_refs = NULL; - if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) - { - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (main_collection_id, refname), NULL); - } - } + g_variant_iter_init (&collection_map_iter, collection_map); - g_autoptr(GVariant) collection_map = NULL; - collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); - if (collection_map != NULL) - { - GVariantIter collection_map_iter; - const char *collection_id; - g_autoptr(GVariant) collection_refs = NULL; + while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs)) + { + if (!ostree_validate_collection_id (collection_id, error)) + goto out; - g_variant_iter_init (&collection_map_iter, collection_map); + for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++) + { + const char *refname; + g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i); - while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs)) - { - if (!ostree_validate_collection_id (collection_id, error)) - goto out; + g_variant_get_child (ref, 0, "&s", &refname); - for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++) - { - const char *refname; - g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i); + if (!ostree_validate_rev (refname, error)) + goto out; - g_variant_get_child (ref, 0, "&s", &refname); + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (collection_id, refname), NULL); + } + } + } + } - if (!ostree_validate_rev (refname, error)) - goto out; + deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); + pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0; + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; - if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) - { - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (collection_id, refname), NULL); - } - } - } - } + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_INDEXED_DELTAS, "b", &pull_data->has_indexed_deltas); + } - deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); - pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0; - if (!collect_available_deltas_for_pull (pull_data, deltas, error)) - goto out; - } + if (pull_data->summary && + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) && + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits)) + { + if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) + goto out; + pull_data->has_tombstone_commits = tombstone_commits; + remote_mode_loaded = TRUE; + } + } - if (pull_data->summary && - g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) && - g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits)) - { - if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) - goto out; - pull_data->has_tombstone_commits = tombstone_commits; - remote_mode_loaded = TRUE; - } - else if (pull_data->remote_repo_local == NULL) + if (pull_data->require_static_deltas && !pull_data->has_indexed_deltas && !pull_data->summary_has_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Fetch configured to require static deltas, but no summary deltas or delta index found"); + goto out; + } + + if (remote_mode_loaded && pull_data->remote_repo_local == NULL) { /* Fall-back path which loads the necessary config from the remote’s - * `config` file. Doing so is deprecated since it means an + * `config` file (unless we already read it above). Doing so is deprecated since it means an * additional round trip to the remote for each pull. No need to do * it for local pulls. */ - if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) + if (remote_config == NULL && + !load_remote_repo_config (pull_data, &remote_config, cancellable, error)) goto out; if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare", @@ -4550,85 +4667,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, remote_mode_loaded = TRUE; } - if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't pull from archives with mode \"%s\"", - remote_mode_str); - goto out; - } - } - - if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches) - { - if (!bytes_summary) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Fetching all refs was requested in mirror mode, but remote repository does not have a summary"); - goto out; - } - - } - else if (opt_collection_refs_set) - { - const gchar *collection_id, *ref_name, *checksum; - - while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum)) - { - if (!ostree_validate_rev (ref_name, error)) - goto out; - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (collection_id, ref_name), - (*checksum != '\0') ? g_strdup (checksum) : NULL); - } - } - else if (refs_to_fetch != NULL) - { - char **strviter = refs_to_fetch; - char **commitid_strviter = override_commit_ids ?: NULL; - - while (*strviter) - { - const char *branch = *strviter; - - if (ostree_validate_checksum_string (branch, NULL)) - { - char *key = g_strdup (branch); - g_hash_table_add (commits_to_fetch, key); - } - else - { - if (!ostree_validate_rev (branch, error)) - goto out; - char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (NULL, branch), commitid); - } - - strviter++; - if (commitid_strviter) - commitid_strviter++; - } - } - else + if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE) { - char **branches_iter; - - branches_iter = configured_branches; - - if (!(branches_iter && *branches_iter)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No configured branches for remote %s", remote_name_or_baseurl); - goto out; - } - for (;branches_iter && *branches_iter; branches_iter++) - { - const char *branch = *branches_iter; - - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (NULL, branch), NULL); - } + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't pull from archives with mode \"%s\"", + remote_mode_str); + goto out; } /* Resolve the checksum for each ref. This has to be done into a new hash table, From bc924ff8709845ab1add41367d51a972d89a7488 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 9 Oct 2020 16:30:29 +0200 Subject: [PATCH 12/14] tests: Add a testcase to ensure we're not using the summary if we don't need it With deltas outside the summary, if a commit is specified when pulling we don't download the summary. Verify this. --- tests/pull-test.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 082ed31512..0892183859 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -55,10 +55,10 @@ function verify_initial_contents() { } if has_gpgme; then - echo "1..35" + echo "1..36" else # 3 tests needs GPG support - echo "1..32" + echo "1..33" fi # Try both syntaxes @@ -381,6 +381,25 @@ assert_file_has_content err.txt "Upgrade.*is chronologically older" ${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${oldrev} origin main@${middlerev} echo "ok pull timestamp checking" +# test pull without override commit use summary, but with doesn't use summary +# We temporarily replace summary with broken one to detect if it is used +mv ostree-srv/gnomerepo/summary ostree-srv/gnomerepo/summary.backup +echo "broken" > ostree-srv/gnomerepo/summary + +repo_init --no-sign-verify +rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) +# This will need summary, so will fail +if ${CMD_PREFIX} ostree --repo=repo -v pull origin main; then + assert_not_reached "Should have failed with broken summary" +fi +# This won't need summary so will not fail +${CMD_PREFIX} ostree --repo=repo pull origin main@${rev} + +# Restore summary +mv ostree-srv/gnomerepo/summary.backup ostree-srv/gnomerepo/summary + +echo "ok pull with override id doesn't use summary" + cd ${test_tmpdir} repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main From 8cd796f3f1d0e82ea57b30229e692c31f9cb2e03 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 20 Oct 2020 08:37:35 +0200 Subject: [PATCH 13/14] Add ostree_repo_gpg_sign_data() This is similar to ostree_sign_data() but for the old gpg code. Flatpak will need this to reproduce a signed summary. --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo.c | 61 +++++++++++++++++++++++++++++++ src/libostree/ostree-repo.h | 10 +++++ 4 files changed, 73 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 81dc889082..64bc68d234 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -447,6 +447,7 @@ ostree_repo_pull_default_console_progress_changed ostree_repo_sign_commit ostree_repo_append_gpg_signature ostree_repo_add_gpg_signature_summary +ostree_repo_gpg_sign_data ostree_repo_gpg_verify_data ostree_repo_verify_commit ostree_repo_verify_commit_ext diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 82d6a9b6f1..435be1908e 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -21,6 +21,7 @@ LIBOSTREE_2020.8 { global: ostree_repo_list_static_delta_indexes; ostree_repo_static_delta_reindex; + ostree_repo_gpg_sign_data; } LIBOSTREE_2020.7; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 82f8db4436..3bbf5ea0f9 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5222,6 +5222,67 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, #endif /* OSTREE_DISABLE_GPGME */ } + +/** + * ostree_repo_gpg_sign_data: + * @self: Self + * @data: Data as a #GBytes + * @old_signatures: Existing signatures to append to (or %NULL) + * @key_id: (array zero-terminated=1) (element-type utf8): NULL-terminated array of GPG keys. + * @homedir: (allow-none): GPG home directory, or %NULL + * @out_signature: (out): in case of success will contain signature + * @cancellable: A #GCancellable + * @error: a #GError + * + * Sign the given @data with the specified keys in @key_id. Similar to + * ostree_repo_add_gpg_signature_summary() but can be used on any + * data. + * + * You can use ostree_repo_gpg_verify_data() to verify the signatures. + * + * Returns: @TRUE if @data has been signed successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.8 + */ +gboolean +ostree_repo_gpg_sign_data (OstreeRepo *self, + GBytes *data, + GBytes *old_signatures, + const gchar **key_id, + const gchar *homedir, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error) +{ +#ifndef OSTREE_DISABLE_GPGME + g_autoptr(GVariant) metadata = NULL; + g_autoptr(GVariant) res = NULL; + + if (old_signatures) + metadata = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), old_signatures, FALSE)); + + for (guint i = 0; key_id[i]; i++) + { + g_autoptr(GBytes) signature_data = NULL; + if (!sign_data (self, data, key_id[i], homedir, + &signature_data, + cancellable, error)) + return FALSE; + + g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata); + metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data); + } + + res = g_variant_get_normal_form (metadata); + *out_signatures = g_variant_get_data_as_bytes (res); + return TRUE; +#else + return glnx_throw (error, "GPG feature is disabled in a build time"); +#endif /* OSTREE_DISABLE_GPGME */ +} + + #ifndef OSTREE_DISABLE_GPGME /* Special remote for _ostree_repo_gpg_verify_with_metadata() */ static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__"; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 6201e7b3c9..e64c3230ce 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1416,6 +1416,16 @@ gboolean ostree_repo_append_gpg_signature (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_gpg_sign_data (OstreeRepo *self, + GBytes *data, + GBytes *old_signatures, + const gchar **key_id, + const gchar *homedir, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo *self, const gchar *commit_checksum, From 654f3d959a4563ccf7d36182f4ee7a25a70fa016 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 20 Oct 2020 15:51:08 +0200 Subject: [PATCH 14/14] ostree pull: Add more g_debug spew around fetching deltas This is useful to debug what is happening when downloading via deltas. --- src/libostree/ostree-repo-pull.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 81956d3d64..da421db383 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2145,6 +2145,7 @@ start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch) { g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i); + g_debug ("starting fetch of deltapart %s", deltapart_path); pull_data->n_outstanding_deltapart_fetches++; g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, @@ -2608,6 +2609,7 @@ start_fetch_delta_superblock (OtPullData *pull_data, g_autofree char *delta_name = _ostree_get_relative_static_delta_superblock_path (fetch_data->from_revision, fetch_data->to_revision); + g_debug ("starting fetch of delta superblock %s", delta_name); _ostree_fetcher_request_to_membuf (pull_data->fetcher, pull_data->content_mirrorlist, delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, @@ -2760,6 +2762,7 @@ start_fetch_delta_index (OtPullData *pull_data, { g_autofree char *delta_name = _ostree_get_relative_static_delta_index_path (fetch_data->to_revision); + g_debug ("starting fetch of delta index %s", delta_name); _ostree_fetcher_request_to_membuf (pull_data->fetcher, pull_data->content_mirrorlist, delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,