From 75a3c4d2938af4c15996f477beef05189529ffb5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 19 Sep 2023 12:31:42 +0200 Subject: [PATCH 01/12] image writer: Correctly handle mtime_nsec on big endiang i_mtime_nsec is 32bit, so use lcfs_u32_to_file, not lcfs_u64_to_file. The later was breaking on big endian arches. Signed-off-by: Alexander Larsson --- libcomposefs/lcfs-writer-erofs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcomposefs/lcfs-writer-erofs.c b/libcomposefs/lcfs-writer-erofs.c index c0617039..8bf66869 100644 --- a/libcomposefs/lcfs-writer-erofs.c +++ b/libcomposefs/lcfs-writer-erofs.c @@ -881,7 +881,7 @@ static int write_erofs_inode_data(struct lcfs_ctx_s *ctx, struct lcfs_node_s *no i.i_uid = lcfs_u32_to_file(node->inode.st_uid); i.i_gid = lcfs_u32_to_file(node->inode.st_gid); i.i_mtime = lcfs_u64_to_file(node->inode.st_mtim_sec); - i.i_mtime_nsec = lcfs_u64_to_file(node->inode.st_mtim_nsec); + i.i_mtime_nsec = lcfs_u32_to_file(node->inode.st_mtim_nsec); if (type == S_IFDIR) { if (node->erofs_n_blocks > 0) { From fd8c6610ec8e1239847efc6c6d7ee22737d147e8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 17:08:09 +0200 Subject: [PATCH 02/12] lib: Add LCFS_BUILD_USER_XATTRS This allows you to build an image with user xattrs, but not get things like e.g. selinux contexts. Signed-off-by: Alexander Larsson --- libcomposefs/lcfs-writer.c | 18 +++++++++++++++--- libcomposefs/lcfs-writer.h | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/libcomposefs/lcfs-writer.c b/libcomposefs/lcfs-writer.c index 13a8e06a..e467eacb 100644 --- a/libcomposefs/lcfs-writer.c +++ b/libcomposefs/lcfs-writer.c @@ -376,13 +376,15 @@ int lcfs_write_to(struct lcfs_node_s *root, struct lcfs_write_options_s *options return 0; } -static int read_xattrs(struct lcfs_node_s *ret, int dirfd, const char *fname) +static int read_xattrs(struct lcfs_node_s *ret, int dirfd, const char *fname, + int buildflags) { char path[PATH_MAX]; ssize_t list_size; cleanup_free char *list = NULL; ssize_t r = 0; cleanup_fd int fd = -1; + bool user_xattr = (buildflags & LCFS_BUILD_USER_XATTRS) != 0; fd = openat(dirfd, fname, O_PATH | O_NOFOLLOW | O_CLOEXEC, 0); if (fd < 0) @@ -409,6 +411,9 @@ static int read_xattrs(struct lcfs_node_s *ret, int dirfd, const char *fname) ssize_t value_size; cleanup_free char *value = NULL; + if (user_xattr && !str_has_prefix(it, "user.")) + continue; + value_size = getxattr(path, it, NULL, 0); if (value_size < 0) { return value_size; @@ -563,7 +568,14 @@ struct lcfs_node_s *lcfs_load_node_from_file(int dirfd, const char *fname, if (buildflags & ~(LCFS_BUILD_SKIP_XATTRS | LCFS_BUILD_USE_EPOCH | LCFS_BUILD_SKIP_DEVICES | LCFS_BUILD_COMPUTE_DIGEST | - LCFS_BUILD_NO_INLINE)) { + LCFS_BUILD_NO_INLINE | LCFS_BUILD_USER_XATTRS)) { + errno = EINVAL; + return NULL; + } + + if ((buildflags & LCFS_BUILD_SKIP_XATTRS) && + (buildflags & LCFS_BUILD_USER_XATTRS)) { + /* These conflict */ errno = EINVAL; return NULL; } @@ -621,7 +633,7 @@ struct lcfs_node_s *lcfs_load_node_from_file(int dirfd, const char *fname, } if ((buildflags & LCFS_BUILD_SKIP_XATTRS) == 0) { - r = read_xattrs(ret, dirfd, fname); + r = read_xattrs(ret, dirfd, fname, buildflags); if (r < 0) return NULL; } diff --git a/libcomposefs/lcfs-writer.h b/libcomposefs/lcfs-writer.h index 136de48b..b350b37c 100644 --- a/libcomposefs/lcfs-writer.h +++ b/libcomposefs/lcfs-writer.h @@ -35,6 +35,7 @@ enum { LCFS_BUILD_SKIP_DEVICES = (1 << 2), LCFS_BUILD_COMPUTE_DIGEST = (1 << 3), LCFS_BUILD_NO_INLINE = (1 << 4), + LCFS_BUILD_USER_XATTRS = (1 << 5), /* Only read user.* xattrs */ }; enum lcfs_format_t { From e6e5939fd37cbaa67d9a0af9aa67d242d4f22210 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 17:09:02 +0200 Subject: [PATCH 03/12] mkcomposefs: Add --user-xattrs Like --skip-xattrs, but keeps user xattrs. Signed-off-by: Alexander Larsson --- tools/mkcomposefs.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/mkcomposefs.c b/tools/mkcomposefs.c index 1e88e7ba..c6267814 100644 --- a/tools/mkcomposefs.c +++ b/tools/mkcomposefs.c @@ -369,6 +369,7 @@ static void usage(const char *argv0) #define OPT_PRINT_DIGEST 109 #define OPT_FORMAT 110 #define OPT_PRINT_DIGEST_ONLY 111 +#define OPT_USER_XATTRS 112 static ssize_t write_cb(void *_file, void *buf, size_t count) { @@ -392,6 +393,12 @@ int main(int argc, char **argv) flag: NULL, val: OPT_SKIP_XATTRS }, + { + name: "user-xattrs", + has_arg: no_argument, + flag: NULL, + val: OPT_USER_XATTRS + }, { name: "skip-devices", has_arg: no_argument, @@ -469,6 +476,9 @@ int main(int argc, char **argv) case OPT_SKIP_XATTRS: buildflags |= LCFS_BUILD_SKIP_XATTRS; break; + case OPT_USER_XATTRS: + buildflags |= LCFS_BUILD_USER_XATTRS; + break; case OPT_SKIP_DEVICES: buildflags |= LCFS_BUILD_SKIP_DEVICES; break; From 0a247df5f5e4b411cadaaecb91541547d255f20b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 11:55:23 +0200 Subject: [PATCH 04/12] composefs-fuse: Fix xattr handling We don't want to rewrite trusted.overlay to user.overlay anymore, as that was a leftover from when using the fuse fs combined with overlayfs, instead we need to support xattr escapes. Also, until fuse supports non-user xattrs, don't return them from the fuse APIs, because listing the xattr will work, but the getattr will then be filtered out and get ENODATA. Signed-off-by: Alexander Larsson --- tools/cfs-fuse.c | 71 +++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/tools/cfs-fuse.c b/tools/cfs-fuse.c index 9e87555f..0009a0d1 100644 --- a/tools/cfs-fuse.c +++ b/tools/cfs-fuse.c @@ -641,30 +641,28 @@ static void cfs_init(void *userdata, struct fuse_conn_info *conn) #define OVERLAY_XATTR_PARTIAL_PREFIX "overlay." -static int cfs_rewrite_xattr_prefix_from_image(int name_index, const char *name, - size_t name_len) +static const char *cfs_xattr_rewrite(int name_index, const char *name, + uint8_t *_name_len) { - /* We rewrite trusted.overlay.* to user.overlay.* */ - if (name_index == EROFS_XATTR_INDEX_TRUSTED && - name_len > strlen(OVERLAY_XATTR_PARTIAL_PREFIX) && - memcmp(name, OVERLAY_XATTR_PARTIAL_PREFIX, - strlen(OVERLAY_XATTR_PARTIAL_PREFIX)) == 0) - return EROFS_XATTR_INDEX_USER; - - return name_index; -} + uint8_t name_len = *_name_len; -static int cfs_rewrite_xattr_prefix_to_image(int name_index, const char *name, - size_t name_len) -{ - /* We rewrite trusted.overlay.* to user.overlay.* */ - if (name_index == EROFS_XATTR_INDEX_USER && + if (name_index == EROFS_XATTR_INDEX_TRUSTED && name_len > strlen(OVERLAY_XATTR_PARTIAL_PREFIX) && memcmp(name, OVERLAY_XATTR_PARTIAL_PREFIX, - strlen(OVERLAY_XATTR_PARTIAL_PREFIX)) == 0) - return EROFS_XATTR_INDEX_TRUSTED; + strlen(OVERLAY_XATTR_PARTIAL_PREFIX)) == 0) { + name += strlen(OVERLAY_XATTR_PARTIAL_PREFIX); + name_len -= strlen(OVERLAY_XATTR_PARTIAL_PREFIX); + /* Check for escapes */ + if (name_len > strlen(OVERLAY_XATTR_PARTIAL_PREFIX) && + memcmp(name, OVERLAY_XATTR_PARTIAL_PREFIX, + strlen(OVERLAY_XATTR_PARTIAL_PREFIX)) == 0) { + *_name_len = name_len; + return name; + } + return NULL; + } - return name_index; + return name; } static int cfs_listxattr_element(const struct erofs_xattr_entry *entry, @@ -676,7 +674,13 @@ static int cfs_listxattr_element(const struct erofs_xattr_entry *entry, size_t full_name_len; const char *prefix; - name_index = cfs_rewrite_xattr_prefix_from_image(name_index, name, name_len); + name = cfs_xattr_rewrite(name_index, name, &name_len); + if (name == NULL) + return 0; + + /* Until we support mount option to use all xattrs, only support user.* */ + if (name_index != EROFS_XATTR_INDEX_USER) + return 0; prefix = erofs_xattr_prefixes[name_index]; full_name_len = name_len + strlen(prefix); @@ -782,19 +786,22 @@ static void cfs_listxattr(fuse_req_t req, fuse_ino_t ino, size_t max_size) } } -static int match_xattr_entry(const struct erofs_xattr_entry *entry, - int name_prefix, const char *name, size_t name_len) +static int match_xattr_entry(const struct erofs_xattr_entry *entry, int name_prefix, + const char *name, size_t name_len, bool filter) { uint8_t e_name_len = entry->e_name_len; uint8_t e_name_prefix = entry->e_name_index; const char *e_name = (const char *)entry + sizeof(struct erofs_xattr_entry); - return e_name_prefix == name_prefix && e_name_len == name_len && - memcmp(name, e_name, name_len) == 0; + if (filter) + e_name = cfs_xattr_rewrite(e_name_prefix, e_name, &e_name_len); + + return e_name != NULL && e_name_prefix == name_prefix && + e_name_len == name_len && memcmp(name, e_name, name_len) == 0; } static const char *do_getxattr(const erofs_inode *cino, int name_prefix, - const char *name, uint16_t *value_size_out) + const char *name, uint16_t *value_size_out, bool filter) { size_t name_len = strlen(name); uint32_t mode; @@ -809,6 +816,10 @@ static const char *do_getxattr(const erofs_inode *cino, int name_prefix, const uint8_t *xattrs_start; const uint8_t *xattrs_end; + /* Until we support mount option to use all xattrs, only support user.* */ + if (filter && name_prefix != EROFS_XATTR_INDEX_USER) + return NULL; + erofs_inode_get_info(cino, &mode, &file_size, &xattr_icount, &raw_blkaddr, &isize); @@ -836,7 +847,7 @@ static const char *do_getxattr(const erofs_inode *cino, int name_prefix, e_name_len + value_size, 4); - if (match_xattr_entry(entry, name_prefix, name, name_len)) { + if (match_xattr_entry(entry, name_prefix, name, name_len, filter)) { const char *value = (const char *)entry + sizeof(struct erofs_xattr_entry) + e_name_len; @@ -853,7 +864,7 @@ static const char *do_getxattr(const erofs_inode *cino, int name_prefix, const struct erofs_xattr_entry *entry = (const struct erofs_xattr_entry *)(erofs_xattrdata + idx * 4); - if (match_xattr_entry(entry, name_prefix, name, name_len)) { + if (match_xattr_entry(entry, name_prefix, name, name_len, filter)) { uint16_t value_size = lcfs_u16_from_file(entry->e_value_size); uint8_t e_name_len = entry->e_name_len; @@ -887,8 +898,6 @@ static void cfs_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, name += strlen(erofs_xattr_prefixes[name_prefix]); name_len = strlen(name); - name_prefix = cfs_rewrite_xattr_prefix_to_image(name_prefix, name, name_len); - /* When acls are not used, send EOPTNOTSUPP, as this informs userspace to stop constantly looking for acls */ if (!erofs_use_acl && erofs_is_acl_xattr(name_prefix, name, name_len)) { @@ -896,7 +905,7 @@ static void cfs_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, return; } - value = do_getxattr(cino, name_prefix, name, &value_size); + value = do_getxattr(cino, name_prefix, name, &value_size, true); if (value == NULL) { fuse_reply_err(req, ENODATA); return; @@ -927,7 +936,7 @@ static void cfs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) } redirect = do_getxattr(cino, EROFS_XATTR_INDEX_TRUSTED, - "overlay.redirect", &value_size); + "overlay.redirect", &value_size, false); if (redirect == NULL) { /* Empty files have no redirect */ From 68060955abed8da8d05e0937c5eafa25ba7bdf3a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 13:40:04 +0200 Subject: [PATCH 05/12] composefs-fuse: Filter out real whiteouts This gets rid of the 00-ff files we generate Signed-off-by: Alexander Larsson --- tools/cfs-fuse.c | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/tools/cfs-fuse.c b/tools/cfs-fuse.c index 0009a0d1..ea3ae0df 100644 --- a/tools/cfs-fuse.c +++ b/tools/cfs-fuse.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "libcomposefs/lcfs-erofs-internal.h" #include "libcomposefs/lcfs-internal.h" @@ -97,6 +98,24 @@ static const erofs_inode *cfs_get_erofs_inode(fuse_ino_t ino) return (const erofs_inode *)(erofs_metadata + (nid << EROFS_ISLOTBITS)); } +static int erofs_inode_is_whiteout(const erofs_inode *cino) +{ + mode_t mode; + dev_t rdev; + if (erofs_inode_is_compact(cino)) { + const struct erofs_inode_compact *c = &cino->compact; + mode = lcfs_u16_from_file(c->i_mode); + rdev = lcfs_u32_from_file(c->i_u.rdev); + } else { + const struct erofs_inode_extended *e = &cino->extended; + mode = lcfs_u16_from_file(e->i_mode); + rdev = lcfs_u32_from_file(e->i_u.rdev); + } + + int type = mode & S_IFMT; + return (type == S_IFCHR || type == S_IFBLK) && rdev == makedev(0, 0); +} + static int cfs_stat(fuse_ino_t ino, const erofs_inode *cino, struct stat *stbuf) { stbuf->st_ino = ino; @@ -221,13 +240,17 @@ static bool cfs_lookup_block(fuse_req_t req, const uint8_t *block, const erofs_inode *child_cino = cfs_get_erofs_inode(nid); struct fuse_entry_param e; - memset(&e, 0, sizeof(e)); - e.ino = cfs_ino_from_nid(nid); - e.attr_timeout = CFS_ATTR_TIMEOUT; - e.entry_timeout = CFS_ENTRY_TIMEOUT; - cfs_stat(e.ino, child_cino, &e.attr); + if (erofs_inode_is_whiteout(child_cino)) { + fuse_reply_err(req, ENOENT); + } else { + memset(&e, 0, sizeof(e)); + e.ino = cfs_ino_from_nid(nid); + e.attr_timeout = CFS_ATTR_TIMEOUT; + e.entry_timeout = CFS_ENTRY_TIMEOUT; + cfs_stat(e.ino, child_cino, &e.attr); - fuse_reply_entry(req, &e); + fuse_reply_entry(req, &e); + } return true; } else { @@ -426,8 +449,13 @@ static bool cfs_readdir_block(fuse_req_t req, struct dirbuf *buf, name_buf[child_name_len] = 0; remaining_size = buf->max_size - buf->current_size; - if (use_plus) { - const erofs_inode *child_cino = cfs_get_erofs_inode(nid); + + const erofs_inode *child_cino = cfs_get_erofs_inode(nid); + uint8_t type = dirents[i].file_type; + if (type == EROFS_FT_CHRDEV && erofs_inode_is_whiteout(child_cino)) { + /* Filtered */ + res = 0; + } else if (use_plus) { struct fuse_entry_param e; memset(&e, 0, sizeof(e)); @@ -440,7 +468,6 @@ static bool cfs_readdir_block(fuse_req_t req, struct dirbuf *buf, req, (char *)(buf->buf + buf->current_size), remaining_size, name_buf, &e, next_offset); } else { - uint8_t type = dirents[i].file_type; memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = cfs_ino_from_nid(nid); stbuf.st_mode = erofs_file_type_to_mode(type); From 9d3ef12bfe9c75a806fbb303066081c88ff673fc Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 13:40:51 +0200 Subject: [PATCH 06/12] composefs-fuse: Fill in statbuf.st_rdev Signed-off-by: Alexander Larsson --- tools/cfs-fuse.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/cfs-fuse.c b/tools/cfs-fuse.c index ea3ae0df..a85c9e1f 100644 --- a/tools/cfs-fuse.c +++ b/tools/cfs-fuse.c @@ -131,6 +131,11 @@ static int cfs_stat(fuse_ino_t ino, const erofs_inode *cino, struct stat *stbuf) stbuf->st_mtim.tv_sec = erofs_build_time; stbuf->st_mtim.tv_nsec = erofs_build_time_nsec; + + int type = stbuf->st_mode & S_IFMT; + if (type == S_IFCHR || type == S_IFBLK) + stbuf->st_rdev = lcfs_u32_from_file(c->i_u.rdev); + } else { const struct erofs_inode_extended *e = &cino->extended; @@ -141,6 +146,10 @@ static int cfs_stat(fuse_ino_t ino, const erofs_inode *cino, struct stat *stbuf) stbuf->st_mtim.tv_sec = lcfs_u64_from_file(e->i_mtime); stbuf->st_mtim.tv_nsec = lcfs_u32_from_file(e->i_mtime); stbuf->st_nlink = lcfs_u32_from_file(e->i_nlink); + + int type = stbuf->st_mode & S_IFMT; + if (type == S_IFCHR || type == S_IFBLK) + stbuf->st_rdev = lcfs_u32_from_file(e->i_u.rdev); } return 0; From 9e3c3d35071cadd133a2584c95e97bff4445ad49 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 13:41:11 +0200 Subject: [PATCH 07/12] composefs-fuse: Fix st_mtim.tv_nsec Signed-off-by: Alexander Larsson --- tools/cfs-fuse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cfs-fuse.c b/tools/cfs-fuse.c index a85c9e1f..63c994d7 100644 --- a/tools/cfs-fuse.c +++ b/tools/cfs-fuse.c @@ -144,7 +144,7 @@ static int cfs_stat(fuse_ino_t ino, const erofs_inode *cino, struct stat *stbuf) stbuf->st_uid = lcfs_u32_from_file(e->i_uid); stbuf->st_gid = lcfs_u32_from_file(e->i_gid); stbuf->st_mtim.tv_sec = lcfs_u64_from_file(e->i_mtime); - stbuf->st_mtim.tv_nsec = lcfs_u32_from_file(e->i_mtime); + stbuf->st_mtim.tv_nsec = lcfs_u32_from_file(e->i_mtime_nsec); stbuf->st_nlink = lcfs_u32_from_file(e->i_nlink); int type = stbuf->st_mode & S_IFMT; From 1618f766cf67bbea9e513a87ec3967ee5a55222a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 13:41:38 +0200 Subject: [PATCH 08/12] composefs-fuse: Fill in atime and ctime from mtime Signed-off-by: Alexander Larsson --- tools/cfs-fuse.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/cfs-fuse.c b/tools/cfs-fuse.c index 63c994d7..5e24bfa3 100644 --- a/tools/cfs-fuse.c +++ b/tools/cfs-fuse.c @@ -152,6 +152,11 @@ static int cfs_stat(fuse_ino_t ino, const erofs_inode *cino, struct stat *stbuf) stbuf->st_rdev = lcfs_u32_from_file(e->i_u.rdev); } + stbuf->st_atim.tv_sec = stbuf->st_mtim.tv_sec; + stbuf->st_atim.tv_nsec = stbuf->st_mtim.tv_nsec; + stbuf->st_ctim.tv_sec = stbuf->st_mtim.tv_sec; + stbuf->st_ctim.tv_nsec = stbuf->st_mtim.tv_nsec; + return 0; } From 0f8eb529cfc006356e8010caa9d692509a4abee3 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 16:08:18 +0200 Subject: [PATCH 09/12] tests: Pass in valgrind prefix separately Pass in $VALGRIND_PREFIX separately rather than via $BINDIR. This means we don't have to valgrind everything in $BINDIR, which will be useful later. Signed-off-by: Alexander Larsson --- tests/Makefile.am | 4 ++-- tests/test-checksums.sh | 4 ++-- tests/test-units.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 127d39cf..46258b07 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -12,9 +12,9 @@ endif EXTRA_DIST = test-checksums.sh $(patsubst %,assets/%,${TEST_ASSETS_SMALL}) $(patsubst %,assets/%.sha256_erofs,${TEST_ASSETS_SMALL}) check-checksums: - $(srcdir)/test-checksums.sh "${VALGRIND_PREFIX} $(builddir)/../tools/" "$(srcdir)/assets" "${TEST_ASSETS}" + VALGRIND_PREFIX="${VALGRIND_PREFIX}" $(srcdir)/test-checksums.sh "$(builddir)/../tools/" "$(srcdir)/assets" "${TEST_ASSETS}" check-units: - $(srcdir)/test-units.sh "${VALGRIND_PREFIX} $(builddir)/../tools/" + VALGRIND_PREFIX="${VALGRIND_PREFIX}" $(srcdir)/test-units.sh "$(builddir)/../tools/" check: check-units check-checksums diff --git a/tests/test-checksums.sh b/tests/test-checksums.sh index 47e5a0ab..609a9c73 100755 --- a/tests/test-checksums.sh +++ b/tests/test-checksums.sh @@ -27,7 +27,7 @@ for format in erofs ; do CAT=cat fi - $CAT $ASSET_DIR/$file | ${BINDIR}/composefs-from-json --format=$format --out=$tmpfile - + $CAT $ASSET_DIR/$file | ${VALGRIND_PREFIX} ${BINDIR}/composefs-from-json --format=$format --out=$tmpfile - SHA=$(sha256sum $tmpfile | awk "{print \$1}") # Run fsck.erofs to make sure we're not generating anything weird @@ -41,7 +41,7 @@ for format in erofs ; do fi # Ensure dump reproduces the same file - ${BINDIR}/composefs-dump $tmpfile $tmpfile2 + ${VALGRIND_PREFIX} ${BINDIR}/composefs-dump $tmpfile $tmpfile2 if ! cmp $tmpfile $tmpfile2; then echo Dump is not reproducible exit 1 diff --git a/tests/test-units.sh b/tests/test-units.sh index 5372d9b1..2769ecf0 100755 --- a/tests/test-units.sh +++ b/tests/test-units.sh @@ -9,7 +9,7 @@ trap 'rm -rf -- "$workdir"' EXIT function makeimage () { local dir=$1 - $BINDIR/mkcomposefs --digest-store=$dir/objects $dir/root $dir/test.cfs + ${VALGRIND_PREFIX} $BINDIR/mkcomposefs --digest-store=$dir/objects $dir/root $dir/test.cfs } function countobjects () { From 9177ce97c10d080d13277ccd5b02ff3ba9dd1829 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 13:54:26 +0200 Subject: [PATCH 10/12] tests/dumpdir: Add various options to change what/how dumpdir works These options tweak the output of dumpdir, similar to what we expect some composefs tools to do. In particular: --userxattrs: Only dump user.* xattrs. This is what composefs-fuse will do. --noescaped: Don't dump escaped xattrs. This is what a real composefs mount will do if kernel overlayfs doesn't support xattr escapes. --whiteouts: Convert regular whiteouts to xattr-based whiteouts. This is what mkcomposefs does to whiteouts. Signed-off-by: Alexander Larsson --- tests/dumpdir | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/tests/dumpdir b/tests/dumpdir index 4ad80775..20352079 100755 --- a/tests/dumpdir +++ b/tests/dumpdir @@ -10,22 +10,58 @@ import argparse def log_error(error): print("Readdir error: " + error) +def should_convert_whiteout(stbuf): + return args.whiteout and stat.S_ISCHR(stbuf.st_mode) and stbuf.st_rdev == os.makedev(0,0) + +def has_whiteout_child(path): + for f in os.listdir(path): + s = os.lstat(os.path.join(path, f)) + if should_convert_whiteout(s): + return True + return False + def dumpfile(file, root): rel = os.path.relpath(file, root) s = os.lstat(file) + + st_mode = s.st_mode + + content = None + xattrs = {} + if should_convert_whiteout(s): + content = b'' + st_mode = (st_mode & ~stat.S_IFMT(st_mode)) | stat.S_IFREG + xattrs["trusted.overlay.overlay.whiteout"] = b'' + xattrs["user.overlay.whiteout"] = b'' + + if stat.S_ISDIR(st_mode) and has_whiteout_child(file): + xattrs["trusted.overlay.overlay.whiteouts"] = b'' + xattrs["user.overlay.whiteouts"] = b'' + nlink = s.st_nlink; if args.no_nlink: nlink = 1 - print(f"{shlex.quote(rel)} {oct(s.st_mode)} {nlink} {s.st_uid}:{s.st_gid} {s.st_rdev} {s.st_mtime_ns}",end="") - if stat.S_ISREG(s.st_mode): - digest = hashlib.sha256(open(file,'rb').read()).hexdigest() + print(f"{shlex.quote(rel)} {oct(st_mode)} {nlink} {s.st_uid}:{s.st_gid} {s.st_rdev} {s.st_mtime_ns}",end="") + if stat.S_ISREG(st_mode): + if content == None: + content = open(file,'rb').read() + digest = hashlib.sha256(content).hexdigest() print(f" {s.st_size} sha256:{digest}",end="") - elif stat.S_ISLNK(s.st_mode): + elif stat.S_ISLNK(st_mode): link = os.readlink(file) print(f" ->{shlex.quote(link)}",end="") for attr in sorted(os.listxattr(file, follow_symlinks=False)): v = os.getxattr(file, attr, follow_symlinks=False) + xattrs[attr] = v + + for attr in sorted(xattrs.keys()): + v = xattrs[attr] + if args.userxattr and not attr.startswith("user."): + continue + if args.noescaped and attr.startswith("trusted.overlay.overlay."): + continue + print(f" {attr}={v}", end="") print() @@ -40,6 +76,9 @@ def dumpdir(root): argParser = argparse.ArgumentParser() argParser.add_argument("--no-nlink", action='store_true') +argParser.add_argument("--userxattr", action='store_true') +argParser.add_argument("--whiteout", action='store_true') +argParser.add_argument("--noescaped", action='store_true') argParser.add_argument('path') args = argParser.parse_args() From a7a3dc9c4483d4b1246405141eecd9143995c461 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 19 Sep 2023 10:44:32 +0200 Subject: [PATCH 11/12] tools: Update .gitignore Signed-off-by: Alexander Larsson --- tools/.gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/.gitignore b/tools/.gitignore index 6509c89f..7ca77441 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -1,4 +1,7 @@ dump mkcomposefs -writer-json +composefs-dump +composefs-from-json +composefs-fuse +composefs-info mount.composefs From 2c365e26e3e0d30ebeb3bcea99ae7660f6a6a685 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Sep 2023 17:16:20 +0200 Subject: [PATCH 12/12] Add tests Generates 10 random images and runs some test on them. The tests run are: * Dump (parse and re-write it) image and esure we get identical results * Run fsck.erofs on image * Mount image using fuse and ensure dumpdir is the same (sans non-user-xattr and whiteout) * Run mkcomposefs on the fuse mount and ensure the result is similar. (It produces same fuse mount dump, but due to non-user xattrs and whiteouts its not producing an identical composefs image) Signed-off-by: Alexander Larsson --- .github/workflows/test.yaml | 2 + .gitignore | 1 + hacking/installdeps.sh | 2 +- tests/Makefile.am | 5 +- tests/dumpdir | 1 + tests/gendir | 195 ++++++++++++++++++++++++++++++++++++ tests/test-checksums.sh | 7 +- tests/test-lib.sh | 38 +++++++ tests/test-random-fuse.sh | 87 ++++++++++++++++ 9 files changed, 332 insertions(+), 6 deletions(-) create mode 100755 tests/gendir create mode 100644 tests/test-lib.sh create mode 100755 tests/test-random-fuse.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 96aa5831..d1f4ada9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,8 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies run: sudo ./hacking/installdeps.sh + - name: Install fsck.erofs + run: sudo apt install erofs-utils - name: Configure run: ./autogen.sh && ./configure --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH) CFLAGS='-Wall -Werror' - name: Build diff --git a/.gitignore b/.gitignore index 97a7132d..9252feda 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /Makefile libcomposefs/Makefile tools/Makefile +tests/Makefile Makefile.in aclocal.m4 autom4te.cache diff --git a/hacking/installdeps.sh b/hacking/installdeps.sh index d4c70b87..367f7227 100755 --- a/hacking/installdeps.sh +++ b/hacking/installdeps.sh @@ -1,4 +1,4 @@ #!/bin/bash set -xeuo pipefail export DEBIAN_FRONTEND=noninteractive -apt-get install -y automake libtool autoconf autotools-dev git make gcc libyajl-dev libssl-dev libfsverity-dev pkg-config libfuse3-dev +apt-get install -y automake libtool autoconf autotools-dev git make gcc libyajl-dev libssl-dev libfsverity-dev pkg-config libfuse3-dev python3 libcap2-bin diff --git a/tests/Makefile.am b/tests/Makefile.am index 46258b07..070152e7 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,4 +17,7 @@ check-checksums: check-units: VALGRIND_PREFIX="${VALGRIND_PREFIX}" $(srcdir)/test-units.sh "$(builddir)/../tools/" -check: check-units check-checksums +check-random-fuse: + VALGRIND_PREFIX="${VALGRIND_PREFIX}" $(srcdir)/test-random-fuse.sh "$(builddir)/../tools/" + +check: check-units check-checksums check-random-fuse diff --git a/tests/dumpdir b/tests/dumpdir index 20352079..6aff0a02 100755 --- a/tests/dumpdir +++ b/tests/dumpdir @@ -71,6 +71,7 @@ def dumpfile(file, root): def dumpdir(root): dumpfile(root, root) for parent, dirs, files in os.walk(root, topdown=True, onerror=log_error): + dirs.sort() for file in sorted(dirs + files): dumpfile(os.path.join(parent, file), root) diff --git a/tests/gendir b/tests/gendir new file mode 100755 index 00000000..78a532a5 --- /dev/null +++ b/tests/gendir @@ -0,0 +1,195 @@ +#!/usr/bin/python3 + +import argparse +import hashlib +import os +import random +import shlex +import shutil +import stat +import string +import sys + +adjectives = ["adorable", "adventurous", "aggressive", "agreeable", "alert", "alive", "amused", "angry", "annoyed", "annoying", "anxious", "arrogant", "ashamed", "attractive", "average", "awful", "bad", "beautiful", "better", "bewildered", "black", "bloody", "blue", "blue-eyed", "blushing", "bored", "brainy", "brave", "breakable", "bright", "busy", "calm", "careful", "cautious", "charming", "cheerful", "clean", "clear", "clever", "cloudy", "clumsy", "colorful", "combative", "comfortable", "concerned", "condemned", "confused", "cooperative", "courageous", "crazy", "creepy", "crowded", "cruel", "curious", "cute", "dangerous", "dark", "dead", "defeated", "defiant", "delightful", "depressed", "determined", "different", "difficult", "disgusted", "distinct", "disturbed", "dizzy", "doubtful", "drab", "dull", "eager", "easy", "elated", "elegant", "embarrassed", "enchanting", "encouraging", "energetic", "enthusiastic", "envious", "evil", "excited", "expensive", "exuberant", "fair", "faithful", "famous", "fancy", "fantastic", "fierce", "filthy", "fine", "foolish", "fragile", "frail", "frantic", "friendly", "frightened", "funny", "gentle", "gifted", "glamorous", "gleaming", "glorious", "good", "gorgeous", "graceful", "grieving", "grotesque", "grumpy", "handsome", "happy", "healthy", "helpful", "helpless", "hilarious", "homeless", "homely", "horrible", "hungry", "hurt", "ill", "important", "impossible", "inexpensive", "innocent", "inquisitive", "itchy", "jealous", "jittery", "jolly", "joyous", "kind", "lazy", "light", "lively", "lonely", "long", "lovely", "lucky", "magnificent", "misty", "modern", "motionless", "muddy", "mushy", "mysterious", "nasty", "naughty", "nervous", "nice", "nutty", "obedient", "obnoxious", "odd", "old-fashioned", "open", "outrageous", "outstanding", "panicky", "perfect", "plain", "pleasant", "poised", "poor", "powerful", "precious", "prickly", "proud", "putrid", "puzzled", "quaint", "real", "relieved", "repulsive", "rich", "scary", "selfish", "shiny", "shy", "silly", "sleepy", "smiling", "smoggy", "sore", "sparkling", "splendid", "spotless", "stormy", "strange", "stupid", "successful", "super", "talented", "tame", "tasty", "tender", "tense", "terrible", "thankful", "thoughtful", "thoughtless", "tired", "tough", "troubled", "ugliest", "ugly", "uninterested", "unsightly", "unusual", "upset", "uptight", "vast", "victorious", "vivacious", "wandering", "weary", "wicked", "wide-eyed", "wild", "witty", "worried", "worrisome", "wrong", "zany", "zealous"] + +nouns = ["apple", "air", "conditioner", "airport", "ambulance", "aircraft", "apartment", "arrow", "antlers", "apro", "alligator", "architect", "ankle", "armchair", "aunt", "ball", "bermudas", "beans", "balloon", "bear", "blouse", "bed", "bow", "bread", "black", "board", "bones", "bill", "bitterness", "boxers", "belt", "brain", "buffalo", "bird", "baby", "book", "back", "butter", "bulb", "buckles", "bat", "bank", "bag", "bra", "boots", "blazer", "bikini", "bookcase", "bookstore", "bus", "stop", "brass", "brother", "boy", "blender", "bucket", "bakery", "bow", "bridge", "boat", "car", "cow", "cap", "cooker", "cheeks", "cheese", "credenza", "carpet", "crow", "crest", "chest", "chair", "candy", "cabinet", "cat", "coffee", "children", "cookware", "chaise", "longue", "chicken", "casino", "cabin", "castle", "church", "cafe", "cinema", "choker", "cravat", "cane", "costume", "cardigan", "chocolate", "crib", "couch", "cello", "cashier", "composer", "cave", "country", "computer", "canoe", "clock", "charlie", "dog", "deer", "donkey", "desk", "desktop", "dress", "dolphin", "doctor", "dentist", "drum", "dresser", "designer", "detective", "daughter", "egg", "elephant", "earrings", "ears", "eyes", "estate", "finger", "fox", "frock", "frog", "fan", "freezer", "fish", "film", "foot", "flag", "factory", "father", "farm", "forest", "flower", "fruit", "fork", "grapes", "goat", "gown", "garlic", "ginger", "giraffe", "gauva", "grains", "gas", "station", "garage", "gloves", "glasses", "gift", "galaxy", "guitar", "grandmother", "grandfather", "governor", "girl", "guest", "hamburger", "hand", "head", "hair", "heart", "house", "horse", "hen", "horn", "hat", "hammer", "hostel", "hospital", "hotel", "heels", "herbs", "host", "jacket", "jersey", "jewelry", "jaw", "jumper", "judge", "juicer", "keyboard", "kid", "kangaroo", "koala", "knife", "lemon", "lion", "leggings", "leg", "laptop", "library", "lamb", "london", "lips", "lung", "lighter", "luggage", "lamp", "lawyer", "mouse", "monkey", "mouth", "mango", "mobile", "milk", "music", "mirror", "musician", "mother", "man", "model", "mall", "museum", "market", "moonlight", "medicine", "microscope", "newspaper", "nose", "notebook", "neck", "noodles", "nurse", "necklace", "noise", "ocean", "ostrich", "oil", "orange", "onion", "oven", "owl", "paper", "panda", "pants", "palm", "pasta", "pumpkin", "pharmacist", "potato", "parfume", "panther", "pad", "pencil", "pipe", "police", "pen", "pharmacy", "petrol", "station", "police", "station", "parrot", "plane", "pigeon", "phone", "peacock", "pencil", "pig", "pouch", "pagoda", "pyramid", "purse", "pancake", "popcorn", "piano", "physician", "photographer", "professor", "painter", "park", "plant", "parfume", "radio", "razor", "ribs", "rainbow", "ring", "rabbit", "rice", "refrigerator", "remote", "restaurant", "road", "surgeon", "scale", "shampoo", "sink", "salt", "shark", "sandals", "shoulder", "spoon", "soap", "sand", "sheep", "sari", "stomach", "stairs", "soup", "shoes", "scissors", "sparrow", "shirt", "suitcase", "stove", "stairs", "snowman", "shower", "swan", "suit", "sweater", "smoke", "skirt", "sofa", "socks", "stadium", "skyscraper", "school", "sunglasses", "sandals", "slippers", "shorts", "sandwich", "strawberry", "spaghetti", "shrimp", "saxophone", "sister", "son", "singer", "senator", "street", "supermarket", "swimming", "pool", "star", "sky", "sun", "spoon", "ship", "smile", "table", "turkey", "tie", "toes", "truck", "train", "taxi", "tiger", "trousers", "tongue", "television", "teacher", "turtle", "tablet", "train", "station", "toothpaste", "tail", "theater", "trench", "coat", "tea", "tomato", "teen", "tunnel", "temple", "town", "toothbrush", "tree", "toy", "tissue", "telephone", "underwear", "uncle", "umbrella", "vest", "voice", "veterinarian", "villa", "violin", "village", "vehicle", "vase", "wallet", "wolf", "waist", "wrist", "water", "melon", "whale", "water", "wings", "whisker", "watch", "woman", "washing", "machine", "wheelchair", "waiter", "wound", "xylophone", "zebra", "zoo"] + +def with_chance(chance): + return random.random() <= chance + +class Chance(): + def __init__(self): + self.value = random.random() + self.start = 0 + + def with_chance(self, chance): + if self.start > 1: + print("Too many choices") + start = self.start + end = self.start + chance + self.start = end + return self.value >= start and self.value < end + + # Choose one of weighted options + def choice(self, options): + for value, chance in options: + if self.with_chance(chance): + return value + # Default to first + value, chance = options[0] + return value + +def gen_dir_mode(): + # For creation to work we want all dirs u+rwx + return random.choice([0o777, 0o755, 0o750, 0o700]) + +def gen_file_mode(): + return random.choice([0o644, 0o666, 0o755, 0o777]) + +def gen_filename(): + if not args.unreadable: + name = bytes(random.choice(adjectives) + "_" + random.choice(nouns) + str(random.randint(1,999)), "utf-8") + if len(name) > 255: + return gen_filename() + return name + + name_len = random.randrange(1, 255) + name = [0] * name_len + for i in range(name_len): + c = random.randrange(1, 255) + while c == ord('/'): + c = random.randrange(1, 255) + name[i] = c + name=bytes(name) + if name == b'.' or name == b'..': + return gen_filename() + return name + +def gen_filenames(): + c = Chance() + # 5% of dirs are huge + if c.with_chance(0.05): + num_files = random.randrange(0, 4096) + else: + num_files = random.randrange(0, 25) + + files = [] + for i in range(num_files): + files.append(gen_filename()) + + return list(sorted(set(files))) + +def gen_xattrname(): + return random.choice(nouns) + str(random.randint(1,9)) + +def gen_xattrdata(): + return bytes(random.choice(adjectives) + str(random.randint(1,9)), "utf-8") + + +def gen_hierarchy(root): + num_dirs = random.randrange(30, 50) + dirs = [] + for i in range(num_dirs): + parent = random.choice([root] * 3 + dirs); + p = os.path.join(parent, gen_filename()) + dirs.append(p) + # Sort and drop any (unlikely) duplicateds + return list(sorted(set(dirs))) + +def set_user_xattr(path): + n_xattrs = random.randrange(0, 3) + for i in range(n_xattrs): + name = "user." + gen_xattrname() + value = gen_xattrdata() + os.setxattr(path, name, value, follow_symlinks=False) + +old_files = [] +def make_regular_file(path): + with os.fdopen(os.open(path, os.O_WRONLY|os.O_CREAT, gen_file_mode()), 'wb') as fd: + c = Chance(); + # 5% of reuse old file data + if len(old_files) > 0 and c.with_chance(0.05): + reused = random.choice(old_files) + with os.fdopen(os.open(reused, os.O_RDONLY), 'rb') as src: + shutil.copyfileobj(src, fd) + return + + # 5% of files are large + if c.with_chance(0.05): + size = random.randrange(0, 4*1024*1024) + else: # Rest are small + size = random.randrange(0, 256) + + data = random.randbytes(size) + fd.write(data) + # Save path for reuse + old_files.append(path) + + set_user_xattr(path) + +def make_symlink(path): + target = gen_filename() + os.symlink(target, path) + +def make_node(path): + if not args.privileged: + return + target = gen_filename() + os.mknod(path, gen_file_mode() | random.choice([stat.S_IFCHR,stat.S_IFBLK]), os.makedev(0,0)) + +def make_whiteout(path): + if args.nowhiteout: + return + target = gen_filename() + os.mknod(path, gen_file_mode() | stat.S_IFCHR, device=os.makedev(0,0)) + +def make_fifo(path): + target = gen_filename() + os.mknod(path, gen_file_mode() | stat.S_IFIFO) + +def make_file(path): + c = Chance(); + f = c.choice([ + (make_regular_file, 0.7), + (make_symlink, 0.15), + (make_fifo, 0.05), + (make_node, 0.05), + (make_whiteout, 0.05) + ]) + f(path) + +def make_dir(path, dirs): + os.mkdir(path, mode=gen_dir_mode()) + set_user_xattr(path) + files = gen_filenames() + for f in files: + child_path = os.path.join(path, f) + if child_path in dirs: + continue + + func = random.choice([make_file]) + func(child_path) + +argParser = argparse.ArgumentParser() +argParser.add_argument("--seed") +argParser.add_argument("--unreadable", action='store_true') +argParser.add_argument("--privileged", action='store_true') +argParser.add_argument("--nowhiteout", action='store_true') +argParser.add_argument('path') + +args = argParser.parse_args() + +if args.seed: + seed = args.seed +else: + seed = os.urandom(16).hex() +random.seed(seed) +print(f"Using seed '{seed}'") + +# Generate tree structure +root = bytes(args.path,"utf-8") +dirs = gen_hierarchy(root) + +make_dir(root, dirs) +for d in dirs: + make_dir(d, dirs) diff --git a/tests/test-checksums.sh b/tests/test-checksums.sh index 609a9c73..c9acde55 100755 --- a/tests/test-checksums.sh +++ b/tests/test-checksums.sh @@ -4,10 +4,9 @@ BINDIR="$1" ASSET_DIR="$2" TEST_ASSETS="$3" -has_fsck=n -if which fsck.erofs &>/dev/null; then - has_fsck=y -fi +. test-lib.sh + +has_fsck=$(check_erofs_fsck) set -e tmpfile=$(mktemp /tmp/lcfs-test.XXXXXX) diff --git a/tests/test-lib.sh b/tests/test-lib.sh new file mode 100644 index 00000000..a373f666 --- /dev/null +++ b/tests/test-lib.sh @@ -0,0 +1,38 @@ +#!/usr/bin/bash + +check_whiteout () { + tmpfile=$(mktemp /tmp/lcfs-whiteout.XXXXXX) + rm -f $tmpfile + if mknod $tmpfile c 0 0 &> /dev/null; then + echo y + else + echo n + fi + rm -f $tmpfile +} + +check_fuse () { + fusermount --version >/dev/null 2>&1 || return 1 + + capsh --print | grep -q 'Bounding set.*[^a-z]cap_sys_admin' || \ + return 1 + + [ -w /dev/fuse ] || return 1 + [ -e /etc/mtab ] || return 1 + + return 0 +} + +check_erofs_fsck () { + if which fsck.erofs &>/dev/null; then + echo y + else + echo n + fi +} + +[[ -v can_whiteout ]] || can_whiteout=$(check_whiteout) +[[ -v has_fuse ]] || has_fuse=$(if check_fuse; then echo y; else echo n; fi) +[[ -v has_fsck ]] || has_fsck=$(check_erofs_fsck) + +echo Test options: can_whiteout=$can_whiteout has_fuse=$has_fuse has_fsck=$has_fsck diff --git a/tests/test-random-fuse.sh b/tests/test-random-fuse.sh new file mode 100755 index 00000000..6c93dda5 --- /dev/null +++ b/tests/test-random-fuse.sh @@ -0,0 +1,87 @@ +#!/usr/bin/bash + +BINDIR="$1" + +set -e + +workdir=$(mktemp -d /var/tmp/lcfs-test.XXXXXX) +exit_cleanup() { + umount "$workdir/mnt" &> /dev/null || true + rm -rf -- "$workdir" +} + +trap exit_cleanup EXIT + +. test-lib.sh + +GENDIRARGS="" +if [ ${can_whiteout} == "n" ]; then + GENDIRARGS="$GENDIRARGS --nowhiteout" +fi + +if [[ -v seed ]]; then + GENDIRARGS="$GENDIRARGS --seed=$seed" +fi + +test_random() { + echo Generating root dir + ./gendir $GENDIRARGS $workdir/root + ./dumpdir --userxattr --whiteout $workdir/root > $workdir/root.dump + echo Generating composefs image + ${VALGRIND_PREFIX} ${BINDIR}/mkcomposefs --digest-store=$workdir/objects $workdir/root $workdir/root.cfs + if [ $has_fsck == y ]; then + fsck.erofs $workdir/root.cfs + fi + + # Loading and dumping should produce the identical results + echo Dumping composefs image + ${VALGRIND_PREFIX} ${BINDIR}/composefs-dump $workdir/root.cfs $workdir/dump.cfs + if ! cmp $workdir/root.cfs $workdir/dump.cfs; then + echo Dump is not reproducible + diff -u <(${BINDIR}/composefs-info dump $workdir/root.cfs) <(${BINDIR}/composefs-info dump $workdir/dump.cfs) + exit 1 + fi + + if [ $has_fuse == 'n' ]; then + return; + fi + + mkdir -p $workdir/mnt + echo Mounting composefs image using fuse + ${BINDIR}/composefs-fuse -o source=$workdir/root.cfs,basedir=$workdir/objects $workdir/mnt + ./dumpdir --userxattr --whiteout $workdir/mnt > $workdir/fuse.dump + + ${VALGRIND_PREFIX} ${BINDIR}/mkcomposefs --digest-store=$workdir/objects $workdir/mnt $workdir/fuse.cfs + if [ $has_fsck == y ]; then + fsck.erofs $workdir/fuse.cfs + fi + + umount $workdir/mnt + + if ! cmp $workdir/root.dump $workdir/fuse.dump; then + echo Real dir and fuse dump differ + diff -u $workdir/root.dump $workdir/fuse.dump + exit 1 + fi + + ${BINDIR}/composefs-fuse -o source=$workdir/fuse.cfs,basedir=$workdir/objects $workdir/mnt + ./dumpdir --userxattr --whiteout $workdir/mnt > $workdir/fuse2.dump + umount $workdir/mnt + + # fuse.cfs and fuse2.cfs files differ due to whiteout conversions and non-user xattrs. + # However, the listed output should be the same: + if ! cmp $workdir/fuse.dump $workdir/fuse2.dump; then + echo Fuse and fuse2 dump differ + diff -u $workdir/fuse.dump $workdir/fuse2.dump + exit 1 + fi +} + +if [[ -v seed ]]; then + test_random +else + for i in $(seq 10) ; do + test_random + rm -rf $workdir/* + done +fi