Add mkcomposefs --from-file

Makefile.am

@@ -2,9 +2,14 @@ SUBDIRS=libcomposefs tools tests
 
 CLEANFILES=
 
-MANPAGES=\
+MAN1PAGES=\
 	man/mount.composefs.md \
-	man/mkcomposefs.md
+	man/mkcomposefs.md \
+	man/composefs-info.md
+MAN5PAGES=\
+	man/composefs-dump.md
+
+MANPAGES=${MAN1PAGES} ${MAN5PAGES}
 
 EXTRA_DIST=\
 	composefs.pc.in \
@@ -20,8 +25,13 @@ man/%.1: man/%.md
 	mkdir -p man
 	${PANDOC} $+ -s -t man > $@
 
-man1_MANS = $(MANPAGES:.md=.1)
+man/%.5: man/%.md
+	mkdir -p man
+	${PANDOC} $+ -s -t man > $@
+
+man1_MANS = $(MAN1PAGES:.md=.1)
+man5_MANS = $(MAN5PAGES:.md=.5)
 
-CLEANFILES += ${man1_MANS}
+CLEANFILES += ${man1_MANS} ${man5_MANS}
 
 endif

libcomposefs/lcfs-mount.c

@@ -170,44 +170,6 @@ static char *escape_mount_option(const char *str)
 	return res;
 }
 
-static int hexdigit(char c)
-{
-	if (c >= '0' && c <= '9')
-		return c - '0';
-	if (c >= 'a' && c <= 'f')
-		return 10 + (c - 'a');
-	if (c >= 'A' && c <= 'F')
-		return 10 + (c - 'A');
-	return -1;
-}
-
-static int digest_to_raw(const char *digest, uint8_t *raw, int max_size)
-{
-	int size = 0;
-
-	while (*digest) {
-		char c1, c2;
-		int n1, n2;
-
-		if (size >= max_size)
-			return -1;
-
-		c1 = *digest++;
-		n1 = hexdigit(c1);
-		if (n1 < 0)
-			return -1;
-
-		c2 = *digest++;
-		n2 = hexdigit(c2);
-		if (n2 < 0)
-			return -1;
-
-		raw[size++] = (n1 & 0xf) << 4 | (n2 & 0xf);
-	}
-
-	return size;
-}
-
 static int lcfs_validate_mount_options(struct lcfs_mount_state_s *state)
 {
 	struct lcfs_mount_options_s *options = state->options;

libcomposefs/lcfs-utils.h

@@ -42,6 +42,44 @@ static inline char *memdup(const char *s, size_t len)
 	return s2;
 }
 
+static inline int hexdigit(char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	if (c >= 'a' && c <= 'f')
+		return 10 + (c - 'a');
+	if (c >= 'A' && c <= 'F')
+		return 10 + (c - 'A');
+	return -1;
+}
+
+static inline int digest_to_raw(const char *digest, uint8_t *raw, int max_size)
+{
+	int size = 0;
+
+	while (*digest) {
+		char c1, c2;
+		int n1, n2;
+
+		if (size >= max_size)
+			return -1;
+
+		c1 = *digest++;
+		n1 = hexdigit(c1);
+		if (n1 < 0)
+			return -1;
+
+		c2 = *digest++;
+		n2 = hexdigit(c2);
+		if (n2 < 0)
+			return -1;
+
+		raw[size++] = (n1 & 0xf) << 4 | (n2 & 0xf);
+	}
+
+	return size;
+}
+
 static inline char *str_join(const char *a, const char *b)
 {
 	size_t a_len = strlen(a);

libcomposefs/lcfs-writer.c

@@ -243,7 +243,8 @@ int lcfs_compute_tree(struct lcfs_ctx_s *ctx, struct lcfs_node_s *root)
 	for (node = root; node != NULL; node = node->next) {
 		for (size_t i = 0; i < node->children_size; i++) {
 			struct lcfs_node_s *child = node->children[i];
-			if (child->link_to != NULL && !child->link_to->in_tree) {
+			struct lcfs_node_s *link_to = follow_links(child);
+			if (child->link_to != NULL && !link_to->in_tree) {
 				/* Link to inode outside tree */
 				errno = EINVAL;
 				return -1;
@@ -720,10 +721,13 @@ struct lcfs_node_s *lcfs_load_node_from_fd(int fd)
 
 int lcfs_node_set_payload(struct lcfs_node_s *node, const char *payload)
 {
-	char *dup = strdup(payload);
-	if (dup == NULL) {
-		errno = ENOMEM;
-		return -1;
+	char *dup = NULL;
+	if (payload) {
+		dup = strdup(payload);
+		if (dup == NULL) {
+			errno = ENOMEM;
+			return -1;
+		}
 	}
 	free(node->payload);
 	node->payload = dup;

man/.gitignore

@@ -1 +1,2 @@
 *.1
+*.5

man/composefs-dump.md

@@ -0,0 +1,121 @@
+% composefs-dump(5) composefs | User Commands
+
+# NAME
+
+composefs-dump - textual file format for composefs content
+
+# DESCRIPTION
+
+Both the *composefs-info* and the *mkcompose* commands support
+generation/consumptions of a textual descriptions of the contents of a
+composefs image. This can be used to inspect or modify an image, or to
+generate an image without having to have a local directory with the
+files in it.
+
+The file format is very simple, with one file per line, first with a
+11 fixed fields, followed by a variable number of extended attributes
+for the file.
+
+Fields are separated by a single space, and lines by a single
+newline. Extended attributes further use '=' to separate key from
+value. Therefore all these characters, as well as non-printable
+characters are escaped in the fields ('=' only in xattr fields).
+Also, back-slashes have to be escaped as they are used as the
+escape mechanism.
+
+Escapes are of the form \xXY which escapes a single byte using two hex
+digits. For example \x00 is the zero byte and \xff is the 255 byte.
+Optionally, these custom escapes are suppored:
+
+ **\\\\**
+ :    backslash.
+
+ **\\n**
+ :    newline.
+
+ **\\r**
+ :    carriage return.
+
+ **\\t**
+ :    tab
+
+ **\\=**
+ :    equal
+
+Optional fields tha are not set contain '-', and if a field actually
+has that particular value it is escaped.
+
+The fixed fields on a line are (all numbers in base 10 unless
+otherwise specified):
+
+**PATH**
+:   The full, absolute path of the file in the image. Any directories
+    used as prefix in the path must have been in the file before this
+    line.
+
+**SIZE**
+:   The size of the file. This is ignored for directories.
+
+**MODE**
+:    The st_mode stat field the file in octal, which includes both the
+     permissions and the file type.
+
+     Additionally, if the file is a hardlink, then this field will
+     start with a single '@' character, and the payload field points
+     to the target file. Note that all other fields are typically
+     filled out for a hardlink as the target, but for generation
+     of a new file we ignore all the fields except the payload.
+
+**NLINK**
+:    The st_nlink stat field.
+
+**UID**
+:    The owner uid.
+
+**GID**
+:    The owner gid.
+
+**RDEV**
+:    The st_rdev stat field.
+
+**MTIME**
+:    The modification time in seconds and nanoseconds since the unix
+     epoch, separated by '.'. Note this is not a float, "1.1" means
+     one second and one nanosecond.
+
+**PAYLOAD**
+:   The payload of the file. For symbolic links this means the symlink
+    targets. For regular files this is the relative pathname for the
+    backing files. For hardlinks (see **MODE**), this is the path of
+    another file in this file that this is a hardlink of.
+
+**CONTENT**
+:   Small files can inline the actual content in the composefs
+    image. This contains an escaped version of the content.
+    This must match the size specified in **SIZE**
+
+**DIGEST**
+:   A fs-verity digest for the file (only used for regular files, and
+    not if *CONTENT* is set) that will be validated against backing
+    files when used.
+
+After the fixed fields comes the xattrs, escaped and space-separated in the form
+**KEY**=**VALUE**. Note that '=' must be escaped in **KEY**.
+
+
+# EXAMPLE
+
+```
+/ 4096 40755 4 1000 1000 0 1695372970.944925700 - - - security.selinux=unconfined_u:object_r:unlabeled_t:s0\x00
+/a\x20dir\x20w\x20space 27 40755 2 1000 1000 0 1694598852.869646118 - - - security.selinux=unconfined_u:object_r:unlabeled_t:s0\x00
+/a-dir 45 40755 2 1000 1000 0 1674041780.601887980 - - - security.selinux=unconfined_u:object_r:unlabeled_t:s0\x00
+/a-dir/a-file 259 100644 1 1000 1000 0 1695368732.385062094 35/d02f81325122d77ec1d11baba655bc9bf8a891ab26119a41c50fa03ddfb408 - 35d02f81325122d77ec1d11baba655bc9bf8a891ab26119a41c50fa03ddfb408 security.selinux=unconfined_u:object_r:unlabeled_t:s0\x00
+/a-hardlink 259 @100644 1 1000 1000 0 1695368732.385062094 /a-dir/a-file - 35d02f81325122d77ec1d11baba655bc9bf8a891ab26119a41c50fa03ddfb408 security.selinux=unconfined_u:object_r:unlabeled_t:s0\x00
+/inline.txt 10 100644 1 1000 1000 0 1697019909.446146440 - some-text\n - security.selinux=unconfined_u:object_r:unlabeled_t:s0\x00
+```
+
+# SEE ALSO
+
+**composefs-info(1)**, **mkcomposefs(1)**
+
+[composefs upstream](https://github.com/containers/composefs)

man/composefs-info.md

@@ -0,0 +1,49 @@
+% composefs-info(1) composefs | User Commands
+
+# NAME
+
+composefs-info - print information about a composefs image
+
+# SYNOPSIS
+**composefs-info** [ls|objects|missing-objects|dump] *IMAGE* [*IMAGE2* *IMAGE3* ...]
+
+# DESCRIPTION
+
+The composefs-info command lets you inspect a composefs image. It has
+several sub-commands:
+
+**ls**
+:   Prints a simple list of the files and directorie in the images as
+    well as their backing file or symlink target.
+
+**objects**
+:   Prints a list of all the backing files referenced by the images,
+    in sorted order.
+
+**missing-objects**
+:   Prints a list of all the missing backing files referenced by the
+    images, in sorted order, given a backing file store passed in
+    using the --basedir option.
+
+**dump**
+:   Prints a full dump of the images in a line based textual format.
+    See **composefs-dump(5)** for more details. This format is also
+    accepted as input to mkcomposefs if the --from-file
+    option is used.
+
+# OPTIONS
+
+The provided *IMAGE* argument must be a composefs file. Multiple images
+can be specified.
+
+**compoosefs-info** accepts the following options:
+
+
+**\-\-basedir**=*PATH*
+:   This should point to a directory of backing files, and will be used
+    by the **missing-objects** command to know what files are available.
+
+# SEE ALSO
+**composefs-info(1)**, **composefs-dump(5)**
+
+[composefs upstream](https://github.com/containers/composefs)

man/mkcomposefs.md

@@ -5,7 +5,7 @@
 mkcomposefs - create a composefs filesystem image
 
 # SYNOPSIS
-**mkcomposefs** *SOURCEDIR* *IMAGE*
+**mkcomposefs** *SOURCE* *IMAGE*
 
 # DESCRIPTION
 
@@ -14,8 +14,9 @@ or more separate directories containing content-addressed backing data
 for regular files.
 
 **mkcomposefs** constructs the mountable "composefs image" using the
-source directory as input. It can also create the backing store
-directory.
+source as input. It can also create the backing store directory.
+Typically the source is a directory, but with *--from-file* it can
+also be a file.
 
 # OPTIONS
 
@@ -55,7 +56,12 @@ will be a mountable composefs image.
 **\-\-user-xattrs**
 :   Only add xattrs with the "user." prefix to files in the image.
 
+**\-\-from-file**
+:   The source is a file in the **composefs-dump(5)** format. If
+    the specified file is "-", the data is read from stdin.
+
 
 # SEE ALSO
+**composefs-info(1)**, **mount.composefs(1)**, **composefs-dump(5)**
 
-- [composefs upstream](https://github.com/containers/composefs)
+[composefs upstream](https://github.com/containers/composefs)

man/mount.composefs.md

@@ -62,5 +62,6 @@ options when passed via the `-o OPTIONS` argument.
 :  Specifies an overlayfs workdir to go with **upperdir**.
 
 # SEE ALSO
+**composefs-info(1)**, **mount.composefs(1)**
 
-- [composefs upstream](https://github.com/containers/composefs)
+[composefs upstream](https://github.com/containers/composefs)

tests/test-checksums.sh

@@ -45,5 +45,11 @@ for format in erofs ; do
             echo Dump is not reproducible
             exit 1
         fi
+
+        ${VALGRIND_PREFIX} ${BINDIR}/composefs-info dump $tmpfile | ${VALGRIND_PREFIX} ${BINDIR}/mkcomposefs --from-file - $tmpfile2
+        if ! cmp $tmpfile $tmpfile2; then
+            echo Dump is not reproducible via composefs-info dump
+            exit 1
+        fi
     done
 done

tests/test-random-fuse.sh

@@ -42,6 +42,15 @@ test_random() {
         exit 1
     fi
 
+    # dump + mkcomposefs should produce the identical results
+    echo Dumping composefs image
+    ${VALGRIND_PREFIX} ${BINDIR}/composefs-info dump $workdir/root.cfs | ${VALGRIND_PREFIX} ${BINDIR}/mkcomposefs --from-file - $workdir/dump.cfs
+    if ! cmp $workdir/root.cfs $workdir/dump.cfs; then
+        echo Dump + mkcomposefs is not reproducible
+        diff -u <(${BINDIR}/composefs-info dump $workdir/root.cfs) <(${BINDIR}/composefs-info dump $workdir/dump.cfs)
+        exit 1
+    fi
+
     if [ $has_fuse == 'n' ]; then
         return;
     fi