WIP: Compute an uncompressed digest for chunked layers

docs/containers-storage.conf.5.md

@@ -124,6 +124,27 @@ The `storage.options.pull_options` table supports the following keys:
   It is an expensive operation so it is not enabled by default.
   This is a "string bool": "false"|"true" (cannot be native TOML boolean)
 
+**insecure_allow_unpredictable_image_contents="false"|"true"**
+  This should _almost never_ be set.
+  It allows partial pulls ofof images without guaranteeing that "partial
+  pulls" and non-partial pulls both result in consistent image contents.
+  This allows pulling estargz images and early versions of zstd:chunked images;
+  otherwise, these layers always use the traditional non-partial pull path.
+
+  This option should be enabled _extremely_ rarely, only if _all_ images that could
+  EVER be concievably pulled on this system are _guaranteed_ (e.g. using a signature policy)
+  to come from a build system trusted to never attack image integrity.
+
+  If this consistency enforcement were disabled, malicious images could be built
+  in a way designed to evade other audit mechanisms, so presence of most other audit
+  mechanisms is not a replacement for the above-mentioned need for all images to come
+  from a trusted build system.
+
+  As a side effect, enabling this option will also make image IDs unpredictable
+  (usually not equal to the traditional value matching the config digest).
+
+  This is a "string bool": "false"|"true" (cannot be native TOML boolean)
+
 ### STORAGE OPTIONS FOR AUFS TABLE
 
 The `storage.options.aufs` table supports the following options:

drivers/driver.go

@@ -231,8 +231,8 @@ const (
 	// DifferOutputFormatDir means the output is a directory and it will
 	// keep the original layout.
 	DifferOutputFormatDir = iota
-	// DifferOutputFormatFlat will store the files by their checksum, in the form
-	// checksum[0:2]/checksum[2:]
+	// DifferOutputFormatFlat will store the files by their checksum, per
+	// pkg/chunked/internal/composefs.RegularFilePathForValidatedDigest.
 	DifferOutputFormatFlat
 )
 

pkg/chunked/cache_linux.go

@@ -710,7 +710,7 @@ func prepareCacheFile(manifest []byte, format graphdriver.DifferOutputFormat) ([
 	switch format {
 	case graphdriver.DifferOutputFormatDir:
 	case graphdriver.DifferOutputFormatFlat:
-		entries, err = makeEntriesFlat(entries)
+		entries, err = makeEntriesFlat(entries, nil)
 		if err != nil {
 			return nil, err
 		}

pkg/chunked/dump/dump.go

@@ -9,11 +9,11 @@ import (
 	"io"
 	"path/filepath"
 	"reflect"
-	"strings"
 	"time"
 
 	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	storagePath "github.com/containers/storage/pkg/chunked/internal/path"
+	"github.com/opencontainers/go-digest"
 	"golang.org/x/sys/unix"
 )
 
@@ -165,11 +165,16 @@ func dumpNode(out io.Writer, added map[string]*minimal.FileMetadata, links map[s
 		} else {
 			payload = storagePath.CleanAbsPath(entry.Linkname)
 		}
-	} else {
-		if len(entry.Digest) > 10 {
-			d := strings.Replace(entry.Digest, "sha256:", "", 1)
-			payload = d[:2] + "/" + d[2:]
+	} else if entry.Digest != "" {
+		d, err := digest.Parse(entry.Digest)
+		if err != nil {
+			return fmt.Errorf("invalid digest %q for %q: %w", entry.Digest, entry.Name, err)
+		}
+		path, err := storagePath.RegularFilePathForValidatedDigest(d)
+		if err != nil {
+			return fmt.Errorf("determining physical file path for %q: %w", entry.Name, err)
 		}
+		payload = path
 	}
 
 	if _, err := fmt.Fprint(out, escapedOptional([]byte(payload), ESCAPE_LONE_DASH)); err != nil {

pkg/chunked/dump/dump_test.go

@@ -59,7 +59,7 @@ func TestDumpNode(t *testing.T) {
 		Devminor: 0,
 		ModTime:  &modTime,
 		Linkname: "",
-		Digest:   "sha256:abcdef1234567890",
+		Digest:   "sha256:0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef",
 		Xattrs: map[string]string{
 			"user.key1": base64.StdEncoding.EncodeToString([]byte("value1")),
 		},
@@ -150,15 +150,15 @@ func TestDumpNode(t *testing.T) {
 			entries: []*minimal.FileMetadata{
 				regularFileEntry,
 			},
-			expected: "/example.txt 100 100000 1 1000 1000 0 1672531200.0 ab/cdef1234567890 - - user.key1=value1\n",
+			expected: "/example.txt 100 100000 1 1000 1000 0 1672531200.0 01/23456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef - - user.key1=value1\n",
 		},
 		{
 			name: "root entry with file",
 			entries: []*minimal.FileMetadata{
 				rootEntry,
 				regularFileEntry,
 			},
-			expected:            "/ 0 40000 1 0 0 0 1672531200.0 - - -\n/example.txt 100 100000 1 1000 1000 0 1672531200.0 ab/cdef1234567890 - - user.key1=value1\n",
+			expected:            "/ 0 40000 1 0 0 0 1672531200.0 - - -\n/example.txt 100 100000 1 1000 1000 0 1672531200.0 01/23456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef - - user.key1=value1\n",
 			skipAddingRootEntry: true,
 		},
 		{
@@ -196,7 +196,7 @@ func TestDumpNode(t *testing.T) {
 				regularFileEntry,
 				directoryEntry,
 			},
-			expected:            "/ 0 40000 1 0 0 0 1672531200.0 - - -\n/example.txt 100 100000 1 1000 1000 0 1672531200.0 ab/cdef1234567890 - - user.key1=value1\n/mydir 0 40000 1 1000 1000 0 1672531200.0 - - - user.key2=value2\n",
+			expected:            "/ 0 40000 1 0 0 0 1672531200.0 - - -\n/example.txt 100 100000 1 1000 1000 0 1672531200.0 01/23456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef - - user.key1=value1\n/mydir 0 40000 1 1000 1000 0 1672531200.0 - - - user.key2=value2\n",
 			skipAddingRootEntry: true,
 		},
 	}

pkg/chunked/internal/path/path.go

@@ -1,7 +1,10 @@
 package path
 
 import (
+	"fmt"
 	"path/filepath"
+
+	"github.com/opencontainers/go-digest"
 )
 
 // CleanAbsPath removes any ".." and "." from the path
@@ -10,3 +13,15 @@ import (
 func CleanAbsPath(path string) string {
 	return filepath.Clean("/" + path)
 }
+
+// RegularFilePath returns the path used in the composefs backing store for a
+// regular file with the provided content digest.
+//
+// The caller MUST ensure d is a valid digest (in particular, that it contains no path separators or .. entries)
+func RegularFilePathForValidatedDigest(d digest.Digest) (string, error) {
+	if algo := d.Algorithm(); algo != digest.SHA256 {
+		return "", fmt.Errorf("unexpected digest algorithm %q", algo)
+	}
+	e := d.Encoded()
+	return e[0:2] + "/" + e[2:], nil
+}

pkg/chunked/internal/path/path_test.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/opencontainers/go-digest"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCleanAbsPath(t *testing.T) {
@@ -46,3 +48,16 @@ func TestCleanAbsPath(t *testing.T) {
 		assert.Equal(t, test.expected, CleanAbsPath(test.path), fmt.Sprintf("path %q failed", test.path))
 	}
 }
+
+func TestRegularFilePathForValidatedDigest(t *testing.T) {
+	d, err := digest.Parse("sha256:0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef")
+	require.NoError(t, err)
+	res, err := RegularFilePathForValidatedDigest(d)
+	require.NoError(t, err)
+	assert.Equal(t, "01/23456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef", res)
+
+	d, err = digest.Parse("sha512:0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef")
+	require.NoError(t, err)
+	_, err = RegularFilePathForValidatedDigest(d)
+	assert.Error(t, err)
+}