tarfs: export nydus tarfs image as block with verity

Enhance tarfs implementation to support following operations:
- export an tarfs image as a block device
- export an tarfs image as a block device with verity
- export an tarfs layer as a block device
- export an tarfs layer as a block device with verity

Signed-off-by: Jiang Liu <[email protected]>

config/config.go

@@ -107,11 +107,16 @@ const (
 )
 
 type Experimental struct {
-	EnableStargz           bool `toml:"enable_stargz"`
-	EnableReferrerDetect   bool `toml:"enable_referrer_detect"`
-	EnableTarfs            bool `toml:"enable_tarfs"`
-	TarfsHint              bool `toml:"tarfs_hint"`
-	TarfsMaxConcurrentProc int  `toml:"tarfs_max_concurrent_proc"`
+	EnableStargz         bool        `toml:"enable_stargz"`
+	EnableReferrerDetect bool        `toml:"enable_referrer_detect"`
+	TarfsConfig          TarfsConfig `toml:"tarfs"`
+}
+
+type TarfsConfig struct {
+	EnableTarfs       bool   `toml:"enable_tarfs"`
+	TarfsHint         bool   `toml:"tarfs_hint"`
+	MaxConcurrentProc int    `toml:"max_concurrent_proc"`
+	ExportMode        string `toml:"export_mode"`
 }
 
 type CgroupConfig struct {

config/global.go

@@ -107,6 +107,44 @@ func GetDaemonProfileCPUDuration() int64 {
 	return globalConfig.origin.SystemControllerConfig.DebugConfig.ProfileDuration
 }
 
+func GetTarfsExportEnabled() bool {
+	switch globalConfig.origin.Experimental.TarfsConfig.ExportMode {
+	case "layer_verity_only":
+		return true
+	case "image_verity_only":
+		return true
+	case "layer_block":
+		return true
+	case "image_block":
+		return true
+	case "layer_block_with_verity":
+		return true
+	case "image_block_with_verity":
+		return true
+	default:
+		return false
+	}
+}
+
+func GetTarfsExportFlags() (bool, bool, bool) {
+	switch globalConfig.origin.Experimental.TarfsConfig.ExportMode {
+	case "layer_verity_only":
+		return false, false, true
+	case "image_verity_only":
+		return true, false, true
+	case "layer_block":
+		return false, true, false
+	case "image_block":
+		return true, true, false
+	case "layer_block_with_verity":
+		return false, true, true
+	case "image_block_with_verity":
+		return true, true, true
+	default:
+		return false, false, false
+	}
+}
+
 func ProcessConfigurations(c *SnapshotterConfig) error {
 	if c.LoggingConfig.LogDir == "" {
 		c.LoggingConfig.LogDir = filepath.Join(c.Root, logging.DefaultLogDirName)

misc/snapshotter/config.toml

@@ -101,3 +101,19 @@ enable_stargz = false
 # The option enables trying to fetch the Nydus image associated with the OCI image and run it.
 # Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers
 enable_referrer_detect = false
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode
+enable_tarfs = false
+# Only enable nydus tarfs mode for images with `tarfs hint` label when true
+tarfs_hint = false
+# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default
+max_concurrent_proc = 0
+# Mode to export tarfs images: 
+# - "none"/""
+# - "layer_verity_only"
+# - "image_verity_only"
+# - "layer_block"
+# - "image_block"
+# - "layer_block_with_verity"
+# - "image_block_with_verity"
+export_mode = ""

pkg/cache/manager.go

@@ -21,8 +21,10 @@ import (
 )
 
 const (
-	chunkMapFileSuffix = ".chunk_map"
-	metaFileSuffix     = ".blob.meta"
+	imageDiskFileSuffix = ".image.disk"
+	layerDiskFileSuffix = ".layer.disk"
+	chunkMapFileSuffix  = ".chunk_map"
+	metaFileSuffix      = ".blob.meta"
 	// Blob cache is suffixed after nydus v2.1
 	dataFileSuffix = ".blob.data"
 )
@@ -72,8 +74,10 @@ func (m *Manager) CacheUsage(ctx context.Context, blobID string) (snapshots.Usag
 	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
 	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
 	blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix)
+	imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix)
+	layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix)
 
-	stuffs := []string{blobCachePath, blobCacheSuffixedPath, blobChunkMap, blobMeta}
+	stuffs := []string{blobCachePath, blobCacheSuffixedPath, blobChunkMap, blobMeta, imageDisk, layerDisk}
 
 	for _, f := range stuffs {
 		du, err := fs.DiskUsage(ctx, f)
@@ -95,9 +99,11 @@ func (m *Manager) RemoveBlobCache(blobID string) error {
 	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
 	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
 	blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix)
+	imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix)
+	layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix)
 
 	// NOTE: Delete chunk bitmap file before data blob
-	stuffs := []string{blobChunkMap, blobMeta, blobCachePath, blobCacheSuffixedPath}
+	stuffs := []string{blobChunkMap, blobMeta, blobCachePath, blobCacheSuffixedPath, imageDisk, layerDisk}
 
 	for _, f := range stuffs {
 		err := os.Remove(f)

pkg/filesystem/tarfs_adaptor.go

@@ -49,18 +49,21 @@ func (fs *Filesystem) PrepareTarfsLayer(ctx context.Context, labels map[string]s
 		}
 	}
 
-	go func() {
-		if err := fs.tarfsMgr.PrepareLayer(snapshotID, ref, manifestDigest, layerDigest, upperDirPath); err != nil {
-			log.L.WithError(err).Errorf("async prepare tarfs layer of snapshot ID %s", snapshotID)
-		}
-		if limiter != nil {
-			limiter.Release(1)
-		}
-	}()
+	if err := fs.tarfsMgr.PrepareLayer(snapshotID, ref, manifestDigest, layerDigest, upperDirPath); err != nil {
+		log.L.WithError(err).Errorf("async prepare tarfs layer of snapshot ID %s", snapshotID)
+	}
+	if limiter != nil {
+		limiter.Release(1)
+	}
 
 	return nil
 }
 
+func (fs *Filesystem) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[string]string,
+	storageLocater func(string) string) ([]string, error) {
+	return fs.tarfsMgr.ExportBlockData(s, perLayer, labels, storageLocater)
+}
+
 func (fs *Filesystem) MergeTarfsLayers(s storage.Snapshot, storageLocater func(string) string) error {
 	return fs.tarfsMgr.MergeLayers(s, storageLocater)
 }

pkg/label/label.go

@@ -42,6 +42,10 @@ const (
 	NydusImagePullUsername = "containerd.io/snapshot/pullusername"
 	// A bool flag to enable integrity verification of meta data blob
 	NydusSignature = "containerd.io/snapshot/nydus-signature"
+	// Information for image block device
+	NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block"
+	// Information for layer block device
+	NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block"
 
 	// A bool flag to mark the blob as a estargz data blob, set by the snapshotter.
 	StargzLayer = "containerd.io/snapshot/stargz"