executor: fix overlay layer limit for non-rootfs mounts

Historic layer limit for Docker images is 127. Because in
overlayfs mounting 127 layers usually reaches the page size
limit of mount options in Linux kernel, there is special code
to work around the limitation.

This custom code was used for rootfs of container because
runc takes rootfs as a directory path, meaning buildkit needs
to mount it and then pass the path. For non-rootfs mounts
runc takes them as direct mount configuration and performs
the mount itself. As runc does not have this special way to
mount long overlayfs mounts it will perform the mount with clipped
options what will fail in some way in kernel depending on the
precise cutoff point.

Workaround is to detect when the mount passed to runc is too
long for runc to mount it itself and it that case let
BuildKit mount it and in runc perform bind of the BuildKit mount.

Signed-off-by: Tonis Tiigi <[email protected]>

client/client_test.go

@@ -216,6 +216,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testExportLocalNoPlatformSplitOverwrite,
 	testSolverOptLocalDirsStillWorks,
 	testOCIIndexMediatype,
+	testLayerLimitOnMounts,
 }
 
 func TestIntegration(t *testing.T) {
@@ -10242,6 +10243,40 @@ func testLLBMountPerformance(t *testing.T, sb integration.Sandbox) {
 	require.NoError(t, err)
 }
 
+func testLayerLimitOnMounts(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
+
+	ctx := sb.Context()
+
+	c, err := New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	base := llb.Image("busybox:latest")
+
+	const numLayers = 110
+
+	for i := 0; i < numLayers; i++ {
+		base = base.Run(llb.Shlex("sh -c 'echo hello >> /hello'")).Root()
+	}
+
+	def, err := base.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	_, err = c.Solve(ctx, def, SolveOpt{}, nil)
+	require.NoError(t, err)
+
+	ls := llb.Image("busybox:latest").
+		Run(llb.Shlexf("ls -l /base/hello"))
+	ls.AddMount("/base", base, llb.Readonly)
+
+	def, err = ls.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	_, err = c.Solve(ctx, def, SolveOpt{}, nil)
+	require.NoError(t, err)
+}
+
 func testClientCustomGRPCOpts(t *testing.T, sb integration.Sandbox) {
 	var interceptedMethods []string
 	intercept := func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {

executor/oci/spec.go

@@ -183,6 +183,16 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 		}
 		releasers = append(releasers, release)
 		for _, mount := range mounts {
+			mount, release, err := compactLongOverlayMount(mount, m.Readonly)
+			if err != nil {
+				releaseAll()
+				return nil, nil, err
+			}
+
+			if release != nil {
+				releasers = append(releasers, release)
+			}
+
 			mount, err = sm.subMount(mount, m.Selector)
 			if err != nil {
 				releaseAll()
@@ -261,26 +271,8 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
 		return mount.Mount{}, err
 	}
 
-	var mntType string
-	opts := []string{}
-	if m.ReadOnly() {
-		opts = append(opts, "ro")
-	}
-
-	if runtime.GOOS != "windows" {
-		// Windows uses a mechanism similar to bind mounts, but will err out if we request
-		// a mount type it does not understand. Leaving the mount type empty on Windows will
-		// yield the same result.
-		mntType = "bind"
-		opts = append(opts, "rbind")
-	}
-
 	s.m[h] = mountRef{
-		mount: mount.Mount{
-			Source:  mp,
-			Type:    mntType,
-			Options: opts,
-		},
+		mount:   bind(mp, m.ReadOnly()),
 		unmount: lm.Unmount,
 		subRefs: map[string]mountRef{},
 	}
@@ -312,3 +304,45 @@ func (s *submounts) cleanup() {
 	}
 	wg.Wait()
 }
+
+func bind(p string, ro bool) mount.Mount {
+	m := mount.Mount{
+		Source: p,
+	}
+	if runtime.GOOS != "windows" {
+		// Windows uses a mechanism similar to bind mounts, but will err out if we request
+		// a mount type it does not understand. Leaving the mount type empty on Windows will
+		// yield the same result.
+		m.Type = "bind"
+		m.Options = []string{"rbind"}
+	}
+	if ro {
+		m.Options = append(m.Options, "ro")
+	}
+	return m
+}
+
+func compactLongOverlayMount(m mount.Mount, ro bool) (mount.Mount, func() error, error) {
+	if m.Type != "overlay" {
+		return m, nil, nil
+	}
+
+	sz := 0
+	for _, opt := range m.Options {
+		sz += len(opt) + 1
+	}
+
+	// can fit to single page, no need to compact
+	if sz < 4096-512 {
+		return m, nil, nil
+	}
+
+	lm := snapshot.LocalMounterWithMounts([]mount.Mount{m})
+
+	mp, err := lm.Mount()
+	if err != nil {
+		return mount.Mount{}, nil, err
+	}
+
+	return bind(mp, ro), lm.Unmount, nil
+}