Skip to content

Commit

Permalink
Merge pull request #1789 from buildpacks/fix/kaniko-cache
Browse files Browse the repository at this point in the history 
Stack fix & create new volume cache for kaniko instead of re-using build cache
  • Loading branch information
@jkutner
jkutner authored Jun 21, 2023
2 parents 1e2da27 + 2d62080 commit 21b9a3a
Show file tree
Hide file tree
Showing 7 changed files with 102 additions and 68 deletions.
4 changes: 2 additions & 2 deletions acceptance/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,12 +797,12 @@ func testAcceptance(
it("creates builder", func() {
// Linux containers (including Linux containers on Windows)
extSimpleLayersDiffID := "sha256:b9e4a0ddfb650c7aa71d1e6aceea1665365e409b3078bfdc1e51c2b07ab2b423"
extReadEnvDiffID := "sha256:ab7419c5e0b1a0789bd07cef2ed0573ec6e98eb05d7f05eb95d4f035243e331c"
extReadEnvDiffID := "sha256:6801a0398d023ff06a43c4fd03ef325a45daaa4540fc3ee140e2fb22bf5143a7"
bpSimpleLayersDiffID := "sha256:285ff6683c99e5ae19805f6a62168fb40dca64d813c53b782604c9652d745c70"
bpReadEnvDiffID := "sha256:dd1e0efcbf3f08b014ef6eff9cfe7a9eac1cf20bd9b6a71a946f0a74575aa56f"
if imageManager.HostOS() == "windows" { // Windows containers on Windows
extSimpleLayersDiffID = "sha256:a063cf949b9c267133e451ac8cd95b4e77571bb7c629dd817461dca769170810"
extReadEnvDiffID = "sha256:a4e7f114efa3692939974da9c9f08e47b3fdb5c779688dc8f5a950e0f804bef1"
extReadEnvDiffID = "sha256:4c37e22595762315d28805f32e1f5fd48b4ddfc293ef7b41e0e609a4241b8479"
bpSimpleLayersDiffID = "sha256:ccd1234cc5685e8a412b70c5f9a8e7b584b8e4f2a20c987ec242c9055de3e45e"
bpReadEnvDiffID = "sha256:8b22a7742ffdfbdd978787c6937456b68afb27c3585a3903048be7434d251e3f"
}
Expand Down
3 changes: 3 additions & 0 deletions acceptance/testdata/mock_buildpacks/0.2/read-env-extension/bin/generate
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,8 @@ FROM \${base_image}
USER root
RUN echo "Hello World" > /from-ext.txt
ARG user_id
USER \${user_id}
EOL
fi
2 changes: 1 addition & 1 deletion acceptance/testdata/pack_fixtures/report_output.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Pack:
Version: {{ .Version }}
OS/Arch: {{ .OS }}/{{ .Arch }}

Default Lifecycle Version: 0.17.0-pre.2
Default Lifecycle Version: 0.17.0-rc.1

Supported Platform APIs: 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12

Expand Down
79 changes: 42 additions & 37 deletions internal/build/lifecycle_execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,18 +219,39 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
}
}

var kanikoCache Cache
if l.PlatformAPI().AtLeast("0.12") {
// lifecycle 0.17.0 (introduces support for Platform API 0.12) and above will ensure that
// this volume is owned by the CNB user,
// and hence the restorer (after dropping privileges) will be able to write to it.
kanikoCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker)
} else {
switch {
case buildCache.Type() == cache.Volume:
// Re-use the build cache as the kaniko cache. Earlier versions of the lifecycle (0.16.x and below)
// already ensure this volume is owned by the CNB user.
kanikoCache = buildCache
case l.hasExtensionsForBuild():
// We need a usable kaniko cache, so error in this case.
return fmt.Errorf("build cache must be volume cache when building with extensions")
default:
// The kaniko cache is unused, so it doesn't matter that it's not usable.
kanikoCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker)
}
}

l.logger.Info(style.Step("RESTORING"))
if l.opts.ClearCache && l.PlatformAPI().LessThan("0.10") {
l.logger.Info("Skipping 'restore' due to clearing cache")
} else if err := l.Restore(ctx, buildCache, phaseFactory); err != nil {
} else if err := l.Restore(ctx, buildCache, kanikoCache, phaseFactory); err != nil {
return err
}

group, _ := errgroup.WithContext(context.TODO())
if l.platformAPI.AtLeast("0.10") && l.hasExtensionsForBuild() {
group.Go(func() error {
l.logger.Info(style.Step("EXTENDING (BUILD)"))
return l.ExtendBuild(ctx, buildCache, phaseFactory)
return l.ExtendBuild(ctx, kanikoCache, phaseFactory)
})
} else {
group.Go(func() error {
Expand All @@ -249,7 +270,7 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
if l.platformAPI.AtLeast("0.12") && l.hasExtensionsForRun() {
group.Go(func() error {
l.logger.Info(style.Step("EXTENDING (RUN)"))
return l.ExtendRun(ctx, buildCache, phaseFactory)
return l.ExtendRun(ctx, kanikoCache, phaseFactory)
})
}

Expand All @@ -258,7 +279,7 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
}

l.logger.Info(style.Step("EXPORTING"))
return l.Export(ctx, buildCache, launchCache, phaseFactory)
return l.Export(ctx, buildCache, launchCache, kanikoCache, phaseFactory)
}

if l.platformAPI.AtLeast("0.10") && l.hasExtensions() {
Expand Down Expand Up @@ -421,7 +442,7 @@ func (l *LifecycleExecution) Detect(ctx context.Context, phaseFactory PhaseFacto
return detect.Run(ctx)
}

func (l *LifecycleExecution) Restore(ctx context.Context, buildCache Cache, phaseFactory PhaseFactory) error {
func (l *LifecycleExecution) Restore(ctx context.Context, buildCache Cache, kanikoCache Cache, phaseFactory PhaseFactory) error {
// build up flags and ops
var flags []string
if l.opts.ClearCache {
Expand Down Expand Up @@ -454,12 +475,7 @@ func (l *LifecycleExecution) Restore(ctx context.Context, buildCache Cache, phas
registryImages = append(registryImages, l.opts.BuilderImage)
}

switch buildCache.Type() {
case cache.Volume:
kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.kanikoCacheDir()))
default:
return fmt.Errorf("build cache must be volume cache when building with extensions")
}
kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir()))
}

// for auths
Expand Down Expand Up @@ -569,9 +585,13 @@ func (l *LifecycleExecution) Analyze(ctx context.Context, buildCache, launchCach
if l.opts.RunImage != "" {
args = append([]string{"-run-image", l.opts.RunImage}, args...)
}
args = append([]string{"-stack", l.mountPaths.stackPath()}, args...)
stackOp = WithContainerOperations(WriteStackToml(l.mountPaths.stackPath(), l.opts.Builder.Stack(), l.os))
runOp = WithContainerOperations(WriteRunToml(l.mountPaths.runPath(), l.opts.Builder.RunImages(), l.os))
if l.platformAPI.LessThan("0.12") {
args = append([]string{"-stack", l.mountPaths.stackPath()}, args...)
stackOp = WithContainerOperations(WriteStackToml(l.mountPaths.stackPath(), l.opts.Builder.Stack(), l.os))
} else {
args = append([]string{"-run", l.mountPaths.runPath()}, args...)
runOp = WithContainerOperations(WriteRunToml(l.mountPaths.runPath(), l.opts.Builder.RunImages(), l.os))
}
}

flagsOp := WithFlags(flags...)
Expand Down Expand Up @@ -645,18 +665,9 @@ func (l *LifecycleExecution) Build(ctx context.Context, phaseFactory PhaseFactor
return build.Run(ctx)
}

func (l *LifecycleExecution) ExtendBuild(ctx context.Context, buildCache Cache, phaseFactory PhaseFactory) error {
func (l *LifecycleExecution) ExtendBuild(ctx context.Context, kanikoCache Cache, phaseFactory PhaseFactory) error {
flags := []string{"-app", l.mountPaths.appDir()}

// set kaniko cache opt
var kanikoCacheBindOp PhaseConfigProviderOperation
switch buildCache.Type() {
case cache.Volume:
kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.kanikoCacheDir()))
default:
return fmt.Errorf("build cache must be volume cache when building with extensions")
}

configProvider := NewPhaseConfigProvider(
"extender",
l,
Expand All @@ -667,26 +678,17 @@ func (l *LifecycleExecution) ExtendBuild(ctx context.Context, buildCache Cache,
WithFlags(flags...),
WithNetwork(l.opts.Network),
WithRoot(),
kanikoCacheBindOp,
WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir())),
)

extend := phaseFactory.New(configProvider)
defer extend.Cleanup()
return extend.Run(ctx)
}

func (l *LifecycleExecution) ExtendRun(ctx context.Context, buildCache Cache, phaseFactory PhaseFactory) error {
func (l *LifecycleExecution) ExtendRun(ctx context.Context, kanikoCache Cache, phaseFactory PhaseFactory) error {
flags := []string{"-app", l.mountPaths.appDir(), "-kind", "run"}

// set kaniko cache opt
var kanikoCacheBindOp PhaseConfigProviderOperation
switch buildCache.Type() {
case cache.Volume:
kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.kanikoCacheDir()))
default:
return fmt.Errorf("build cache must be volume cache when building with extensions")
}

configProvider := NewPhaseConfigProvider(
"extender",
l,
Expand All @@ -699,7 +701,7 @@ func (l *LifecycleExecution) ExtendRun(ctx context.Context, buildCache Cache, ph
WithRoot(),
WithImage(l.runImageAfterExtensions()),
WithBinds(fmt.Sprintf("%s:%s", filepath.Join(l.tmpDir, "cnb"), l.mountPaths.cnbDir())),
kanikoCacheBindOp,
WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir())),
)

extend := phaseFactory.New(configProvider)
Expand All @@ -717,19 +719,21 @@ func determineDefaultProcessType(platformAPI *api.Version, providedValue string)
return providedValue
}

func (l *LifecycleExecution) Export(ctx context.Context, buildCache, launchCache Cache, phaseFactory PhaseFactory) error {
func (l *LifecycleExecution) Export(ctx context.Context, buildCache, launchCache, kanikoCache Cache, phaseFactory PhaseFactory) error {
flags := []string{
"-app", l.mountPaths.appDir(),
"-cache-dir", l.mountPaths.cacheDir(),
}

expEnv := NullOp()
kanikoCacheBindOp := NullOp()
if l.platformAPI.LessThan("0.12") {
flags = append(flags, "-stack", l.mountPaths.stackPath())
} else {
flags = append(flags, "-run", l.mountPaths.runPath())
if l.hasExtensionsForRun() {
expEnv = WithEnv("CNB_EXPERIMENTAL_MODE=warn")
kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir()))
}
}

Expand Down Expand Up @@ -771,6 +775,7 @@ func (l *LifecycleExecution) Export(ctx context.Context, buildCache, launchCache
WithRoot(),
WithNetwork(l.opts.Network),
cacheBindOp,
kanikoCacheBindOp,
WithContainerOperations(WriteStackToml(l.mountPaths.stackPath(), l.opts.Builder.Stack(), l.os)),
WithContainerOperations(WriteRunToml(l.mountPaths.runPath(), l.opts.Builder.RunImages(), l.os)),
WithContainerOperations(WriteProjectMetadata(l.mountPaths.projectPath(), l.opts.ProjectMetadata, l.os)),
Expand Down
50 changes: 38 additions & 12 deletions internal/build/lifecycle_execution_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
lifecycle *build.LifecycleExecution
fakeBuildCache = newFakeVolumeCache()
fakeLaunchCache *fakes.FakeCache
fakeKanikoCache *fakes.FakeCache
fakePhase *fakes.FakePhase
fakePhaseFactory *fakes.FakePhaseFactory
fakeFetcher fakeImageFetcher
Expand Down Expand Up @@ -164,6 +165,10 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
fakeLaunchCache.ReturnForType = cache.Volume
fakeLaunchCache.ReturnForName = "some-launch-cache"

fakeKanikoCache = fakes.NewFakeCache()
fakeKanikoCache.ReturnForType = cache.Volume
fakeKanikoCache.ReturnForName = "some-kaniko-cache"

fakePhase = &fakes.FakePhase{}
fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))
})
Expand Down Expand Up @@ -1396,6 +1401,21 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
})
})

when("platform >= 0.12", func() {
platformAPI = api.MustParse("0.12")

it("passes run", func() {
h.AssertIncludeAllExpectedPatterns(t,
configProvider.ContainerConfig().Cmd,
[]string{"-run", "/layers/run.toml"},
)
h.AssertSliceNotContains(t,
configProvider.ContainerConfig().Cmd,
"-stack",
)
})
})

when("publish", func() {
providedPublish = true

Expand Down Expand Up @@ -1637,7 +1657,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

when("#Restore", func() {
it.Before(func() {
err := lifecycle.Restore(context.Background(), fakeBuildCache, fakePhaseFactory)
err := lifecycle.Restore(context.Background(), fakeBuildCache, fakeKanikoCache, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
Expand Down Expand Up @@ -1749,7 +1769,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

it("does not provide -build-image or /kaniko bind", func() {
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-build-image")
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-cache:/kaniko")
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko")
})
})

Expand All @@ -1759,7 +1779,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
it("provides -build-image and /kaniko bind", func() {
h.AssertSliceContainsInOrder(t, configProvider.ContainerConfig().Cmd, "-build-image", providedBuilderImage)
h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_REGISTRY_AUTH={}")
h.AssertSliceContains(t, configProvider.HostConfig().Binds, "some-cache:/kaniko")
h.AssertSliceContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko")
})
})
})
Expand All @@ -1769,7 +1789,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

it("does not provide -build-image or /kaniko bind", func() {
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-build-image")
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-cache:/kaniko")
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko")
})
})
})
Expand All @@ -1783,15 +1803,15 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
platformAPI = api.MustParse("0.12")

it("provides /kaniko bind", func() {
h.AssertSliceContains(t, configProvider.HostConfig().Binds, "some-cache:/kaniko")
h.AssertSliceContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko")
})
})

when("platform < 0.12", func() {
platformAPI = api.MustParse("0.11")

it("does not provide /kaniko bind", func() {
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-cache:/kaniko")
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko")
})
})
})
Expand All @@ -1800,7 +1820,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
platformAPI = api.MustParse("0.12")

it("does not provide /kaniko bind", func() {
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-cache:/kaniko")
h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko")
})
})
})
Expand Down Expand Up @@ -1843,7 +1863,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

when("#ExtendBuild", func() {
it.Before(func() {
err := lifecycle.ExtendBuild(context.Background(), fakeBuildCache, fakePhaseFactory)
err := lifecycle.ExtendBuild(context.Background(), fakeKanikoCache, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
Expand All @@ -1865,7 +1885,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

it("configures the phase with binds", func() {
expectedBinds := providedVolumes
expectedBinds = append(expectedBinds, "some-cache:/kaniko")
expectedBinds = append(expectedBinds, "some-kaniko-cache:/kaniko")

h.AssertSliceContains(t, configProvider.HostConfig().Binds, expectedBinds...)
})
Expand All @@ -1885,7 +1905,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

when("#ExtendRun", func() {
it.Before(func() {
err := lifecycle.ExtendRun(context.Background(), fakeBuildCache, fakePhaseFactory)
err := lifecycle.ExtendRun(context.Background(), fakeKanikoCache, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
Expand Down Expand Up @@ -1921,7 +1941,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

it("configures the phase with binds", func() {
expectedBinds := providedVolumes
expectedBinds = append(expectedBinds, "some-cache:/kaniko", fmt.Sprintf("%s:/cnb", filepath.Join(tmpDir, "cnb")))
expectedBinds = append(expectedBinds, "some-kaniko-cache:/kaniko", fmt.Sprintf("%s:/cnb", filepath.Join(tmpDir, "cnb")))

h.AssertSliceContains(t, configProvider.HostConfig().Binds, expectedBinds...)
})
Expand All @@ -1941,7 +1961,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {

when("#Export", func() {
it.Before(func() {
err := lifecycle.Export(context.Background(), fakeBuildCache, fakeLaunchCache, fakePhaseFactory)
err := lifecycle.Export(context.Background(), fakeBuildCache, fakeLaunchCache, fakeKanikoCache, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
Expand Down Expand Up @@ -1986,6 +2006,12 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
it("sets CNB_EXPERIMENTAL_MODE=warn in the environment", func() {
h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_EXPERIMENTAL_MODE=warn")
})

it("configures the phase with binds", func() {
expectedBinds := []string{"some-cache:/cache", "some-launch-cache:/launch-cache", "some-kaniko-cache:/kaniko"}

h.AssertSliceContains(t, configProvider.HostConfig().Binds, expectedBinds...)
})
})
})
})
Expand Down
2 changes: 1 addition & 1 deletion internal/builder/lifecycle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (

// A snapshot of the latest tested lifecycle version values
const (
DefaultLifecycleVersion = "0.17.0-pre.2"
DefaultLifecycleVersion = "0.17.0-rc.1"
DefaultBuildpackAPIVersion = "0.2"
)

Expand Down
Loading

0 comments on commit 21b9a3a

Please sign in to comment.