From 86e4a656607e842e7e2a069d3b3418635d5904af Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 30 Oct 2024 15:45:25 -0700 Subject: [PATCH 1/8] tests: Fix some missing parallel calls and missing test spans Signed-off-by: Brian Goff --- test/azlinux_test.go | 24 ++++++++++++++++++++-- test/fs_test.go | 40 ++++++++++++++++++++++++++++-------- test/handlers_test.go | 12 +++++++---- test/signing_test.go | 14 +++++++++++++ test/source_test.go | 25 ++++++++++++++++------ test/var_passthrough_test.go | 2 ++ test/windows_test.go | 2 ++ 7 files changed, 99 insertions(+), 20 deletions(-) diff --git a/test/azlinux_test.go b/test/azlinux_test.go index 7a0b3314..71f7e7a4 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -217,6 +217,8 @@ type OSRelease struct { func testLinuxDistro(ctx context.Context, t *testing.T, testConfig testLinuxConfig) { t.Run("Fail when non-zero exit code during build", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(ctx, t) + spec := dalec.Spec{ Name: "test-build-commands-fail", Version: "0.0.1", @@ -247,6 +249,9 @@ func testLinuxDistro(ctx context.Context, t *testing.T, testConfig testLinuxConf }) t.Run("container", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + const src2Patch3File = "patch3" src2Patch3Content := []byte(` diff --git a/file3 b/file3 @@ -523,6 +528,8 @@ echo "$BAR" > bar.txt t.Run("test systemd unit single", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-systemd-unit", Description: "Test systemd unit", @@ -646,6 +653,8 @@ WantedBy=multi-user.target t.Run("test systemd unit multiple components", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-systemd-unit", Description: "Test systemd unit", @@ -760,6 +769,8 @@ Environment="FOO_ARGS=--some-foo-args" t.Run("test systemd with only config dropin", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-systemd-unit", Description: "Test systemd unit", @@ -871,6 +882,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test directory creation", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(ctx, t) + spec := &dalec.Spec{ Name: "test-directory-creation", Version: "0.0.1", @@ -937,6 +950,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test data file installation", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-data-file-installation", Version: "0.0.1", @@ -1029,6 +1044,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test libexec file installation", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "libexec-test", Version: "0.0.1", @@ -1133,6 +1150,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test config files handled", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-config-files-work", Version: "0.0.1", @@ -1192,6 +1211,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("docs and headers and licenses are handled correctly", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-docs-handled", Version: "0.0.1", @@ -1364,9 +1385,7 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }) t.Run("custom repo", func(t *testing.T) { - t.Parallel() - ctx := startTestSpan(baseCtx, t) testCustomRepo(ctx, t, testConfig.Worker, testConfig.Target) }) @@ -1606,6 +1625,7 @@ func testPinnedBuildDeps(ctx context.Context, t *testing.T, cfg testLinuxConfig) t.Run(tt.name, func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { worker := getWorker(ctx, t, gwc) diff --git a/test/fs_test.go b/test/fs_test.go index a777506b..c6911978 100644 --- a/test/fs_test.go +++ b/test/fs_test.go @@ -16,9 +16,12 @@ import ( ) func TestStateWrapper_ReadAt(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkfile("/foo", 0644, []byte("hello world"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -45,8 +48,11 @@ func TestStateWrapper_ReadAt(t *testing.T) { } func TestStateWrapper_OpenInvalidPath(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkfile("/bar", 0644, []byte("hello world"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -61,10 +67,13 @@ func TestStateWrapper_OpenInvalidPath(t *testing.T) { } func TestStateWrapper_Open(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch(). File(llb.Mkfile("/foo", 0644, []byte("hello world"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { fs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -84,8 +93,11 @@ func TestStateWrapper_Open(t *testing.T) { } func TestStateWrapper_Stat(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkfile("/foo", 0755, []byte("contents"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -103,6 +115,9 @@ func TestStateWrapper_Stat(t *testing.T) { } func TestStateWrapper_ReadDir(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkdir("/bar", 0644)). File(llb.Mkfile("/bar/foo", 0644, []byte("file contents"))). File(llb.Mkdir("/bar/baz", 0644)) @@ -124,7 +139,7 @@ func TestStateWrapper_ReadDir(t *testing.T) { }, } - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -148,6 +163,9 @@ func TestStateWrapper_ReadDir(t *testing.T) { } func TestStateWrapper_Walk(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + // create a simple test file structure like so: /* dir/ @@ -216,7 +234,7 @@ func TestStateWrapper_Walk(t *testing.T) { }, } - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) totalCalls := 0 @@ -259,6 +277,9 @@ func TestStateWrapper_Walk(t *testing.T) { } func TestStateWrapper_ReadPartial(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + contents := []byte(` This is a multline @@ -266,7 +287,7 @@ func TestStateWrapper_ReadPartial(t *testing.T) { `) st := llb.Scratch().File(llb.Mkfile("/foo", 0644, contents)) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -308,6 +329,9 @@ func TestStateWrapper_ReadPartial(t *testing.T) { } func TestStateWrapper_ReadAll(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + // purposefully exceed initial length of io.ReadAll buffer (512) b := make([]byte, 520) for i := 0; i < 520; i++ { @@ -316,7 +340,7 @@ func TestStateWrapper_ReadAll(t *testing.T) { st := llb.Scratch().File(llb.Mkfile("/file", 0644, b)) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) diff --git a/test/handlers_test.go b/test/handlers_test.go index c20d7aa7..93439e67 100644 --- a/test/handlers_test.go +++ b/test/handlers_test.go @@ -21,6 +21,8 @@ import ( // TestHandlerTargetForwarding tests that targets are forwarded to the correct frontend. // We do this by registering a phony frontend and then forwarding a target to it and checking the outputs. func TestHandlerTargetForwarding(t *testing.T) { + t.Parallel() + runTest := func(t *testing.T, f testenv.TestFunc) { t.Helper() ctx := startTestSpan(baseCtx, t) @@ -129,10 +131,12 @@ func TestHandlerTargetForwarding(t *testing.T) { func TestHandlerSubrequestResolve(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) - testPlatforms := func(t *testing.T, pls ...string) func(t *testing.T) { + testPlatforms := func(ctx context.Context, pls ...string) func(t *testing.T) { return func(t *testing.T) { t.Parallel() + startTestSpan(ctx, t) runTest(t, func(ctx context.Context, gwc gwclient.Client) { spec := &dalec.Spec{ @@ -190,7 +194,7 @@ func TestHandlerSubrequestResolve(t *testing.T) { } } - t.Run("no platform", testPlatforms(t)) - t.Run("single platform", testPlatforms(t, "linux/amd64")) - t.Run("multi-platform", testPlatforms(t, "linux/amd64", "linux/arm64")) + t.Run("no platform", testPlatforms(ctx)) + t.Run("single platform", testPlatforms(ctx, "linux/amd64")) + t.Run("multi-platform", testPlatforms(ctx, "linux/amd64", "linux/arm64")) } diff --git a/test/signing_test.go b/test/signing_test.go index fcf19bdd..6212997c 100644 --- a/test/signing_test.go +++ b/test/signing_test.go @@ -50,6 +50,9 @@ func newSimpleSpec() *dalec.Spec { func linuxSigningTests(ctx context.Context, testConfig testLinuxConfig) func(*testing.T) { return func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + newSigningSpec := func() *dalec.Spec { spec := newSimpleSpec() spec.PackageConfig = &dalec.PackageConfig{ @@ -133,6 +136,8 @@ func linuxSigningTests(ctx context.Context, testConfig testLinuxConfig) func(*te }) t.Run("with path build arg and build context", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer = nil @@ -153,6 +158,8 @@ signer: }) t.Run("path build arg takes precedence over spec config", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer.Frontend.Image = "notexist" @@ -191,6 +198,8 @@ signer: }) t.Run("with path build arg and build context", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer = nil @@ -211,6 +220,8 @@ signer: }) t.Run("with no build context and config path build arg", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer = nil @@ -231,6 +242,8 @@ signer: }) t.Run("local context with config path takes precedence over spec", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer.Frontend.Image = "notexist" @@ -362,6 +375,7 @@ signer: } func windowsSigningTests(t *testing.T) { + t.Parallel() t.Run("target spec config", func(t *testing.T) { t.Parallel() runTest(t, func(ctx context.Context, gwc gwclient.Client) { diff --git a/test/source_test.go b/test/source_test.go index 0094ea4e..6b8c0c3a 100644 --- a/test/source_test.go +++ b/test/source_test.go @@ -65,16 +65,24 @@ func TestSourceCmd(t *testing.T) { } } - testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { - spec := testSpec() - req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, spec)) - res := solveT(ctx, t, gwc, req) + t.Run("base", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { + spec := testSpec() + req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, spec)) + res := solveT(ctx, t, gwc, req) - checkFile(ctx, t, filepath.Join(sourceName, "foo"), res, []byte("foo bar\n")) - checkFile(ctx, t, filepath.Join(sourceName, "hello"), res, []byte("hello\n")) + checkFile(ctx, t, filepath.Join(sourceName, "foo"), res, []byte("foo bar\n")) + checkFile(ctx, t, filepath.Join(sourceName, "hello"), res, []byte("hello\n")) + }) }) t.Run("with mounted file", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { spec := testSpec() spec.Sources[sourceName].DockerImage.Cmd.Steps = []*dalec.BuildStep{ @@ -139,6 +147,7 @@ func TestSourceBuild(t *testing.T) { } t.Run("inline", func(t *testing.T) { + t.Parallel() fileSrc := func() dalec.Source { return dalec.Source{ Inline: &dalec.SourceInline{ @@ -165,16 +174,19 @@ func TestSourceBuild(t *testing.T) { } t.Run("unspecified build file path", func(t *testing.T) { + t.Parallel() doBuildTest(t, "file", newBuildSpec("", fileSrc)) doBuildTest(t, "dir", newBuildSpec("", dirSrc("Dockerfile"))) }) t.Run("Dockerfile as build file path", func(t *testing.T) { + t.Parallel() doBuildTest(t, "file", newBuildSpec("Dockerfile", fileSrc)) doBuildTest(t, "dir", newBuildSpec("Dockerfile", dirSrc("Dockerfile"))) }) t.Run("non-standard build file path", func(t *testing.T) { + t.Parallel() doBuildTest(t, "file", newBuildSpec("foo", fileSrc)) doBuildTest(t, "dir", newBuildSpec("foo", dirSrc("foo"))) }) @@ -371,6 +383,7 @@ index ea874f5..ba38f84 100644 }) t.Run("with patch", func(t *testing.T) { + t.Parallel() t.Run("file", func(t *testing.T) { t.Parallel() testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { diff --git a/test/var_passthrough_test.go b/test/var_passthrough_test.go index 9aca4a37..67e08c2f 100644 --- a/test/var_passthrough_test.go +++ b/test/var_passthrough_test.go @@ -38,6 +38,8 @@ func getBuildPlatform(ctx context.Context, t *testing.T) *ocispecs.Platform { } func TestPassthroughVars(t *testing.T) { + t.Parallel() + runTest := func(t *testing.T, f testenv.TestFunc) { t.Helper() ctx := startTestSpan(baseCtx, t) diff --git a/test/windows_test.go b/test/windows_test.go index 0813f033..e7cea462 100644 --- a/test/windows_test.go +++ b/test/windows_test.go @@ -147,6 +147,8 @@ func testWindows(ctx context.Context, t *testing.T, cfg targetConfig) { }) t.Run("container", func(t *testing.T) { + t.Parallel() + spec := dalec.Spec{ Name: "test-container-build", Version: "0.0.1", From 93d1ad928b3bcda8dfdcdeb249e69814867ad2ae Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 31 Oct 2024 14:55:24 -0700 Subject: [PATCH 2/8] test: write build logs by line Before this was calling `t.Log` for every read from the log file. What we really want is to to call `t.Log` for every line since a `Read` may return a partial line which will look weird in the output and be difficult to read. This also changes things to always write the build logs to the test log instead of just on failure so its easier to see what's happening even on a successful test run. Signed-off-by: Brian Goff --- test/testenv/{builld.go => build.go} | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) rename test/testenv/{builld.go => build.go} (95%) diff --git a/test/testenv/builld.go b/test/testenv/build.go similarity index 95% rename from test/testenv/builld.go rename to test/testenv/build.go index a0ce5945..4cd5ae55 100644 --- a/test/testenv/builld.go +++ b/test/testenv/build.go @@ -1,6 +1,7 @@ package testenv import ( + "bufio" "context" "encoding/json" "io" @@ -154,35 +155,24 @@ func displaySolveStatus(ctx context.Context, t *testing.T) (chan *client.SolveSt } defer f.Close() - if !t.Failed() { - return - } - - sz, _ := f.Seek(0, io.SeekEnd) _, err = f.Seek(0, io.SeekStart) if err != nil { t.Log(err) return } - _, err = io.CopyN(&testWriter{t}, f, sz) - if err != nil { + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + t.Log(scanner.Text()) + } + if err := scanner.Err(); err != nil { t.Log(err) - return } }() return ch, done } -type testWriter struct { - t *testing.T -} - -func (t *testWriter) Write(p []byte) (n int, err error) { - t.t.Log(string(p)) - return len(p), nil -} - // withProjectRoot adds the current project root as the build context for the solve request. func withProjectRoot(t *testing.T, opts *client.SolveOpt) { t.Helper() From 380c6a2ad8845239321bae4e24581511d77aa833 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Sat, 2 Nov 2024 15:14:21 -0700 Subject: [PATCH 3/8] CI: Precache frontend/worker images This prevents the skew of test times due to having to build the frontend and worker images. Signed-off-by: Brian Goff --- .github/workflows/ci.yml | 2 + cmd/ci-precache/main.go | 219 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 cmd/ci-precache/main.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0dcebad..7a76980a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,8 @@ jobs: uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: download deps run: go mod download + - name: Precache base images + run: go run ./cmd/ci-precache - name: Run integration tests run: go test -v -json ./test | go run ./cmd/test2json2gha - name: dump logs diff --git a/cmd/ci-precache/main.go b/cmd/ci-precache/main.go new file mode 100644 index 00000000..1dfe04b2 --- /dev/null +++ b/cmd/ci-precache/main.go @@ -0,0 +1,219 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/Azure/dalec/frontend/azlinux" + "github.com/Azure/dalec/frontend/jammy" + "github.com/Azure/dalec/frontend/windows" + "github.com/Azure/dalec/test/testenv" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/frontend/dockerui" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/progress/progressui" + "github.com/moby/patternmatcher/ignorefile" + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil" + "golang.org/x/sync/errgroup" +) + +func main() { + ctx, done := signal.NotifyContext(context.Background(), os.Interrupt) + go func() { + <-ctx.Done() + // The context was cancelled due to interupt + // This _should_ trigger builds to cancel naturally and exit the program, + // but in some cases it may not (due to timing, bugs in buildkit, uninteruptable operations, etc.). + // Cancel our signal handler so the normal handler takes over from here. + // This allows subsequent interupts to use the default behavior (exit the program) + done() + + <-time.After(30 * time.Second) + fmt.Fprintln(os.Stderr, "Timeout waiting for builds to cancel after interupt") + os.Exit(int(syscall.SIGINT)) + }() + + bklog.G(ctx).Logger.SetOutput(os.Stderr) + + if err := run(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(ctx context.Context) (retErr error) { + env := testenv.New() + + c, err := env.Buildkit(ctx) + if err != nil { + return err + } + defer c.Close() + + ch := make(chan *client.SolveStatus) + d, err := progressui.NewDisplay(os.Stderr, progressui.AutoMode) + if err != nil { + return err + } + + chErr := make(chan error, 1) + go func() { + warnings, err := d.UpdateFrom(ctx, ch) + for _, w := range warnings { + bklog.G(ctx).Warn(string(w.Short)) + } + chErr <- err + }() + + defer func() { + e := <-chErr + if retErr == nil { + retErr = e + } + }() + + localFS, err := fsutil.NewFS(".") + if err != nil { + return err + } + + ignoreF, err := localFS.Open(".dockerignore") + if err != nil { + return err + } + defer ignoreF.Close() + excludes, err := ignorefile.ReadAll(ignoreF) + if err != nil { + return err + } + + localFS, err = fsutil.NewFilterFS(localFS, &fsutil.FilterOpt{ + ExcludePatterns: excludes, + }) + if err != nil { + return err + } + + so := client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + ".": localFS, + }, + } + _, err = c.Build(ctx, so, "", build, ch) + return err +} + +func build(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { + res, err := buildFrontend(ctx, client) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + st, err := ref.ToState() + if err != nil { + return nil, err + } + + frontend, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + workers := []string{ + azlinux.Mariner2TargetKey, + azlinux.AzLinux3TargetKey, + windows.DefaultTargetKey, + jammy.DefaultTargetKey, + } + + eg, ctx := errgroup.WithContext(ctx) + + metaDt, err := json.Marshal(res.Metadata) + if err != nil { + return nil, err + } + + nullDef, err := nullDockerfile.Marshal(ctx) + if err != nil { + return nil, err + } + + id := identity.NewID() + for _, t := range workers { + eg.Go(func() error { + sr := gwclient.SolveRequest{ + Evaluate: true, + Frontend: "gateway.v0", + FrontendOpt: map[string]string{ + "source": id, + "target": t + "/worker", + "context:" + id: "input:" + id, + "input-metadata:" + id: string(metaDt), + }, + FrontendInputs: map[string]*pb.Definition{ + id: frontend.ToPB(), + dockerui.DefaultLocalNameDockerfile: nullDef.ToPB(), + }, + } + + _, err := client.Solve(ctx, sr) + if err != nil { + return fmt.Errorf("target %q: %w", t, err) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + return gwclient.NewResult(), nil +} + +var nullDockerfile = llb.Scratch().File( + llb.Mkfile("Dockerfile", 0o644, []byte("null")), +) + +func buildFrontend(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { + bctx, err := llb.Local(".").Marshal(ctx) + if err != nil { + return nil, err + } + + dt, err := os.ReadFile("Dockerfile") + if err != nil { + return nil, errors.Wrap(err, "error reading Dockerfile") + } + + dockerfile, err := llb.Scratch().File( + llb.Mkfile(dockerui.DefaultLocalNameDockerfile, 0o644, dt), + ).Marshal(ctx) + if err != nil { + return nil, err + } + + sr := gwclient.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendInputs: map[string]*pb.Definition{ + dockerui.DefaultLocalNameContext: bctx.ToPB(), + dockerui.DefaultLocalNameDockerfile: dockerfile.ToPB(), + }, + } + + return client.Solve(ctx, sr) +} From 346fb413be8fb496fbe846e57455101e8d753fab Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Sat, 2 Nov 2024 18:30:18 -0700 Subject: [PATCH 4/8] tests: fixtures: Use main go mod cache keys This makes sure we use the same cache keys for both the main Dockerfile and the test fixtures so that we can avoid needing to pull down a bunch of dependencies that we've already pulled down before. Signed-off-by: Brian Goff --- test/fixtures/phony.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fixtures/phony.go b/test/fixtures/phony.go index ee60f234..2ec2b537 100644 --- a/test/fixtures/phony.go +++ b/test/fixtures/phony.go @@ -13,8 +13,8 @@ import ( ) var ( - goModCache = llb.AddMount("/go/pkg/mod", llb.Scratch(), llb.AsPersistentCacheDir("dalec-go-mod-cache", llb.CacheMountShared)) - goBuildCache = llb.AddMount("/root/.cache/go-build", llb.Scratch(), llb.AsPersistentCacheDir("dalec-go-build-cache", llb.CacheMountShared)) + goModCache = llb.AddMount("/go/pkg/mod", llb.Scratch(), llb.AsPersistentCacheDir("/go/pkg/mod", llb.CacheMountShared)) + goBuildCache = llb.AddMount("/root/.cache/go-build", llb.Scratch(), llb.AsPersistentCacheDir("/root/.cache/go-build", llb.CacheMountShared)) ) func PhonyFrontend(ctx context.Context, gwc gwclient.Client) (*gwclient.Result, error) { From 835273081b732572e74ebb066ea2b27dc80166df Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 1 Nov 2024 10:48:45 -0700 Subject: [PATCH 5/8] Add test stats to GHA summary Signed-off-by: Brian Goff --- .github/workflows/ci.yml | 3 +- cmd/test2json2gha/event.go | 56 ++++++++++++++ cmd/test2json2gha/main.go | 135 ++++++++++++++++++++------------- cmd/test2json2gha/main_test.go | 26 +++++++ cmd/test2json2gha/markdown.go | 34 +++++++++ go.mod | 1 + go.sum | 4 + 7 files changed, 206 insertions(+), 53 deletions(-) create mode 100644 cmd/test2json2gha/event.go create mode 100644 cmd/test2json2gha/main_test.go create mode 100644 cmd/test2json2gha/markdown.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a76980a..c2b298ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,12 +96,11 @@ jobs: - name: Precache base images run: go run ./cmd/ci-precache - name: Run integration tests - run: go test -v -json ./test | go run ./cmd/test2json2gha + run: go test -v -json ./test | go run ./cmd/test2json2gha --slow 5s - name: dump logs if: failure() run: sudo journalctl -u docker - unit: runs-on: ubuntu-22.04 steps: diff --git a/cmd/test2json2gha/event.go b/cmd/test2json2gha/event.go new file mode 100644 index 00000000..e52128cf --- /dev/null +++ b/cmd/test2json2gha/event.go @@ -0,0 +1,56 @@ +package main + +import ( + "os" + "time" + + "github.com/pkg/errors" +) + +// TestEvent is the go test2json event data structure we receive from `go test` +// This is defined in https://pkg.go.dev/cmd/test2json#hdr-Output_Format +type TestEvent struct { + Time time.Time + Action string + Package string + Test string + Elapsed float64 // seconds + Output string +} + +// TestResult is where we collect all the data about a test +type TestResult struct { + output *os.File + failed bool + pkg string + name string + elapsed float64 + skipped bool +} + +func (r *TestResult) Close() { + r.output.Close() +} + +func handlEvent(te *TestEvent, tr *TestResult) error { + if te.Output != "" { + _, err := tr.output.Write([]byte(te.Output)) + if err != nil { + return errors.Wrap(err, "error collecting test event output") + } + } + + tr.pkg = te.Package + tr.name = te.Test + if te.Elapsed > 0 { + tr.elapsed = te.Elapsed + } + + if te.Action == "fail" { + tr.failed = true + } + if te.Action == "skip" { + tr.skipped = true + } + return nil +} diff --git a/cmd/test2json2gha/main.go b/cmd/test2json2gha/main.go index b4679535..1badfba8 100644 --- a/cmd/test2json2gha/main.go +++ b/cmd/test2json2gha/main.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "flag" "fmt" "io" "log/slog" @@ -15,6 +16,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/vearutop/dynhist-go" ) func main() { @@ -23,6 +25,11 @@ func main() { panic(err) } + var slowThreshold time.Duration + flag.DurationVar(&slowThreshold, "slow", 500*time.Millisecond, "Threshold to mark test as slow") + + flag.Parse() + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) slog.SetDefault(logger) @@ -38,7 +45,7 @@ func main() { cleanup := func() { os.RemoveAll(tmp) } - anyFail, err := do(os.Stdin, os.Stdout, mod) + anyFail, err := do(os.Stdin, os.Stdout, mod, slowThreshold) if err != nil { fmt.Fprintf(os.Stderr, "%+v", err) cleanup() @@ -53,31 +60,7 @@ func main() { } } -// TestEvent is the go test2json event data structure we receive from `go test` -// This is defined in https://pkg.go.dev/cmd/test2json#hdr-Output_Format -type TestEvent struct { - Time time.Time - Action string - Package string - Test string - Elapsed float64 // seconds - Output string -} - -// TestResult is where we collect all the data about a test -type TestResult struct { - output *os.File - failed bool - pkg string - name string - elapsed float64 -} - -func (r *TestResult) Close() { - r.output.Close() -} - -func do(in io.Reader, out io.Writer, modName string) (bool, error) { +func do(in io.Reader, out io.Writer, modName string, slowThreshold time.Duration) (bool, error) { dec := json.NewDecoder(in) te := &TestEvent{} @@ -128,14 +111,36 @@ func do(in io.Reader, out io.Writer, modName string) (bool, error) { } buf := bufio.NewWriter(out) - var anyFail bool + + summaryF := getSummaryFile() + defer summaryF.Close() + + var failCount, skipCount int + var elapsed float64 + + failBuf := bytes.NewBuffer(nil) + hist := &dynhist.Collector{ + PrintSum: true, + WeightFunc: dynhist.ExpWidth(1.2, 0.9), + BucketsLimit: 10, + } + + slowBuf := bytes.NewBuffer(nil) + slow := slowThreshold.Seconds() for _, tr := range outs { + if tr.skipped { + skipCount++ + } + if tr.failed { - anyFail = true + failCount++ } - if err := writeResult(tr, buf, modName); err != nil { + hist.Add(tr.elapsed) + elapsed += tr.elapsed + + if err := writeResult(tr, buf, failBuf, slowBuf, slow, modName); err != nil { slog.Error("Error writing result", "error", err) continue } @@ -145,28 +150,42 @@ func do(in io.Reader, out io.Writer, modName string) (bool, error) { } } - return anyFail, nil -} + fmt.Fprintln(summaryF, "## Test metrics") + separator := strings.Repeat(" ", 4) + fmt.Fprintln(summaryF, mdBold("Skipped:"), skipCount, separator, mdBold("Failed:"), failCount, separator, mdBold("Total:"), len(outs), separator, mdBold("Elapsed:"), fmt.Sprintf("%.3fs", elapsed)) -func handlEvent(te *TestEvent, tr *TestResult) error { - if te.Output != "" { - _, err := tr.output.Write([]byte(te.Output)) - if err != nil { - return errors.Wrap(err, "error collecting test event output") - } + fmt.Fprintln(summaryF, mdPreformat(hist.String())) + + if failBuf.Len() > 0 { + fmt.Fprintln(summaryF, "## Failures") + fmt.Fprintln(summaryF, failBuf.String()) + } + + if slowBuf.Len() > 0 { + fmt.Fprintln(summaryF, "## Slow Tests") + fmt.Fprintln(summaryF, slowBuf.String()) } - tr.pkg = te.Package - tr.name = te.Test - tr.elapsed = te.Elapsed + return failCount > 0, nil +} + +func (c *nopWriteCloser) Close() error { return nil } - if te.Action == "fail" { - tr.failed = true +func getSummaryFile() io.WriteCloser { + v := os.Getenv("GITHUB_STEP_SUMMARY") + if v == "" { + return &nopWriteCloser{io.Discard} } - return nil + + f, err := os.OpenFile(v, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + slog.Error("Error opening step summary file", "error", err) + return &nopWriteCloser{io.Discard} + } + return f } -func writeResult(tr *TestResult, out io.Writer, modName string) error { +func writeResult(tr *TestResult, out, failBuf, slowBuf io.Writer, slow float64, modName string) error { if tr.name == "" { return nil } @@ -190,31 +209,44 @@ func writeResult(tr *TestResult, out io.Writer, modName string) error { prefix = "\u274c " } - dur := time.Duration(tr.elapsed * float64(time.Second)) - fmt.Fprintln(out, "::group::"+prefix+group, dur) + fmt.Fprintf(out, "::group::%s %.3fs\n", prefix+group, tr.elapsed) defer func() { fmt.Fprintln(out, "::endgroup::") }() - dt, err := io.ReadAll(tr.output) - if err != nil { - return fmt.Errorf("error reading test output: %w", err) + var rdr io.Reader = tr.output + + if tr.elapsed > slow { + buf := bytes.NewBuffer(nil) + rdr = io.TeeReader(rdr, buf) + defer func() { + if buf.Len() > 0 { + fmt.Fprintln(slowBuf, mdLog(fmt.Sprintf("%s %.3fs", tr.name, tr.elapsed), buf)) + } + }() } if !tr.failed { - if _, err := out.Write(dt); err != nil { + if _, err := io.Copy(out, rdr); err != nil { return fmt.Errorf("error writing test output to output stream: %w", err) } return nil } - scanner := bufio.NewScanner(bytes.NewReader(dt)) + failLog := bytes.NewBuffer(nil) + rdr = io.TeeReader(rdr, failLog) + defer func() { + fmt.Fprintln(failBuf, mdLog(tr.name+fmt.Sprintf(" %3.fs", tr.elapsed), failLog)) + }() + + scanner := bufio.NewScanner(rdr) var ( file, line string ) buf := bytes.NewBuffer(nil) + for scanner.Scan() { txt := scanner.Text() f, l, ok := getTestOutputLoc(txt) @@ -235,6 +267,7 @@ func writeResult(tr *TestResult, out io.Writer, modName string) error { file = filepath.Join(pkg, file) fmt.Fprintf(out, "::error file=%s,line=%s::%s\n", file, line, buf) + return nil } diff --git a/cmd/test2json2gha/main_test.go b/cmd/test2json2gha/main_test.go new file mode 100644 index 00000000..27af1fbd --- /dev/null +++ b/cmd/test2json2gha/main_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + "time" +) + +func TestFail(t *testing.T) { + t.Log("This is for checking what a test failure looks like") + t.Fatal("This is a failure") +} + +func TestAnotherFail(t *testing.T) { + t.Log("This is for checking what a test failure looks like") + t.Fatal("This is yet another failure!") +} + +func TestSlow(t *testing.T) { + time.Sleep(10 * time.Second) + t.Log("This is a slow test!") +} + +func TestAnothrSlow(t *testing.T) { + time.Sleep(5 * time.Second) + t.Log("This is yet another slow test!") +} diff --git a/cmd/test2json2gha/markdown.go b/cmd/test2json2gha/markdown.go new file mode 100644 index 00000000..4f85e8e5 --- /dev/null +++ b/cmd/test2json2gha/markdown.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "io" + "strings" +) + +func mdBold(v string) string { + return "**" + v + "**" +} + +func mdDetails(s string) string { + return fmt.Sprintf("\n
\n%s\n
\n", s) +} + +func mdSummary(s string) string { + return "" + s + "\n" +} + +func mdPreformat(s string) string { + return fmt.Sprintf("\n```\n%s\n```\n", s) +} + +type nopWriteCloser struct { + io.Writer +} + +func mdLog(head string, content fmt.Stringer) string { + sb := &strings.Builder{} + sb.WriteString(mdSummary(head)) + sb.WriteString(mdPreformat(content.String())) + return mdDetails(sb.String()) +} diff --git a/go.mod b/go.mod index 217a7fc4..0b66f678 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c + github.com/vearutop/dynhist-go v1.2.3 go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/sdk v1.21.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 diff --git a/go.sum b/go.sum index cdbcf8a9..aa6b9758 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1 github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bool64/dev v0.2.28 h1:6ayDfrB/jnNr2iQAZHI+uT3Qi6rErSbJYQs1y8rSrwM= +github.com/bool64/dev v0.2.28/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -192,6 +194,8 @@ github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Q github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +github.com/vearutop/dynhist-go v1.2.3 h1:EIMWszSDm6b7zmqySgx8zW2qNctE3IXUJggGlDFwJBE= +github.com/vearutop/dynhist-go v1.2.3/go.mod h1:liiiYiwAi8ixC3DbkxooEhASTF6ysJSXy+piCrBtxEg= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From d2161dc3e3c2a8f32b90a34c62fd8640bf170ae4 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 4 Nov 2024 14:23:02 -0800 Subject: [PATCH 6/8] CI: Get tracing reports from integration tests Signed-off-by: Brian Goff --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2b298ae..dcd52931 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,11 +86,36 @@ jobs: - name: Expose GitHub tokens for caching uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4 # v3.0.0 + - name: Setup jaeger + run: | + set -e + docker run -d --net=host --restart=always --name jaeger -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.62.0 + docker0_ip="$(ip -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')" + echo "OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318" >> "${GITHUB_ENV}" + echo "OTEL_SERVICE_NAME=dalec-integration-test" >> "${GITHUB_ENV}" + + tmp="$(mktemp)" + echo "Environment=\"OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318\"" > "${tmp}" + sudo mkdir -p /etc/systemd/system/docker.service.d + sudo mkdir -p /etc/systemd/system/containerd.service.d + sudo cp "${tmp}" /etc/systemd/system/docker.service.d/otlp.conf + sudo cp "${tmp}" /etc/systemd/system/containerd.service.d/otlp.conf + + sudo systemctl daemon-reload + sudo systemctl restart containerd + sudo systemctl restart docker + # Tests currently require buildkit v0.12.0 or higher # The version of buildkit builtin to moby currently (v24) is too old # So we need to setup a custom builder. - name: Set up builder uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + with: + driver-opts: | + network=host + env.OTLP_EXPORTER_OPTELP_ENDPOINT=http://127.0.0.1:4318 + env.OTEL_SERVICE_NAME=buildkitd + - name: download deps run: go mod download - name: Precache base images @@ -100,6 +125,22 @@ jobs: - name: dump logs if: failure() run: sudo journalctl -u docker + - name: Get traces + if: always() + run: | + set -ex + mkdir -p /tmp/reports + curl -sSLf localhost:16686/api/traces?service=${OTEL_SERVICE_NAME} > /tmp/reports/jaeger-tests.json + curl -sSLf localhost:16686/api/traces?service=containerd > /tmp/reports/jaeger-containerd.json + curl -sSLf localhost:16686/api/traces?service=buildkitd > /tmp/reports/jaeger-buildkitd.json + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-reports + path: /tmp/reports/* + retention-days: 1 + unit: runs-on: ubuntu-22.04 From a9c74ce6e1a1a6623ea1780ed1d887729ba9ba4d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 4 Nov 2024 16:22:16 -0800 Subject: [PATCH 7/8] CI: force integration tests to run in parallel --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcd52931..8c2cfaec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: - name: Precache base images run: go run ./cmd/ci-precache - name: Run integration tests - run: go test -v -json ./test | go run ./cmd/test2json2gha --slow 5s + run: go test -p 16 -v -json ./test | go run ./cmd/test2json2gha --slow 5s - name: dump logs if: failure() run: sudo journalctl -u docker From d0a6da17d68238c95eae92210b34f420b31ee9b8 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 8 Nov 2024 12:03:11 -0800 Subject: [PATCH 8/8] test: Disable GH cache --- test/testenv/buildx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testenv/buildx.go b/test/testenv/buildx.go index 8d038beb..45681d62 100644 --- a/test/testenv/buildx.go +++ b/test/testenv/buildx.go @@ -290,7 +290,7 @@ func (b *BuildxEnv) RunTest(ctx context.Context, t *testing.T, f TestFunc, opts var so client.SolveOpt withProjectRoot(t, &so) - withGHCache(t, &so) + // withGHCache(t, &so) withResolveLocal(&so) _, err = c.Build(ctx, so, "", func(ctx context.Context, gwc gwclient.Client) (*gwclient.Result, error) {