tests: Fix some missing parallel calls and missing test spans

.github/workflows/ci.yml

@@ -86,18 +86,60 @@ jobs:
 
       - name: Expose GitHub tokens for caching
         uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4 # v3.0.0
+      - name: Setup jaeger
+        run: |
+          set -e
+          docker run -d --net=host --restart=always --name jaeger -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.62.0
+          docker0_ip="$(ip -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')"
+          echo "OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318" >> "${GITHUB_ENV}"
+          echo "OTEL_SERVICE_NAME=dalec-integration-test" >> "${GITHUB_ENV}"
+
+          tmp="$(mktemp)"
+          echo "Environment=\"OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318\"" > "${tmp}"
+          sudo mkdir -p /etc/systemd/system/docker.service.d
+          sudo mkdir -p /etc/systemd/system/containerd.service.d
+          sudo cp "${tmp}" /etc/systemd/system/docker.service.d/otlp.conf
+          sudo cp "${tmp}" /etc/systemd/system/containerd.service.d/otlp.conf
+
+          sudo systemctl daemon-reload
+          sudo systemctl restart containerd
+          sudo systemctl restart docker
+
       # Tests currently require buildkit v0.12.0 or higher
       # The version of buildkit builtin to moby currently (v24) is too old
       # So we need to setup a custom builder.
       - name: Set up builder
         uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
+        with:
+          driver-opts: |
+            network=host
+            env.OTLP_EXPORTER_OPTELP_ENDPOINT=http://127.0.0.1:4318
+            env.OTEL_SERVICE_NAME=buildkitd
+
       - name: download deps
         run: go mod download
+      - name: Precache base images
+        run: go run ./cmd/ci-precache
       - name: Run integration tests
-        run: go test -v -json ./test | go run ./cmd/test2json2gha
+        run: go test -p 16 -v -json ./test | go run ./cmd/test2json2gha --slow 5s
       - name: dump logs
         if: failure()
         run: sudo journalctl -u docker
+      - name: Get traces
+        if: always()
+        run: |
+          set -ex
+          mkdir -p /tmp/reports
+          curl -sSLf localhost:16686/api/traces?service=${OTEL_SERVICE_NAME} > /tmp/reports/jaeger-tests.json
+          curl -sSLf localhost:16686/api/traces?service=containerd > /tmp/reports/jaeger-containerd.json
+          curl -sSLf localhost:16686/api/traces?service=buildkitd > /tmp/reports/jaeger-buildkitd.json
+      - name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-reports
+          path: /tmp/reports/*
+          retention-days: 1
 
 
   unit:

cmd/ci-precache/main.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/Azure/dalec/frontend/azlinux"
+	"github.com/Azure/dalec/frontend/jammy"
+	"github.com/Azure/dalec/frontend/windows"
+	"github.com/Azure/dalec/test/testenv"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/dockerui"
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/util/bklog"
+	"github.com/moby/buildkit/util/progress/progressui"
+	"github.com/moby/patternmatcher/ignorefile"
+	"github.com/pkg/errors"
+	"github.com/tonistiigi/fsutil"
+	"golang.org/x/sync/errgroup"
+)
+
+func main() {
+	ctx, done := signal.NotifyContext(context.Background(), os.Interrupt)
+	go func() {
+		<-ctx.Done()
+		// The context was cancelled due to interupt
+		// This _should_ trigger builds to cancel naturally and exit the program,
+		// but in some cases it may not (due to timing, bugs in buildkit, uninteruptable operations, etc.).
+		// Cancel our signal handler so the normal handler takes over from here.
+		// This allows subsequent interupts to use the default behavior (exit the program)
+		done()
+
+		<-time.After(30 * time.Second)
+		fmt.Fprintln(os.Stderr, "Timeout waiting for builds to cancel after interupt")
+		os.Exit(int(syscall.SIGINT))
+	}()
+
+	bklog.G(ctx).Logger.SetOutput(os.Stderr)
+
+	if err := run(ctx); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context) (retErr error) {
+	env := testenv.New()
+
+	c, err := env.Buildkit(ctx)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+
+	ch := make(chan *client.SolveStatus)
+	d, err := progressui.NewDisplay(os.Stderr, progressui.AutoMode)
+	if err != nil {
+		return err
+	}
+
+	chErr := make(chan error, 1)
+	go func() {
+		warnings, err := d.UpdateFrom(ctx, ch)
+		for _, w := range warnings {
+			bklog.G(ctx).Warn(string(w.Short))
+		}
+		chErr <- err
+	}()
+
+	defer func() {
+		e := <-chErr
+		if retErr == nil {
+			retErr = e
+		}
+	}()
+
+	localFS, err := fsutil.NewFS(".")
+	if err != nil {
+		return err
+	}
+
+	ignoreF, err := localFS.Open(".dockerignore")
+	if err != nil {
+		return err
+	}
+	defer ignoreF.Close()
+	excludes, err := ignorefile.ReadAll(ignoreF)
+	if err != nil {
+		return err
+	}
+
+	localFS, err = fsutil.NewFilterFS(localFS, &fsutil.FilterOpt{
+		ExcludePatterns: excludes,
+	})
+	if err != nil {
+		return err
+	}
+
+	so := client.SolveOpt{
+		LocalMounts: map[string]fsutil.FS{
+			".": localFS,
+		},
+	}
+	_, err = c.Build(ctx, so, "", build, ch)
+	return err
+}
+
+func build(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
+	res, err := buildFrontend(ctx, client)
+	if err != nil {
+		return nil, err
+	}
+
+	ref, err := res.SingleRef()
+	if err != nil {
+		return nil, err
+	}
+
+	st, err := ref.ToState()
+	if err != nil {
+		return nil, err
+	}
+
+	frontend, err := st.Marshal(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	workers := []string{
+		azlinux.Mariner2TargetKey,
+		azlinux.AzLinux3TargetKey,
+		windows.DefaultTargetKey,
+		jammy.DefaultTargetKey,
+	}
+
+	eg, ctx := errgroup.WithContext(ctx)
+
+	metaDt, err := json.Marshal(res.Metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	nullDef, err := nullDockerfile.Marshal(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	id := identity.NewID()
+	for _, t := range workers {
+		eg.Go(func() error {
+			sr := gwclient.SolveRequest{
+				Evaluate: true,
+				Frontend: "gateway.v0",
+				FrontendOpt: map[string]string{
+					"source":               id,
+					"target":               t + "/worker",
+					"context:" + id:        "input:" + id,
+					"input-metadata:" + id: string(metaDt),
+				},
+				FrontendInputs: map[string]*pb.Definition{
+					id:                                  frontend.ToPB(),
+					dockerui.DefaultLocalNameDockerfile: nullDef.ToPB(),
+				},
+			}
+
+			_, err := client.Solve(ctx, sr)
+			if err != nil {
+				return fmt.Errorf("target %q: %w", t, err)
+			}
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return gwclient.NewResult(), nil
+}
+
+var nullDockerfile = llb.Scratch().File(
+	llb.Mkfile("Dockerfile", 0o644, []byte("null")),
+)
+
+func buildFrontend(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
+	bctx, err := llb.Local(".").Marshal(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	dt, err := os.ReadFile("Dockerfile")
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading Dockerfile")
+	}
+
+	dockerfile, err := llb.Scratch().File(
+		llb.Mkfile(dockerui.DefaultLocalNameDockerfile, 0o644, dt),
+	).Marshal(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	sr := gwclient.SolveRequest{
+		Frontend: "dockerfile.v0",
+		FrontendInputs: map[string]*pb.Definition{
+			dockerui.DefaultLocalNameContext:    bctx.ToPB(),
+			dockerui.DefaultLocalNameDockerfile: dockerfile.ToPB(),
+		},
+	}
+
+	return client.Solve(ctx, sr)
+}

cmd/test2json2gha/event.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// TestEvent is the go test2json event data structure we receive from `go test`
+// This is defined in https://pkg.go.dev/cmd/test2json#hdr-Output_Format
+type TestEvent struct {
+	Time    time.Time
+	Action  string
+	Package string
+	Test    string
+	Elapsed float64 // seconds
+	Output  string
+}
+
+// TestResult is where we collect all the data about a test
+type TestResult struct {
+	output  *os.File
+	failed  bool
+	pkg     string
+	name    string
+	elapsed float64
+	skipped bool
+}
+
+func (r *TestResult) Close() {
+	r.output.Close()
+}
+
+func handlEvent(te *TestEvent, tr *TestResult) error {
+	if te.Output != "" {
+		_, err := tr.output.Write([]byte(te.Output))
+		if err != nil {
+			return errors.Wrap(err, "error collecting test event output")
+		}
+	}
+
+	tr.pkg = te.Package
+	tr.name = te.Test
+	if te.Elapsed > 0 {
+		tr.elapsed = te.Elapsed
+	}
+
+	if te.Action == "fail" {
+		tr.failed = true
+	}
+	if te.Action == "skip" {
+		tr.skipped = true
+	}
+	return nil
+}