added TestWithSpec to magefile

.buildkite/scripts/agentbeat/prepare_env.py

@@ -105,8 +105,8 @@ def add_to_path(filepath):
     match = re.match(pattern, filepath)
     if match:
         path = f'../build/distributions/{match.group(1)}/agentbeat'
-        log('--- AGENTBEAT_PATH: ' + path)
-        os.environ['AGENTBEAT_PATH'] = path
+        log("--- PATH: " + str(path))
+        os.environ['AGENTBEAT_PATH'] = str(path)
     else:
         log_err("No agentbeat executable found")
         exit(1)

.buildkite/x-pack/pipeline.xpack.agentbeat.yml

@@ -12,7 +12,7 @@ env:
 
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
-  AGENTBEAT_SPEC: "../agentbeat.spec.yml"
+  AGENTBEAT_SPEC: "./agentbeat.spec.yml"
 
 steps:
 #  - group: "Check/Update"
@@ -109,9 +109,10 @@ steps:
           PLATFORM: "linux/amd64"
 #        depends_on:
 #          - agentbeat-package-linux
-        command: |
-          set -euo pipefail
-          ./.buildkite/scripts/agentbeat/prepare_env.py
+        commands:
+          - ./.buildkite/scripts/agentbeat/prepare_env.py
+          - cd x-pack/agentbeat
+          - mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -125,6 +126,8 @@ steps:
 #          - agentbeat-package-linux
         commands:
           - ./.buildkite/scripts/agentbeat/prepare_env.py
+          - cd x-pack/agentbeat
+          - mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"

dev-tools/mage/parse_spec.go → dev-tools/mage/spec.go

@@ -21,6 +21,7 @@ import (
 	"gopkg.in/yaml.v2"
 	"log"
 	"os"
+	"strings"
 )
 
 type spec struct {
@@ -34,13 +35,17 @@ type input struct {
 	Command     command
 }
 
+func (i *input) GetCommand() string {
+	return strings.Join(i.Command.Args, " ")
+}
+
 type command struct {
 	Name string
 	Args []string
 }
 
-// ParseSpec parses agent.beat.spec.yml and generates test command
-func ParseSpec() {
+// SpecCommands parses agent.beat.spec.yml and collects commands for tests
+func SpecCommands() []string {
 	specPath := os.Getenv("AGENTBEAT_SPEC")
 	if specPath == "" {
 		log.Fatal("AGENTBEAT_SPEC is not defined")
@@ -51,20 +56,26 @@ func ParseSpec() {
 		log.Fatal("PLATFORM is not defined")
 	}
 
-	spec, err := parseToObj()
-	if err != nil {
-		log.Fatalf("Error parsing agentbeat.spec.yml: %v", err)
-	}
+	spec, _ := parseToObj(specPath)
 
-	inputList := filter(spec.Inputs, func(input input) bool {
+	filteredInputs := filter(spec.Inputs, func(input input) bool {
 		return contains(input.Platforms, platform)
 	})
 
-	log.Print(inputList)
+	commands := make(map[string]interface{})
+	for _, i := range filteredInputs {
+		commands[i.GetCommand()] = nil
+	}
+	keys := make([]string, 0, len(commands))
+	for k := range commands {
+		keys = append(keys, k)
+	}
+
+	return keys
 }
 
-func parseToObj() (spec, error) {
-	specFile, err := os.ReadFile("../agentbeat.spec.yml")
+func parseToObj(path string) (spec, error) {
+	specFile, err := os.ReadFile(path)
 	if err != nil {
 		log.Fatalf("Error opening agentbeat.spec.yml: %v", err)
 		return spec{}, err

x-pack/agentbeat/magefile.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"time"
 
@@ -142,7 +143,7 @@ func Package() error {
 	return nil
 }
 
-// TestPackages tests the generated packages (i.e. file modes, owners, groups).
+// TestPackages tests the generated packages (i.agentbeatCmd. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()
 }
@@ -214,6 +215,65 @@ func PythonIntegTest(ctx context.Context) error {
 	return devtools.PythonIntegTestFromHost(devtools.DefaultPythonTestIntegrationFromHostArgs())
 }
 
-func TestWithSpecs(ctx context.Context) {
-	mg.Deps(devtools.ParseSpec)
+// TestWithSpec executes unique commands from agentbeat.spec.yml and validates that app haven't exited with non-zero
+func TestWithSpec(ctx context.Context) {
+	var commands = devtools.SpecCommands()
+
+	agentbeatPath := os.Getenv("AGENTBEAT_PATH")
+	fmt.Printf("--- AGENTBEAT_PATH: %s", agentbeatPath)
+
+	cmdResults := make(map[string]bool)
+
+	for _, command := range commands {
+		cmdResults[command] = agentbeatCmd(agentbeatPath, command)
+	}
+
+	hasFailures := false
+	for cmd, res := range cmdResults {
+		if res {
+			fmt.Printf("Command [%s] succeeded", cmd)
+		} else {
+			fmt.Printf("Command [%s] failed", cmd)
+			hasFailures = true
+		}
+	}
+
+	if hasFailures {
+		fmt.Printf("Some inputs failed. Exiting with error")
+		os.Exit(1)
+	}
+}
+
+func agentbeatCmd(agentbeatPath string, command string) bool {
+	cmd := exec.Command(agentbeatPath, command)
+	fmt.Printf("Running command: %v", cmd)
+
+	if err := cmd.Start(); err != nil {
+		_ = fmt.Errorf("failed to start command: %v", err)
+	}
+
+	defer func() {
+		if err := cmd.Process.Kill(); err != nil {
+			_ = fmt.Errorf("failed to kill process: %v", err)
+		} else {
+			_ = fmt.Errorf("command process killed")
+		}
+	}()
+
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+	timeout := 2 * time.Second
+	deadline := time.After(timeout)
+
+	select {
+	case err := <-done:
+		_ = fmt.Errorf("command exited before %s: %v", timeout.String(), err)
+		return false
+
+	case <-deadline:
+		_ = fmt.Errorf("%s", cmd.Stdout)
+		return true
+	}
 }