From 54f329661a984a33cd55ff1aa10f39969e564b7b Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 23 Sep 2024 12:42:01 +0300 Subject: [PATCH 01/25] Go-based changeset definition for MODULE (#40238) Added MODULE env var definition within mage (cherry picked from commit cc0bda76881551540d8434b1b9e5441cc039935e) # Conflicts: # .buildkite/heartbeat/heartbeat-pipeline.yml # .buildkite/x-pack/pipeline.xpack.auditbeat.yml # .buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml # .buildkite/x-pack/pipeline.xpack.metricbeat.yml # .buildkite/x-pack/pipeline.xpack.winlogbeat.yml # x-pack/auditbeat/magefile.go --- .buildkite/heartbeat/heartbeat-pipeline.yml | 43 +++-- .buildkite/metricbeat/pipeline.yml | 17 +- .buildkite/scripts/changesets.psm1 | 64 ------- .../x-pack/pipeline.xpack.auditbeat.yml | 17 +- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 10 +- .../x-pack/pipeline.xpack.metricbeat.yml | 24 +-- .../x-pack/pipeline.xpack.winlogbeat.yml | 12 +- dev-tools/mage/module_changeset.go | 164 ++++++++++++++++++ metricbeat/magefile.go | 8 + x-pack/auditbeat/magefile.go | 18 ++ x-pack/metricbeat/magefile.go | 8 + 11 files changed, 273 insertions(+), 112 deletions(-) delete mode 100644 .buildkite/scripts/changesets.psm1 create mode 100644 dev-tools/mage/module_changeset.go diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index ea73af181e3..7ff598546f2 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -113,17 +113,33 @@ steps: - github_commit_status: context: "heartbeat: Win 2019 Unit Tests" +<<<<<<< HEAD - label: ":ubuntu: Heartbeat: Go (Module) Integration Tests" +======= + - label: ":windows: Heartbeat: Win 2022 Unit Test" + key: "windows-2022" command: | - set -euo pipefail - echo "~~~ Installing @elastic/synthetics" - npm install -g @elastic/synthetics - - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet heartbeat + Set-Location -Path heartbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_type: "pd-ssd" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "heartbeat: Win 2022 Unit Tests" - echo "~~~ Running tests" + # Heartbeat has no modules + - label: ":ubuntu: Heartbeat: Go Integration Tests" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) + command: | cd heartbeat mage -v goIntegTest retry: @@ -140,14 +156,13 @@ steps: - github_commit_status: context: "heartbeat: Go Integration Tests" +<<<<<<< HEAD - label: ":ubuntu: Heartbeat: Python (Module) Integration Tests" +======= + # Heartbeat has no modules yet + - label: ":ubuntu: Heartbeat: Python Integration Tests" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet heartbeat - - echo "~~~ Running tests" cd heartbeat mage pythonIntegTest retry: diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 9be0612101a..71bd8d2080a 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -20,6 +20,9 @@ env: K8S_VERSION: "v1.21.1" ASDF_KIND_VERSION: "0.20.0" + # Module Tests + BEAT_PATH: "metricbeat" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -79,14 +82,10 @@ steps: - github_commit_status: context: "metricbeat: Ubuntu x86_64 Unit Tests" - - label: ":ubuntu: Metricbeat: Go Integration Tests" + - label: ":ubuntu: Metricbeat: Go Integration Tests (Module)" key: "mandatory-int-test" command: | set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" export KUBECONFIG="$$PWD/kubecfg" cd metricbeat @@ -103,16 +102,12 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Go Integration Tests" + context: "metricbeat: Go Integration Tests (Module)" - - label: ":ubuntu: Metricbeat: Python Integration Tests" + - label: ":ubuntu: Metricbeat: Python Integration Tests (Module)" key: "mandatory-python-int-test" command: | set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" export KUBECONFIG="$$PWD/kubecfg" cd metricbeat diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 deleted file mode 100644 index 10e4d31a8b6..00000000000 --- a/.buildkite/scripts/changesets.psm1 +++ /dev/null @@ -1,64 +0,0 @@ -function ArePathsChanged($patterns) { - $changedlist = @() - foreach ($pattern in $patterns) { - $changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch - if ($changedFiles) { - $changedlist += $changedFiles - } - } - if ($changedlist) { - Write-Host "--- Files changed: $changedlist" - return $true - } - else { - Write-Host "--- No files changed within specified changeset: $patterns" - return $false - } -} - -function AreChangedOnlyPaths($patterns) { - $changedFiles = & git diff --name-only "HEAD@{1}" HEAD - Write-Host "--- Git Diff result:" - Write-Host "$changedFiles" - - $matchedFiles = @() - foreach ($pattern in $patterns) { - $matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch - if ($matched) { - $matchedFiles += $matched - } - } - if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) { - return $true - } - return $false -} - -# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument). -# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files. -# `MODULE` will empty if no changes apply. -function DefineModuleFromTheChangeSet($projectPath) { - $projectPathTransformed = $projectPath -replace '/', '\\' - $projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$" - $exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - - $changedModules = '' - - $moduleDirs = Get-ChildItem -Directory "$projectPath\module" - foreach($moduleDir in $moduleDirs) { - if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) { - if(!$changedModules) { - $changedModules = $moduleDir.Name - } - else { - $changedModules += ',' + $moduleDir.Name - } - } - } - - if ($changedModules) { - $env:MODULE = $changedModules - Write-Output "~~~ Set env var MODULE to [$env:MODULE]" - Write-Output "~~~ Resuming commands" - } -} diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 7f96df6b01b..0f889b8cf18 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -25,6 +25,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module tests + BEAT_PATH: "x-pack/auditbeat" + steps: - group: "Check/Update" key: "x-pack-auditbeat-check-update" @@ -58,6 +61,7 @@ steps: key: "x-pack-auditbeat-mandatory-tests" steps: +<<<<<<< HEAD - label: ":ubuntu: x-pack/auditbeat: Ubuntu arm64 Unit Tests" command: | cd x-pack/auditbeat @@ -78,12 +82,11 @@ steps: - label: ":ubuntu: x-pack/auditbeat: Build (MODULE) Tests" key: "mandatory-build-test" +======= + - label: ":ubuntu: x-pack/auditbeat: Build Tests (Module)" + key: "mandatory-linux-unit-test" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/auditbeat - echo "~~~ Running tests" cd x-pack/auditbeat mage update build test retry: @@ -98,7 +101,11 @@ steps: - "x-pack/auditbeat/build/*.json" notify: - github_commit_status: +<<<<<<< HEAD context: "x-pack/auditbeat: Build (MODULE) Tests" +======= + context: "x-pack/auditbeat: Build Tests (Module)" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) - label: ":windows: x-pack/auditbeat: Win 2019 Unit Tests" command: | diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 123ba2e0564..4242e448e02 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -68,14 +68,14 @@ steps: - github_commit_status: context: "x-pack/dockerlogbeat: Ubuntu Unit Tests" +<<<<<<< HEAD - label: ":ubuntu: x-pack/dockerlogbeat: Go (Module) Integration Tests" +======= + # x-pack/dockerlogbeat has no modules yet + - label: ":ubuntu: x-pack/dockerlogbeat: Go Integration Tests" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) key: "mandatory-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/dockerlogbeat - echo "~~~ Running tests" cd x-pack/dockerlogbeat mage -v goIntegTest retry: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 2f756dc692f..65197ed2ffb 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -25,6 +25,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module tests + BEAT_PATH: "x-pack/metricbeat" + steps: - group: "Check/Update" key: "x-pack-metricbeat-check-update" @@ -77,15 +80,20 @@ steps: - github_commit_status: context: "x-pack/metricbeat: Ubuntu x86_64 Unit Tests" - - label: ":ubuntu: x-pack/metricbeat: Go (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/metricbeat: Go Integration Tests (Module)" key: "mandatory-int-test" command: | +<<<<<<< HEAD set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat echo "~~~ Running tests" cd x-pack/metricbeat && mage -v goIntegTest +======= + cd x-pack/metricbeat + mage goIntegTest +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) retry: automatic: - limit: 1 @@ -98,17 +106,13 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Go (MODULE) Integration Tests" + context: "x-pack/metricbeat: Go Integration Tests (Module)" - - label: ":ubuntu: x-pack/metricbeat: Python (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/metricbeat: Python Integration Tests (Module)" key: "mandatory-python-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests" - cd x-pack/metricbeat && mage pythonIntegTest + cd x-pack/metricbeat + mage pythonIntegTest retry: automatic: - limit: 1 @@ -121,7 +125,7 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Python (MODULE) Integration Tests" + context: "x-pack/metricbeat: Python Integration Tests (Module)" - label: ":windows: x-pack/metricbeat: Win 2019 Unit Tests" command: | diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index f51bece7d94..b4b0cf8f116 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -49,12 +49,14 @@ steps: - group: "Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: +<<<<<<< HEAD - label: ":windows: x-pack/winlogbeat Win-2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" +======= + - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests" + key: "mandatory-win-2019-unit-tests" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) command: | - Import-Module ./.buildkite/scripts/changesets.psm1 - defineModuleFromTheChangeSet 'x-pack/winlogbeat' - Write-Output "~~~ Running tests" Set-Location -Path x-pack/winlogbeat mage build unitTest retry: @@ -71,7 +73,11 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: +<<<<<<< HEAD context: "x-pack/winlogbeat Win-2019 Unit (MODULE) Tests" +======= + context: "x-pack/winlogbeat Win 2019 Unit Tests" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) - label: ":windows: x-pack/winlogbeat Win-2022 Unit Tests" command: | diff --git a/dev-tools/mage/module_changeset.go b/dev-tools/mage/module_changeset.go new file mode 100644 index 00000000000..c7ffdbfd9d9 --- /dev/null +++ b/dev-tools/mage/module_changeset.go @@ -0,0 +1,164 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "fmt" + "log" + "os" + "regexp" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +// DefineModules checks which modules were changed and populates MODULE environment variable, +// so that CI would run tests only for the changed ones. +// If no modules were changed MODULE variable won't be defined. +func DefineModules() { + // If MODULE is set in Buildkite pipeline step, skip variable further definition + if os.Getenv("MODULE") != "" { + return + } + + if mg.Verbose() { + fmt.Printf("Detecting changes in modules\n") + } + + beatPath := os.Getenv("BEAT_PATH") + if beatPath == "" { + log.Fatal("BEAT_PATH is not defined") + } + + var modulePattern = fmt.Sprintf("^%s\\/module\\/([^\\/]+)\\/.*", beatPath) + + moduleRegex, err := regexp.Compile(modulePattern) + if err != nil { + log.Fatal("failed to compile regex: " + err.Error()) + } + + modules := map[string]struct{}{} + for _, line := range getDiff() { + if !shouldIgnore(line) { + matches := moduleRegex.FindStringSubmatch(line) + if len(matches) > 0 { + modules[matches[1]] = struct{}{} + } + } + } + + keys := make([]string, len(modules)) + i := 0 + for k := range modules { + keys[i] = k + i++ + } + + moduleVar := strings.Join(keys, ",") + + if moduleVar != "" { + err = os.Setenv("MODULE", moduleVar) + if err != nil { + return + } + + log.Printf("Detected changes in module(s): %s\n", moduleVar) + } else { + log.Printf("No changed modules found") + } +} + +func shouldIgnore(file string) bool { + ignoreList := []string{".asciidoc", ".png"} + for ext := range ignoreList { + if strings.HasSuffix(file, ignoreList[ext]) { + return true + } + } + return false +} + +func getDiff() []string { + commitRange := getCommitRange() + var output, err = sh.Output("git", "diff", "--name-only", commitRange) + if err != nil { + log.Fatal("git Diff failed: %w", err) + } + + printWhenVerbose("Git Diff result: %s\n", output) + + return strings.Split(output, "\n") +} + +func getFromCommit() string { + if baseBranch := os.Getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH"); baseBranch != "" { + printWhenVerbose("PR branch: %s\n", baseBranch) + + return getBranchName(baseBranch) + } + + if previousCommit := getPreviousCommit(); previousCommit != "" { + printWhenVerbose("Git from commit: %s\n", previousCommit) + + return previousCommit + } else { + commit, err := getBuildkiteCommit() + if err != nil { + log.Fatal(err) + } + printWhenVerbose("Git from commit: %s\n", commit) + + return commit + } +} + +func getPreviousCommit() string { + var output, _ = sh.Output("git", "rev-parse", "HEAD^") + printWhenVerbose("Git previous commit: %s\n", output) + + return strings.TrimSpace(output) +} + +func getCommitRange() string { + commit, err := getBuildkiteCommit() + if err != nil { + log.Fatal(err) + } + + return fmt.Sprintf("%s...%s", getFromCommit(), commit) +} + +func getBranchName(branch string) string { + return fmt.Sprintf("origin/%s", branch) +} + +func printWhenVerbose(template string, parameter string) { + if mg.Verbose() { + fmt.Printf(template, parameter) + } +} + +func getBuildkiteCommit() (string, error) { + commit := os.Getenv("BUILDKITE_COMMIT") + if commit == "" { + log.Fatal("BUILDKITE_COMMIT is not defined") + } + + return commit, nil +} diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 86245ec67f2..0cd72c8a424 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -201,6 +201,10 @@ func IntegTest() { // Use TEST_TAGS=tag1,tag2 to add additional build tags. // Use MODULE=module to run only tests for `module`. func GoIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } @@ -213,6 +217,10 @@ func GoIntegTest(ctx context.Context) error { // Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. // Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index c2700e99620..a84087ced40 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -9,8 +9,11 @@ package main import ( "fmt" + "os" "time" + "github.com/elastic/beats/v7/dev-tools/mage/target/test" + "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" @@ -24,8 +27,13 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" +<<<<<<< HEAD // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" +======= + //mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/docker" +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) ) func init() { @@ -135,6 +143,7 @@ func Dashboards() error { return devtools.KibanaDashboards(devtools.OSSBeatDir("module"), "module") } +<<<<<<< HEAD // ----------------------------------------------------------------------------- // - Install the librpm-dev package var ( @@ -230,4 +239,13 @@ func installDependencies(arch string, pkgs ...string) error { "--no-install-recommends", }, pkgs...) return sh.Run("apt-get", args...) +======= +// Test runs all available tests (unitTest + integTest) +func Test() { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + + test.Test() +>>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) } diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index c74e5876ccb..98169a4e2ad 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -216,6 +216,10 @@ func IntegTest() { // Use TEST_TAGS=tag1,tag2 to add additional build tags. // Use MODULE=module to run only tests for `module`. func GoIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } @@ -228,6 +232,10 @@ func GoIntegTest(ctx context.Context) error { // Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. // Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } From 39f9d742fa9bbac8ff2988922aa4a5fa96a5131a Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 13:59:49 +0300 Subject: [PATCH 02/25] fixed conflicts in x-pack/auditbeat --- .buildkite/x-pack/pipeline.xpack.auditbeat.yml | 11 +---------- x-pack/auditbeat/magefile.go | 13 ++----------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 0f889b8cf18..b616c0104e3 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -61,7 +61,6 @@ steps: key: "x-pack-auditbeat-mandatory-tests" steps: -<<<<<<< HEAD - label: ":ubuntu: x-pack/auditbeat: Ubuntu arm64 Unit Tests" command: | cd x-pack/auditbeat @@ -80,12 +79,8 @@ steps: - github_commit_status: context: "x-pack/auditbeat: Ubuntu arm64 Unit Tests" - - label: ":ubuntu: x-pack/auditbeat: Build (MODULE) Tests" - key: "mandatory-build-test" -======= - label: ":ubuntu: x-pack/auditbeat: Build Tests (Module)" - key: "mandatory-linux-unit-test" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) + key: "mandatory-build-test" command: | cd x-pack/auditbeat mage update build test @@ -101,11 +96,7 @@ steps: - "x-pack/auditbeat/build/*.json" notify: - github_commit_status: -<<<<<<< HEAD - context: "x-pack/auditbeat: Build (MODULE) Tests" -======= context: "x-pack/auditbeat: Build Tests (Module)" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) - label: ":windows: x-pack/auditbeat: Win 2019 Unit Tests" command: | diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index a84087ced40..683e8cb5f12 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -9,11 +9,8 @@ package main import ( "fmt" - "os" "time" - "github.com/elastic/beats/v7/dev-tools/mage/target/test" - "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" @@ -27,13 +24,8 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" -<<<<<<< HEAD // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" -======= - //mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/docker" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) ) func init() { @@ -143,7 +135,6 @@ func Dashboards() error { return devtools.KibanaDashboards(devtools.OSSBeatDir("module"), "module") } -<<<<<<< HEAD // ----------------------------------------------------------------------------- // - Install the librpm-dev package var ( @@ -239,7 +230,8 @@ func installDependencies(arch string, pkgs ...string) error { "--no-install-recommends", }, pkgs...) return sh.Run("apt-get", args...) -======= +} + // Test runs all available tests (unitTest + integTest) func Test() { if os.Getenv("CI") == "true" { @@ -247,5 +239,4 @@ func Test() { } test.Test() ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) } From 3fad3f69f44f7e0b472bde6aa2c0d95f79b99be2 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 14:19:08 +0300 Subject: [PATCH 03/25] updated imports in x-pack/auditbeat/magefile.go --- x-pack/auditbeat/magefile.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 683e8cb5f12..451bc90f5b2 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -11,6 +11,7 @@ import ( "fmt" "time" + "github.com/elastic/beats/v7/dev-tools/mage/target/test" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" @@ -24,8 +25,6 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { From b335ca438e41f71391dc8802779dac8215286f96 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 14:20:35 +0300 Subject: [PATCH 04/25] resolved conflicts && updated magefile in x-pack/winlogbeat --- .buildkite/x-pack/pipeline.xpack.winlogbeat.yml | 13 ++----------- x-pack/winlogbeat/magefile.go | 9 +++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index b4b0cf8f116..aa5b45104a1 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -49,13 +49,8 @@ steps: - group: "Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: -<<<<<<< HEAD - - label: ":windows: x-pack/winlogbeat Win-2019 Unit (MODULE) Tests" - key: "mandatory-win-2019-module-unit-tests" -======= - - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests" + - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests (Module)" key: "mandatory-win-2019-unit-tests" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -73,11 +68,7 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: -<<<<<<< HEAD - context: "x-pack/winlogbeat Win-2019 Unit (MODULE) Tests" -======= - context: "x-pack/winlogbeat Win 2019 Unit Tests" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) + context: "x-pack/winlogbeat Win 2019 Unit Tests (Module)" - label: ":windows: x-pack/winlogbeat Win-2022 Unit Tests" command: | diff --git a/x-pack/winlogbeat/magefile.go b/x-pack/winlogbeat/magefile.go index a53e56d62ea..8d463d757b9 100644 --- a/x-pack/winlogbeat/magefile.go +++ b/x-pack/winlogbeat/magefile.go @@ -48,3 +48,12 @@ func Ironbank() error { fmt.Println(">> Ironbank: this module is not subscribed to the IronBank releases.") return nil } + +// Test runs all available tests (unitTest + integTest) +func UnitTest() { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + + unittest.UnitTest() +} From 38675023d118842168dfc86ee7dab8e5fe0d7676 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 14:40:47 +0300 Subject: [PATCH 05/25] resolved conflicts && updated magefile in x-pack/metricbeat --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 65197ed2ffb..9b15e057503 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -83,17 +83,8 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Go Integration Tests (Module)" key: "mandatory-int-test" command: | -<<<<<<< HEAD - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests" - cd x-pack/metricbeat && mage -v goIntegTest -======= cd x-pack/metricbeat - mage goIntegTest ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) + mage -v goIntegTest retry: automatic: - limit: 1 From 305ec66abb7f917415e872b7819019576f3b11dc Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 14:42:09 +0300 Subject: [PATCH 06/25] resolved conflicts in x-pack/dockerlogbeat --- .buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 4242e448e02..3e6effa8e60 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -68,12 +68,8 @@ steps: - github_commit_status: context: "x-pack/dockerlogbeat: Ubuntu Unit Tests" -<<<<<<< HEAD - - label: ":ubuntu: x-pack/dockerlogbeat: Go (Module) Integration Tests" -======= # x-pack/dockerlogbeat has no modules yet - label: ":ubuntu: x-pack/dockerlogbeat: Go Integration Tests" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) key: "mandatory-int-test" command: | cd x-pack/dockerlogbeat From f20afa4e3bf5d890952c7eccad051d8eda850bf8 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 14:47:42 +0300 Subject: [PATCH 07/25] resolved conflicts in heartbeat --- .buildkite/heartbeat/heartbeat-pipeline.yml | 30 +-------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 7ff598546f2..560d5240211 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -113,32 +113,8 @@ steps: - github_commit_status: context: "heartbeat: Win 2019 Unit Tests" -<<<<<<< HEAD - - label: ":ubuntu: Heartbeat: Go (Module) Integration Tests" -======= - - label: ":windows: Heartbeat: Win 2022 Unit Test" - key: "windows-2022" - command: | - Set-Location -Path heartbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_type: "pd-ssd" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Win 2022 Unit Tests" - - # Heartbeat has no modules + # Heartbeat has no modules yet - label: ":ubuntu: Heartbeat: Go Integration Tests" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) command: | cd heartbeat mage -v goIntegTest @@ -156,12 +132,8 @@ steps: - github_commit_status: context: "heartbeat: Go Integration Tests" -<<<<<<< HEAD - - label: ":ubuntu: Heartbeat: Python (Module) Integration Tests" -======= # Heartbeat has no modules yet - label: ":ubuntu: Heartbeat: Python Integration Tests" ->>>>>>> cc0bda7688 (Go-based changeset definition for MODULE (#40238)) command: | cd heartbeat mage pythonIntegTest From e16650008a4af49e07e2f9984fd895081bfe56e6 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 15:13:06 +0300 Subject: [PATCH 08/25] added changeset definition to auditbeat --- .buildkite/auditbeat/auditbeat-pipeline.yml | 4 ---- auditbeat/magefile.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 9849109e8e4..1a9b9b687c7 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -101,10 +101,6 @@ steps: env: STACK_ENVIRONMENT: "next-major" command: | - set -euo pipefail - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet auditbeat - echo "~~~ Running tests" cd auditbeat mage pythonIntegTest retry: diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 4e595582d48..4bd5ee56997 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -22,8 +22,10 @@ package main import ( "fmt" + "os" "time" + "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" "github.com/magefile/mage/mg" auditbeat "github.com/elastic/beats/v7/auditbeat/scripts/mage" @@ -34,8 +36,6 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import - "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) @@ -159,3 +159,12 @@ func Dashboards() error { func Docs() { mg.Deps(auditbeat.ModuleDocs, auditbeat.FieldDocs) } + +// PythonIntegTest executes the python system tests in the integration +func PythinIntegTest() { + if os.Getenv("CI") == "true" && os.Getenv("STACK_ENVIRONMENT") == "next-major" { + mg.Deps(devtools.DefineModules) + } + + integtest.PythonIntegTest() +} From a19061981b68c106eb610d1088cf2cab2af3189b Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 15:25:09 +0300 Subject: [PATCH 09/25] added changeset definition to fileabeat --- .buildkite/filebeat/filebeat-pipeline.yml | 8 -------- filebeat/magefile.go | 4 ++++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 1820f575aad..3de9277578f 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -134,10 +134,6 @@ steps: TESTING_FILEBEAT_SKIP_DIFF: 1 PYTEST_ADDOPTS: "-k test_modules" command: | - set -euo pipefail - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet filebeat - echo "~~~ Running tests" cd filebeat mage pythonIntegTest retry: @@ -160,10 +156,6 @@ steps: STACK_ENVIRONMENT: "next-major" TESTING_FILEBEAT_SKIP_DIFF: 1 command: | - set -euo pipefail - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet filebeat - echo "~~~ Running tests" cd filebeat mage pythonIntegTest retry: diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 16ae678e594..92cf6e03e22 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -203,6 +203,10 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration environment (Docker). func PythonIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" && (os.Getenv("STACK_ENVIRONMENT") == "prev-minor" || os.Getenv("STACK_ENVIRONMENT") == "next-major") { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.Deps(Fields, Dashboards) } From 20c179ad3b4c2f85a8695f253ab36d3f0d0b09a3 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 15:33:45 +0300 Subject: [PATCH 10/25] added changeset to x-pack/filebeat --- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 5 ----- x-pack/filebeat/magefile.go | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 4e3e7d57b71..7b79ff2743b 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -137,10 +137,6 @@ steps: TESTING_FILEBEAT_SKIP_DIFF: 1 PYTEST_ADDOPTS: "-k test_xpack_modules" command: | - set -euo pipefail - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests" cd x-pack/filebeat mage pythonIntegTest retry: @@ -264,7 +260,6 @@ steps: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" MODULE: "aws" - # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" agents: provider: "aws" diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index bb0908af571..3689bf9f78c 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -12,6 +12,7 @@ import ( "fmt" "time" + "github.com/elastic/beats/v7/dev-tools/mage/target/test" "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" @@ -25,8 +26,6 @@ import ( _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" - // mage:import - "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { @@ -172,6 +171,10 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration environment (Docker). func PythonIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" && os.Getenv("STACK_ENVIRONMENT") == "prev-minor" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.Deps(Fields, Dashboards) } @@ -189,3 +192,12 @@ func PythonIntegTest(ctx context.Context) error { return devtools.PythonTest(args) }) } + +// Test runs all available tests (unitTest + integTest). +func Test() { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + + test.Test() +} From 99e392f54ca5fbe5aa4024e25603ff546500618f Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 15:38:02 +0300 Subject: [PATCH 11/25] updated x-pack/winlogbeat --- .buildkite/x-pack/pipeline.xpack.winlogbeat.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index aa5b45104a1..da2c143d236 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -70,7 +70,7 @@ steps: - github_commit_status: context: "x-pack/winlogbeat Win 2019 Unit Tests (Module)" - - label: ":windows: x-pack/winlogbeat Win-2022 Unit Tests" + - label: ":windows: x-pack/winlogbeat Win-2022 Unit Tests (Module)" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -89,14 +89,14 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win-2022 Unit Tests" + context: "x-pack/winlogbeat Win-2022 Unit Tests (Module)" - group: "Extended Windows Tests" key: "extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: x-pack/winlogbeat Win-2016 Unit Tests" + - label: ":windows: x-pack/winlogbeat Win-2016 Unit Tests (Module)" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -112,9 +112,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win-2016 Unit Tests" + context: "x-pack/winlogbeat Win-2016 Unit Tests (Module)" - - label: ":windows: x-pack/winlogbeat Win 10 Unit Tests" + - label: ":windows: x-pack/winlogbeat Win 10 Unit Tests (Module)" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -133,9 +133,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win 10 Unit Tests" + context: "x-pack/winlogbeat Win 10 Unit Tests (Module)" - - label: ":windows: x-pack/winlogbeat Win-2019 Unit Tests" + - label: ":windows: x-pack/winlogbeat Win-2019 Unit Tests (Module)" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest From 6c381083e004350f364f1cbccc070c5a2b7b2a8f Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 15:52:10 +0300 Subject: [PATCH 12/25] added os imports --- .buildkite/libbeat/pipeline.libbeat.yml | 8 +-- .buildkite/scripts/changesets.sh | 66 ------------------------- filebeat/magefile.go | 1 + x-pack/auditbeat/magefile.go | 1 + x-pack/filebeat/magefile.go | 1 + x-pack/metricbeat/magefile.go | 1 + x-pack/winlogbeat/magefile.go | 1 + 7 files changed, 7 insertions(+), 72 deletions(-) delete mode 100644 .buildkite/scripts/changesets.sh diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index f7a86459a99..a5d0bfc4088 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -108,15 +108,11 @@ steps: - github_commit_status: context: "libbeat: Python Integration Tests" - - label: ":ubuntu: Libbeat: Module compat tests: next major" + - label: ":ubuntu: Libbeat: next major" # Run module integration tests under next major of Elastic stack. env: STACK_ENVIRONMENT: "next-major" command: | - set -euo pipefail - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet libbeat - echo "~~~ Running tests" cd libbeat mage pythonIntegTest retry: @@ -131,7 +127,7 @@ steps: - "libbeat/build/*.json" notify: - github_commit_status: - context: "libbeat: Module compat tests: next major" + context: "libbeat: next major" - label: ":ubuntu: Libbeat: Crosscompile" key: "mandatory-cross-compile" diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh deleted file mode 100644 index 1ab80edc585..00000000000 --- a/.buildkite/scripts/changesets.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# This script contains helper functions related to what should be run depending on Git changes - -set -euo pipefail - -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" - -definePattern() { - pattern="${OSS_MODULE_PATTERN}" - - if [[ "$beatPath" == *"x-pack/"* ]]; then - pattern="${XPACK_MODULE_PATTERN}" - fi -} - -defineExclusions() { - exclude="^$beatPath\/module\/(.*(? Date: Mon, 23 Sep 2024 18:26:50 +0300 Subject: [PATCH 13/25] pr fixes --- .buildkite/filebeat/filebeat-pipeline.yml | 2 ++ auditbeat/magefile.go | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 3de9277578f..e084e3e254d 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -133,6 +133,7 @@ steps: STACK_ENVIRONMENT: "prev-minor" TESTING_FILEBEAT_SKIP_DIFF: 1 PYTEST_ADDOPTS: "-k test_modules" + MAGEFILE_VERBOSE: "1" command: | cd filebeat mage pythonIntegTest @@ -155,6 +156,7 @@ steps: env: STACK_ENVIRONMENT: "next-major" TESTING_FILEBEAT_SKIP_DIFF: 1 + MAGEFILE_VERBOSE: "1" command: | cd filebeat mage pythonIntegTest diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 4bd5ee56997..e5fa4d63c91 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -21,6 +21,7 @@ package main import ( + "context" "fmt" "os" "time" @@ -166,5 +167,5 @@ func PythinIntegTest() { mg.Deps(devtools.DefineModules) } - integtest.PythonIntegTest() + integtest.PythonIntegTest(context.TODO()) } From 5431e548b0d6a205f681fa69f317e2f6d2d8c329 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 18:50:01 +0300 Subject: [PATCH 14/25] updated x-pack/filebeat magefile imports --- x-pack/filebeat/magefile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index f5832a36053..b95d69d5572 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -13,9 +13,10 @@ import ( "os" "time" - "github.com/elastic/beats/v7/dev-tools/mage/target/test" "github.com/magefile/mage/mg" + "github.com/elastic/beats/v7/dev-tools/mage/target/test" + devtools "github.com/elastic/beats/v7/dev-tools/mage" filebeat "github.com/elastic/beats/v7/filebeat/scripts/mage" From 29c8c1f9e3a03e1a2984b3e75d7aa6ec0918b8f8 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 18:54:32 +0300 Subject: [PATCH 15/25] updated imports in x-pack/auditbeat --- x-pack/auditbeat/magefile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index fafc3d4e277..e6620ecc015 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -12,11 +12,12 @@ import ( "os" "time" - "github.com/elastic/beats/v7/dev-tools/mage/target/test" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" + "github.com/elastic/beats/v7/dev-tools/mage/target/test" + auditbeat "github.com/elastic/beats/v7/auditbeat/scripts/mage" devtools "github.com/elastic/beats/v7/dev-tools/mage" From ede596ae47a7e725adb37fda455c5506ba31fda8 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 19:10:20 +0300 Subject: [PATCH 16/25] updated imports in auditbaet --- auditbeat/magefile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index e5fa4d63c91..259ffc274fa 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -26,9 +26,10 @@ import ( "os" "time" - "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" "github.com/magefile/mage/mg" + "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + auditbeat "github.com/elastic/beats/v7/auditbeat/scripts/mage" devtools "github.com/elastic/beats/v7/dev-tools/mage" From 2979f95308ff89abb26d0865781c8a8a1f303bb9 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 19:11:24 +0300 Subject: [PATCH 17/25] added neat_path to filebeat --- .buildkite/filebeat/filebeat-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e084e3e254d..d6bb44e16da 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -23,6 +23,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module Tests + BEAT_PATH: "filebeat" + steps: - group: "Check/Update" key: "filebeat-check-update" From c5497ec433c916ec44e8a95721a19c9037921066 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 19:12:50 +0300 Subject: [PATCH 18/25] added beat_path to x-pack/filebeat --- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 7b79ff2743b..147dc3ee8ad 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -23,6 +23,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module Tests + BEAT_PATH: "x-pack/filebeat" + steps: - group: "Check/Update" key: "x-pack-filebeat-check-update" From b793c8e2ee10bf4ce0392e6aa4cf6b9d91a433c6 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 19:14:53 +0300 Subject: [PATCH 19/25] updated imports for x-pack/winlogbeat --- x-pack/winlogbeat/magefile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/winlogbeat/magefile.go b/x-pack/winlogbeat/magefile.go index 3696296bf33..ec707779ec8 100644 --- a/x-pack/winlogbeat/magefile.go +++ b/x-pack/winlogbeat/magefile.go @@ -13,6 +13,8 @@ import ( "github.com/magefile/mage/mg" + "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" + devtools "github.com/elastic/beats/v7/dev-tools/mage" // mage:import @@ -26,8 +28,6 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" // mage:import - "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" - // mage:import winlogbeat "github.com/elastic/beats/v7/winlogbeat/scripts/mage" ) From 3c8e86e25a2098e2c054823594e209b99738efe2 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 19:34:50 +0300 Subject: [PATCH 20/25] fixed typo --- auditbeat/magefile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 259ffc274fa..796e507b01c 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -163,7 +163,7 @@ func Docs() { } // PythonIntegTest executes the python system tests in the integration -func PythinIntegTest() { +func PythonIntegTest() { if os.Getenv("CI") == "true" && os.Getenv("STACK_ENVIRONMENT") == "next-major" { mg.Deps(devtools.DefineModules) } From 618771bc7ef1c3046ed31c9e3866003674806c99 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 23 Sep 2024 20:40:21 +0300 Subject: [PATCH 21/25] added beat_path to auditbeat --- .buildkite/auditbeat/auditbeat-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 1a9b9b687c7..19ce9e996a5 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -26,6 +26,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module Tests + BEAT_PATH: "auditbeat" + steps: - group: "Check/Update" key: "auditbeat-check-update" From 63d1b7d43b01836bcb37c106c9a9b9728efe9c3d Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 24 Sep 2024 12:08:51 +0300 Subject: [PATCH 22/25] debug x-pack/winlogbeat --- .buildkite/filebeat/filebeat-pipeline.yml | 2 -- .buildkite/x-pack/pipeline.xpack.winlogbeat.yml | 16 ++++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index d6bb44e16da..c2ab8e13d64 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -136,7 +136,6 @@ steps: STACK_ENVIRONMENT: "prev-minor" TESTING_FILEBEAT_SKIP_DIFF: 1 PYTEST_ADDOPTS: "-k test_modules" - MAGEFILE_VERBOSE: "1" command: | cd filebeat mage pythonIntegTest @@ -159,7 +158,6 @@ steps: env: STACK_ENVIRONMENT: "next-major" TESTING_FILEBEAT_SKIP_DIFF: 1 - MAGEFILE_VERBOSE: "1" command: | cd filebeat mage pythonIntegTest diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index da2c143d236..abe28978d47 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -70,7 +70,9 @@ steps: - github_commit_status: context: "x-pack/winlogbeat Win 2019 Unit Tests (Module)" - - label: ":windows: x-pack/winlogbeat Win-2022 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2022 Unit Tests (Module)" + env: + MAGEFILE_VERBOSE: "1" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -89,14 +91,14 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win-2022 Unit Tests (Module)" + context: "x-pack/winlogbeat Win 2022 Unit Tests (Module)" - group: "Extended Windows Tests" key: "extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: x-pack/winlogbeat Win-2016 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2016 Unit Tests (Module)" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -112,7 +114,7 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win-2016 Unit Tests (Module)" + context: "x-pack/winlogbeat Win 2016 Unit Tests (Module)" - label: ":windows: x-pack/winlogbeat Win 10 Unit Tests (Module)" command: | @@ -135,7 +137,9 @@ steps: - github_commit_status: context: "x-pack/winlogbeat Win 10 Unit Tests (Module)" - - label: ":windows: x-pack/winlogbeat Win-2019 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests (Module)" + env: + MAGEFILE_VERBOSE: "1" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -154,7 +158,7 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat: Win-2019 Unit Tests" + context: "x-pack/winlogbeat: Win 2019 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed From de759297b2a74add717e8d3e1227a3ce484a8379 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 24 Sep 2024 12:30:43 +0300 Subject: [PATCH 23/25] added beat path to x-pack/winlogbeat --- .buildkite/x-pack/pipeline.xpack.winlogbeat.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index abe28978d47..a32b77f1a3b 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -18,6 +18,8 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + BEAT_PATH: "x-pack/winlogbeat" + steps: - group: "Check/Update" key: "x-pack-winlogbeat-check-update" From 03465b7316dc3dd1d417a5019cc50caac5a89ccc Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 24 Sep 2024 13:45:10 +0300 Subject: [PATCH 24/25] removed magefile_verbose --- .buildkite/x-pack/pipeline.xpack.winlogbeat.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index a32b77f1a3b..c40dc73bc37 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -73,8 +73,6 @@ steps: context: "x-pack/winlogbeat Win 2019 Unit Tests (Module)" - label: ":windows: x-pack/winlogbeat Win 2022 Unit Tests (Module)" - env: - MAGEFILE_VERBOSE: "1" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -140,8 +138,6 @@ steps: context: "x-pack/winlogbeat Win 10 Unit Tests (Module)" - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests (Module)" - env: - MAGEFILE_VERBOSE: "1" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest From 6baf89347cdbcbb9dc3bea5cdcac892cd866bcb3 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 14 Oct 2024 11:50:37 +0300 Subject: [PATCH 25/25] removed module definition for x-pack/winlogbeat --- .../x-pack/pipeline.xpack.winlogbeat.yml | 20 +++++++++---------- x-pack/winlogbeat/magefile.go | 14 ++----------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index c40dc73bc37..6debdd00586 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -18,8 +18,6 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" - BEAT_PATH: "x-pack/winlogbeat" - steps: - group: "Check/Update" key: "x-pack-winlogbeat-check-update" @@ -51,7 +49,7 @@ steps: - group: "Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests" key: "mandatory-win-2019-unit-tests" command: | Set-Location -Path x-pack/winlogbeat @@ -70,9 +68,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win 2019 Unit Tests (Module)" + context: "x-pack/winlogbeat Win 2019 Unit Tests" - - label: ":windows: x-pack/winlogbeat Win 2022 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2022 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -91,14 +89,14 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win 2022 Unit Tests (Module)" + context: "x-pack/winlogbeat Win 2022 Unit Tests" - group: "Extended Windows Tests" key: "extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: x-pack/winlogbeat Win 2016 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2016 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -114,9 +112,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win 2016 Unit Tests (Module)" + context: "x-pack/winlogbeat Win 2016 Unit Tests" - - label: ":windows: x-pack/winlogbeat Win 10 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 10 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -135,9 +133,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win 10 Unit Tests (Module)" + context: "x-pack/winlogbeat Win 10 Unit Tests" - - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests (Module)" + - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest diff --git a/x-pack/winlogbeat/magefile.go b/x-pack/winlogbeat/magefile.go index ec707779ec8..a53e56d62ea 100644 --- a/x-pack/winlogbeat/magefile.go +++ b/x-pack/winlogbeat/magefile.go @@ -9,12 +9,9 @@ package main import ( "fmt" - "os" "github.com/magefile/mage/mg" - "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" - devtools "github.com/elastic/beats/v7/dev-tools/mage" // mage:import @@ -28,6 +25,8 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" // mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" + // mage:import winlogbeat "github.com/elastic/beats/v7/winlogbeat/scripts/mage" ) @@ -49,12 +48,3 @@ func Ironbank() error { fmt.Println(">> Ironbank: this module is not subscribed to the IronBank releases.") return nil } - -// Test runs all available tests (unitTest + integTest) -func UnitTest() { - if os.Getenv("CI") == "true" { - mg.Deps(devtools.DefineModules) - } - - unittest.UnitTest() -}