Deploy k8s pipeline migration to Buildkite

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -3,16 +3,19 @@
 env:
   BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
-  IMAGE_WIN_2016: "family/core-windows-2016"
-  IMAGE_WIN_2019: "family/core-windows-2019"
-  IMAGE_WIN_2022: "family/core-windows-2022"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
+  IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
+  IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
+  IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - group: "Auditbeat Mandatory Testing"
@@ -21,8 +24,7 @@ steps:
 
     steps:
       - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
               context: "Auditbeat: linux/Unit Tests"
@@ -35,8 +37,7 @@ steps:
           - "auditbeat/build/*.json"
 
       - label: ":rhel: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
               context: "Auditbeat: rhel/Unit Tests"
@@ -48,29 +49,41 @@ steps:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
 
-      - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+      - label: ":windows:-2016 Unit Tests"
+        key: "windows-2016"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: windows-2016/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows:-2022 Unit Tests"
+        key: "windows-2022"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: windows/Unit Tests"
+              context: "Auditbeat: windows-2022/Unit Tests"
         agents:
           provider: "gcp"
-          image: "{{matrix.image}}"
+          image: "${IMAGE_WIN_2022}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_WIN_2016}"
-              - "${IMAGE_WIN_2022}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
 
       - label: ":linux: Crosscompile"
         command:
-          - ".buildkite/auditbeat/scripts/crosscompile.sh"
+          - "make -C auditbeat crosscompile"
         env:
           GOX_FLAGS: "-arch amd64"
         notify:
@@ -89,11 +102,10 @@ steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat/Extended: Unit Tests ARM"
+              context: "Auditbeat: Unit Tests ARM"
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -103,27 +115,38 @@ steps:
       - label: ":mac: MacOS Unit Tests"
         key: "macos-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat/Extended: MacOS Unit Tests"
+              context: "Auditbeat: MacOS Unit Tests"
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
         artifact_paths: "auditbeat/build/*.xml"
 
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-extended-arm"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "auditbeat/build/*.xml"
+
   - group: "Windows Extended Testing"
-    key: "extended-tests-win"
+    key: "windows-extended-tests"
     if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
-      - label: ":windows: Win 2019 Unit Tests"
-        key: "win-extended-2019"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+      - label: ":windows:-2019 Unit Tests"
+        key: "windows-extended-2019"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat/Extended: Win-2019 Unit Tests"
+              context: "Auditbeat: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -134,6 +157,38 @@ steps:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
 
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-extended-11"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-11 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-extended-10"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-10 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -1,5 +1,65 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: "Checks"
+    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
+    notify:
+      - github_commit_status:
+          context: "Deploy K8S/Checks"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.29.0"
+    key: "k8s-test-129"
+    env:
+      K8S_VERSION: "v1.29.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.28.0"
+    key: "k8s-test-128"
+    env:
+      K8S_VERSION: "v1.28.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.27.3"
+    key: "k8s-test-1273"
+    env:
+      K8S_VERSION: "v1.27.3"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.26.6"
+    key: "k8s-test-1266"
+    env:
+      K8S_VERSION: "v1.26.6"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"

.buildkite/deploy/kubernetes/scripts/install-kind.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MSG="environment variable missing."
+KIND_VERSION=${KIND_VERSION:?$MSG}
+KIND_BINARY="${BIN}/kind"
+
+if command -v kind
+then
+    set +e
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
+    then
+        echo "--- Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing Kind"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KIND_BINARY}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
+    chmod +x "${KIND_BINARY}"
+    echo "Kind installed: ${KIND_VERSION}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KIND_BINARY}" ] ; then
+        rm "${KIND_BINARY}"
+    fi
+    exit 1
+fi

.buildkite/deploy/kubernetes/scripts/install-kubectl.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MSG="parameter missing."
+K8S_VERSION=${K8S_VERSION:?$MSG}
+KUBECTL_BINARY="${BIN}/kubectl"
+
+if command -v kubectl
+then
+    set +e
+    echo "Found kubectl. Checking version.."
+    FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
+    if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
+    then
+        echo "Kubectl Versions match. No need to install kubectl. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing kubectl"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KUBECTL_BINARY}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then
+    chmod +x "${KUBECTL_BINARY}"
+    echo "Current K8S Version: ${K8S_VERSION}"
+    echo "Kubectl installed: ${KUBECTL_BINARY}"
+else
+    echo "--- Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KUBECTL_BINARY}" ] ; then
+        rm "${KUBECTL_BINARY}"
+    fi
+    exit 1
+fi
+

.buildkite/deploy/kubernetes/scripts/kind-setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kind create cluster --image kindest/node:${K8S_VERSION}
+
+echo "Cluster info: "
+kubectl cluster-info

.buildkite/deploy/kubernetes/scripts/make.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "--- Checking K8S"
+make -C deploy/kubernetes all
+make check-no-changes