[ci] JDK matrix Buildkite pipelines (part 1) (#15539) (#15555)

This commit is the first part of the migration of JDK matrix tests
from Jenkins to Buildkite. There will be two separate pipelines, for
Linux and Windows.

Linux is currently limited to Ubuntu 22.04 and 20.04, but
additional operating systems will be added outside of the Logstash
repository seamlessly through additional VM images.

Steps are created dynamically and the underlying script is meant to be
common for Linux and Windows. Windows is currently a stub and
will be added in a follow up PR.

Relates:

- elastic/ingest-dev#1725
- elastic/ci-agent-images#424

(cherry picked from commit 956bf48)

Co-authored-by: Dimitrios Liappis <[email protected]>

.buildkite/linux_jdk_matrix_pipeline.yml

@@ -1,3 +1,87 @@
 steps:
-  - label: "Test Linux JDK matrix pipeline"
-    command: "echo 'Hello world'"
+  - input: "Test Parameters"
+    if: build.source != "schedule"
+    fields:
+      - select: "Operating System"
+        key: "matrix-os"
+        hint: "The operating system variant(s) to run on:"
+        required: true
+        multiple: true
+        default: "ubuntu-2204"
+        options:
+          - label: "Ubuntu 22.04"
+            value: "ubuntu-2204"
+          - label: "Ubuntu 20.04"
+            value: "ubuntu-2004"
+          - label: "Ubuntu 18.04"
+            value: "ubuntu-1804"
+          - label: "Debian 11"
+            value: "debian-11"
+          - label: "Debian 10"
+            value: "debian-10"
+          - label: "RHEL 9"
+            value: "rhel-9"
+          - label: "RHEL 8"
+            value: "rhel-8"
+          - label: "CentOS 8"
+            value: "centos-8"
+          - label: "CentOS 7"
+            value: "centos-7"
+          - label: "Oracle Linux 8"
+            value: "oraclelinux-8"
+          - label: "Oracle Linux 7"
+            value: "oraclelinux-7"
+          - label: "Rocky Linux 8"
+            value: "rocky-8"
+          - label: "Amazon Linux"
+            value: "amazonlinux-2023"
+          - label: "OpenSUSE Leap 15"
+            value: "opensuse-leap-15"
+
+      - select: "Java"
+        key: "matrix-jdk"
+        hint: "The JDK to test with:"
+        required: true
+        multiple: true
+        default: "adoptiumjdk_17"
+        options:
+          - label: "Adoptium JDK 17 (Eclipse Temurin)"
+            value: "adoptiumjdk_17"
+          - label: "Adopt OpenJDK 11"
+            value: "adoptopenjdk_11"
+          - label: "OpenJDK 17"
+            value: "openjdk_17"
+          - label: "OpenJDK 11"
+            value: "openjdk_11"
+          - label: "Zulu 17"
+            value: "zulu_17"
+          - label: "Zulu 11"
+            value: "zulu_11"
+
+  - wait: ~
+    if: build.source != "schedule"
+
+  - command: |
+      set -euo pipefail
+
+      echo "--- Downloading prerequisites"
+      python3 -m pip install ruamel.yaml
+
+      echo "--- Printing generated dynamic steps"
+      export MATRIX_OSES="$(buildkite-agent meta-data get matrix-os)"
+      export MATRIX_JDKS="$(buildkite-agent meta-data get matrix-jdk)"
+      set +eo pipefail
+      python3 .buildkite/scripts/jdk-matrix-tests/generate-steps.py >pipeline_steps.yml
+      if [[ $$? -ne 0 ]]; then
+        echo "^^^ +++"
+        echo "There was a problem rendering the pipeline steps."
+        cat pipeline_steps.yml
+        echo "Exiting now."
+        exit 1
+      else
+        set -eo pipefail
+        cat pipeline_steps.yml
+      fi
+
+      echo "--- Uploading steps to buildkite"
+      cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/scripts/jdk-matrix-tests/generate-steps.py

@@ -0,0 +1,306 @@
+import abc
+from dataclasses import dataclass
+import os
+import sys
+import typing
+
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString
+
+
+@dataclass
+class JobRetValues:
+    step_label: str
+    command: str
+    step_key: str
+    depends: str
+    default_agent: bool = False
+
+@dataclass
+class BuildkiteEmojis:
+    running: str = ":bk-status-running:"
+    success: str = ":bk-status-passed:"
+    failed: str = ":bk-status-failed:"
+
+def slugify_bk_key(key: str) -> str:
+    """
+    Convert and return key to an acceptable format for Buildkite's key: field
+    Only alphanumerics, dashes and underscores are allowed.
+    """
+
+    mapping_table = str.maketrans({'.': '_', ' ': '_', '/': '_'})
+
+    return key.translate(mapping_table)
+
+def get_bk_metadata(key: str) -> typing.List[str]:
+    try:
+      return os.environ[key].split()
+    except KeyError:
+        print(f"Missing environment variable [{key}]. This should be set before calling this script using buildkite-agent meta-data get. Exiting.")
+        exit(1)
+
+def bk_annotate(body: str, context: str, mode = "") -> str:
+    cmd = f"""buildkite-agent annotate --style=info --context={context} """
+    if mode:
+        cmd += f"--{mode} "
+
+    cmd += f"\"{body}\n\""
+    return cmd
+
+
+class Jobs(abc.ABC):
+    def __init__(self, os: str, jdk: str, group_key: str):
+        self.os = os
+        self.jdk = jdk
+        self.group_key = group_key
+        self.init_annotation_key = f"{os}-{jdk}-initialize-annotation"
+
+    def init_annotation(self) -> JobRetValues:
+        """
+        Command for creating the header of a new annotation for a group step
+        """
+
+        body = f"### Group: {self.os} / {self.jdk}\n| **Status** | **Test** |\n| --- | ----|"
+
+        return JobRetValues(
+            step_label="Initialize annotation",
+            command=LiteralScalarString(bk_annotate(body=body, context=self.group_key)),
+            step_key=self.init_annotation_key,
+            depends="",
+            default_agent=True,
+        )
+
+    @abc.abstractmethod
+    def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
+        pass
+
+
+class WindowsJobs(Jobs):
+    def __init__(self, os: str, jdk: str, group_key: str):
+      super().__init__(os=os, jdk=jdk, group_key=group_key)
+
+    def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]:
+        return [
+          self.unit_tests,
+        ]
+
+    def unit_tests(self) -> JobRetValues:
+        step_name_human = "Java Unit Test"
+        test_command = "# TODO"
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key="java-unit-test",
+            depends="",
+        )
+        return step_name_human, test_command
+
+
+class LinuxJobs(Jobs):
+    def __init__(self, os: str, jdk: str, group_key: str):
+      super().__init__(os=os, jdk=jdk, group_key=group_key)
+
+    def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]:
+        return [
+            self.init_annotation,
+            self.java_unit_test,
+            self.ruby_unit_test,
+            self.integration_tests_part_1,
+            self.integration_tests_part_2,
+            self.pq_integration_tests_part_1,
+            self.pq_integration_tests_part_2,
+            self.x_pack_unit_tests,
+            self.x_pack_integration,
+        ]
+
+    def prepare_shell(self) -> str:
+        jdk_dir = f"/opt/buildkite-agent/.java/{self.jdk}"
+        return f"""#!/usr/bin/env bash
+set -euo pipefail
+
+# unset generic JAVA_HOME
+unset JAVA_HOME
+
+# LS env vars for JDK matrix tests
+export BUILD_JAVA_HOME={jdk_dir}
+export RUNTIME_JAVA_HOME={jdk_dir}
+export LS_JAVA_HOME={jdk_dir}
+
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+eval "$(rbenv init -)"
+"""
+
+    def failed_step_annotation(self, step_name_human) -> str:
+        return bk_annotate(body=f"| {BuildkiteEmojis.failed} | {step_name_human} |", context=self.group_key, mode="append")
+
+    def succeeded_step_annotation(self, step_name_human) -> str:
+        return bk_annotate(body=f"| {BuildkiteEmojis.success} | {step_name_human} |", context=self.group_key, mode="append")
+
+    def emit_command(self, step_name_human, test_command: str) -> str:
+        return LiteralScalarString(f"""
+{self.prepare_shell()}
+# temporarily disable immediate failure on errors, so that we can update the BK annotation
+set +eo pipefail
+{test_command}
+if [[ $$? -ne 0 ]]; then
+    {self.failed_step_annotation(step_name_human)}
+  exit 1
+else
+    {self.succeeded_step_annotation(step_name_human)}
+fi
+      """)
+
+    def java_unit_test(self) -> JobRetValues:
+        step_name_human = "Java Unit Test"
+        step_key = f"{self.group_key}-java-unit-test"
+        test_command = '''
+export ENABLE_SONARQUBE="false"
+ci/unit_tests.sh java
+        '''
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=self.emit_command(step_name_human, test_command),
+            step_key=step_key,
+            depends=self.init_annotation_key,
+        )
+
+    def ruby_unit_test(self) -> JobRetValues:
+        step_name_human = "Ruby Unit Test"
+        step_key = f"{self.group_key}-ruby-unit-test"
+        test_command = """
+ci/unit_tests.sh ruby
+        """
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=self.emit_command(step_name_human, test_command),
+            step_key=step_key,
+            depends=self.init_annotation_key,
+        )
+
+    def integration_tests_part_1(self) -> JobRetValues:
+        return self.integration_tests(part=1)
+
+    def integration_tests_part_2(self) -> JobRetValues:
+        return self.integration_tests(part=2)
+
+    def integration_tests(self, part: int) -> JobRetValues:
+        step_name_human = f"Integration Tests - {part}"
+        step_key = f"{self.group_key}-integration-tests-{part}"
+        test_command = f"""
+ci/integration_tests.sh split {part-1}
+        """
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=self.emit_command(step_name_human, test_command),
+            step_key=step_key,
+            depends=self.init_annotation_key,
+        )
+
+    def pq_integration_tests_part_1(self) -> JobRetValues:
+        return self.pq_integration_tests(part=1)
+
+    def pq_integration_tests_part_2(self) -> JobRetValues:
+        return self.pq_integration_tests(part=2)
+
+    def pq_integration_tests(self, part: int) -> JobRetValues:
+        step_name_human = f"IT Persistent Queues - {part}"
+        step_key = f"{self.group_key}-it-persistent-queues-{part}"
+        test_command = f"""
+export FEATURE_FLAG=persistent_queues
+ci/integration_tests.sh split {part-1}
+        """
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=self.emit_command(step_name_human, test_command),
+            step_key=step_key,
+            depends=self.init_annotation_key,
+        )
+
+    def x_pack_unit_tests(self) -> JobRetValues:
+        step_name_human = "x-pack unit tests"
+        step_key = f"{self.group_key}-x-pack-unit-test"
+        test_command = """
+x-pack/ci/unit_tests.sh
+        """
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=self.emit_command(step_name_human, test_command),
+            step_key=step_key,
+            depends=self.init_annotation_key,
+        )
+
+    def x_pack_integration(self) -> JobRetValues:
+        step_name_human = "x-pack integration"
+        step_key = f"{self.group_key}-x-pack-integration"
+        test_command = """
+x-pack/ci/integration_tests.sh
+        """
+
+        return JobRetValues(
+            step_label=step_name_human,
+            command=self.emit_command(step_name_human, test_command),
+            step_key=step_key,
+            depends=self.init_annotation_key,
+        )
+
+
+if __name__ == "__main__":
+    matrix_oses = get_bk_metadata(key="MATRIX_OSES")
+    matrix_jdkes = get_bk_metadata(key="MATRIX_JDKS")
+
+    pipeline_name = os.environ.get("BUILDKITE_PIPELINE_NAME", "").lower()
+
+    structure = {"steps": []}
+
+
+    for matrix_os in matrix_oses:
+        for matrix_jdk in matrix_jdkes:
+          group_name = f"{matrix_os}/{matrix_jdk}"
+          group_key = slugify_bk_key(group_name)
+
+          if "windows" in pipeline_name:
+            jobs = WindowsJobs(os=matrix_os, jdk=matrix_jdk, group_key=group_key)
+          else:
+            jobs = LinuxJobs(os=matrix_os, jdk=matrix_jdk, group_key=group_key)
+
+          group_steps = []
+          for job in jobs.all_jobs():
+            job_values = job()
+
+            step = {
+              "label": f"{matrix_os} / {matrix_jdk} / {job_values.step_label}",
+              "key": job_values.step_key,
+            }
+
+            if job_values.depends:
+                step["depends_on"] = job_values.depends
+
+            if not job_values.default_agent:
+                step["agents"] = {
+                    "provider": "gcp",
+                    "imageProject": "elastic-images-qa",
+                    "image": f"family/platform-ingest-logstash-multi-jdk-{matrix_os}",
+                    "machineType": "n2-standard-4",
+                    "diskSizeGb": 200,
+                    "diskType": "pd-ssd",
+                }
+
+
+            step["command"] = job_values.command
+
+            group_steps.append(step)
+
+
+          structure["steps"].append({
+            "group": group_name,
+            "key": slugify_bk_key(group_name),
+            "steps": group_steps})
+
+
+    YAML().dump(structure, sys.stdout)