Onboarding: gitlab parallel (#3571)

conftest.py

@@ -46,7 +46,12 @@ def pytest_addoption(parser):
     parser.addoption("--vm-env", type=str, action="store", help="Set virtual machine environment")
     parser.addoption("--vm-provider", type=str, action="store", help="Set provider for VMs")
     parser.addoption("--vm-only-branch", type=str, action="store", help="Filter to execute only one vm branch")
+    parser.addoption("--vm-only", type=str, action="store", help="Filter to execute only one vm name")
     parser.addoption("--vm-skip-branches", type=str, action="store", help="Filter exclude vm branches")
+    parser.addoption(
+        "--vm-gitlab-pipeline", action="store_true", help="Generate pipeline for Gitlab CI. Not run the tests"
+    )
+
     parser.addoption(
         "--vm-default-vms",
         type=str,
@@ -266,7 +271,7 @@ def iter_markers(self, name=None):
         declared_scenarios[item.nodeid] = declared_scenario
 
         # If we are running scenario with the option sleep, we deselect all
-        if session.config.option.sleep:
+        if session.config.option.sleep or session.config.option.vm_gitlab_pipeline:
             deselected.append(item)
             continue
 
@@ -424,6 +429,12 @@ def pytest_sessionfinish(session, exitstatus):
         except Exception:
             logger.exception("Fail to export export reports", exc_info=True)
 
+    if session.config.option.vm_gitlab_pipeline:
+        NO_TESTS_COLLECTED = 5
+        SUCCESS = 0
+        if exitstatus == NO_TESTS_COLLECTED:
+            session.exitstatus = SUCCESS
+
 
 def export_feature_parity_dashboard(session, data):
 

utils/_context/_scenarios/auto_injection.py

@@ -3,7 +3,7 @@
 import copy
 from utils._context.library_version import LibraryVersion
 from utils.tools import logger
-from utils.virtual_machine.utils import get_tested_apps_vms
+from utils.virtual_machine.utils import get_tested_apps_vms, generate_gitlab_pipeline
 
 from utils._context.virtual_machines import (
     Ubuntu20amd64,
@@ -221,6 +221,8 @@ def configure(self, config):
             raise ValueError(
                 f"Invalid value for --vm-default-vms: {self.only_default_vms}. Use 'All', 'True' or 'False'"
             )
+        # Pipeline generation mode. No run tests, no start vms
+        self.vm_gitlab_pipeline = config.option.vm_gitlab_pipeline
 
         provisioner.remove_unsupported_machines(
             self._library.library,
@@ -230,6 +232,7 @@ def configure(self, config):
             config.option.vm_only_branch,
             config.option.vm_skip_branches,
             self.only_default_vms,
+            config.option.vm_only,
         )
         for vm in self.required_vms:
             logger.info(f"Adding provision for {vm.name}")
@@ -249,6 +252,19 @@ def configure(self, config):
             vm.add_app_env(self.app_env)
         self.vm_provider.configure(self.required_vms)
 
+        if self.vm_gitlab_pipeline:
+            pipeline = generate_gitlab_pipeline(
+                config.option.vm_library,
+                self._weblog,
+                self.name,
+                self._env,
+                self.required_vms,
+                os.getenv("DD_INSTALLER_LIBRARY_VERSION", ""),
+                os.getenv("DD_INSTALLER_INJECTOR_VERSION", ""),
+            )
+            with open(f"{self.host_log_folder}/gitlab_pipeline.yml", "w", encoding="utf-8") as f:
+                json.dump(pipeline, f, ensure_ascii=False, indent=4)
+
     def _check_test_environment(self):
         """Check if the test environment is correctly set"""
 
@@ -268,15 +284,15 @@ def _check_test_environment(self):
 
     def get_warmups(self):
         warmups = super().get_warmups()
+        if not self.vm_gitlab_pipeline:
+            if self.is_main_worker:
+                warmups.append(lambda: logger.terminal.write_sep("=", "Provisioning Virtual Machines", bold=True))
+                warmups.append(self.vm_provider.stack_up)
 
-        if self.is_main_worker:
-            warmups.append(lambda: logger.terminal.write_sep("=", "Provisioning Virtual Machines", bold=True))
-            warmups.append(self.vm_provider.stack_up)
-
-        warmups.append(self.fill_context)
+            warmups.append(self.fill_context)
 
-        if self.is_main_worker:
-            warmups.append(self.print_installed_components)
+            if self.is_main_worker:
+                warmups.append(self.print_installed_components)
 
         return warmups
 
@@ -298,7 +314,7 @@ def fill_context(self):
                     del self._tested_components[key]
 
     def close_targets(self):
-        if self.is_main_worker:
+        if self.is_main_worker and not self.vm_gitlab_pipeline:
             logger.info("Destroying virtual machines")
             self.vm_provider.stack_destroy()
 

utils/build/virtual_machine/weblogs/java/provision_test-app-java-multialpine.yml

@@ -26,6 +26,7 @@ lang_variant:
 weblog:
     name: test-app-java-multialpine
     nginx_config: utils/build/virtual_machine/weblogs/java/test-app-java-multialpine/nginx.conf
+    exact_os_branches: [ubuntu24]
     install: 
       - os_type: linux
         copy_files:

utils/build/virtual_machine/weblogs/java/provision_test-app-java-multicontainer.yml

@@ -26,6 +26,7 @@ lang_variant:
 weblog:
     name: test-app-java-multicontainer
     nginx_config: utils/build/virtual_machine/weblogs/java/test-app-java-multicontainer/nginx.conf
+    exact_os_branches: [ubuntu24]
     install: 
       - os_type: linux
         copy_files:

utils/docker_ssi/docker_ssi_matrix_builder.py

@@ -18,37 +18,12 @@ def generate_gitlab_pipeline(languages):
             "dependencies": [],
             "script": ["echo 'DONE'"],
         },
-        "docker_ssi_fpd": {
-            "image": "486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:a58cc31c",
-            "tags": ["arch:amd64"],
-            "stage": "parse_docker_ssi_results",
-            "allow_failure": True,
-            "rules": [
-                {"if": '$PARENT_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"', "when": "always"},
-                {"when": "manual", "allow_failure": True},
-            ],
-            "before_script": [
-                'export FP_IMPORT_URL=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.fp-import-url --with-decryption --query "Parameter.Value" --out text)',
-                'export FP_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.fp-api-key --with-decryption --query "Parameter.Value" --out text)',
-            ],
-            "script": [
-                "for folder in reports/logs*/ ; do",
-                'echo "Checking folder: ${folder}"',
-                "for filename in ./${folder}feature_parity.json; do",
-                "if [ -e ${filename} ]",
-                "then",
-                'echo "Processing report: ${filename}"',
-                'curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json"  --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include || true',
-                "fi",
-                "done",
-                "done",
-            ],
-        },
         ".base_ssi_job": {
             "image": "registry.ddbuild.io/ci/libdatadog-build/system-tests:48436362",
             "script": [
                 "./build.sh -i runner",
                 "source venv/bin/activate",
+                "echo 'Running SSI tests'",
                 'timeout 2700s ./run.sh DOCKER_SSI --ssi-weblog "$weblog" --ssi-library "$TEST_LIBRARY" --ssi-base-image "$base_image" --ssi-arch "$arch" --ssi-installable-runtime "$installable_runtime" --report-run-url ${CI_PIPELINE_URL} --report-environment prod',
             ],
             "rules": [
@@ -68,9 +43,11 @@ def generate_gitlab_pipeline(languages):
             "artifacts": {"when": "always", "paths": ["reports/"]},
         },
     }
+    # Add FPD push script
+    pipeline[".base_ssi_job"]["after_script"].extend(_generate_fpd_gitlab_script())
 
     for language in languages:
-        pipeline["stages"].append(language)
+        pipeline["stages"].append(language if len(languages) > 1 else "DOCKER_SSI")
         matrix = []
 
         filtered = [weblog for weblog in ALL_WEBLOGS if weblog.library == language]
@@ -89,16 +66,36 @@ def generate_gitlab_pipeline(languages):
             pipeline[language] = {
                 "extends": ".base_ssi_job",
                 "tags": ["runner:$runner"],
-                "stage": language,
+                "stage": language if len(languages) > 1 else "DOCKER_SSI",
                 "allow_failure": True,
-                "dependencies": [],
+                # "dependencies": [],
+                "needs": [],
                 "variables": {"TEST_LIBRARY": language,},
                 "parallel": {"matrix": matrix},
             }
-    pipeline["stages"].append("parse_docker_ssi_results")
     return pipeline
 
 
+def _generate_fpd_gitlab_script():
+    fpd_push_script = [
+        'if [ "$CI_COMMIT_BRANCH" = "main" ]; then',
+        'export FP_IMPORT_URL=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.fp-import-url --with-decryption --query "Parameter.Value" --out text)',
+        'export FP_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.fp-api-key --with-decryption --query "Parameter.Value" --out text)',
+        "for folder in reports/logs*/ ; do",
+        '  echo "Checking folder: ${folder}"',
+        "  for filename in ./${folder}feature_parity.json; do",
+        "    if [ -e ${filename} ]",
+        "    then",
+        '      echo "Processing report: ${filename}"',
+        '      curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include',
+        "    fi",
+        "  done",
+        "done",
+        "fi",
+    ]
+    return fpd_push_script
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--format", required=True, type=str, choices=["json", "yaml"], help="json or yaml")

utils/scripts/merge_gitlab_aws_pipelines.py

@@ -0,0 +1,55 @@
+import json
+import yaml
+import argparse
+import os.path
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", required=True, type=str, help="gitlab pipeline to merge")
+    parser.add_argument("--output", required=True, type=str, help="final gitlab pipeline")
+
+    args = parser.parse_args()
+    with open(args.input, "r") as f:
+        pipeline = yaml.safe_load(f)
+
+    if os.path.exists(args.output):
+        # If final file exists, merge the stages and jobs
+        with open(args.output, "r") as f:
+            final_pipeline = yaml.safe_load(f)
+            for stage in pipeline["stages"]:
+                if stage not in final_pipeline["stages"]:
+                    final_pipeline["stages"].append(stage)
+            for job in pipeline:
+                if job not in final_pipeline:
+                    final_pipeline[job] = pipeline[job]
+    else:
+        # If final file does not exist, just copy the pipeline
+        final_pipeline = pipeline
+
+    # Workaround to set cache stage as the last stage
+    # and split in stages with not more than 100 jobs
+    set_cache_2 = False
+    for key in final_pipeline:
+        if "stage" in final_pipeline[key] and (
+            final_pipeline[key]["stage"] == "Cache" or final_pipeline[key]["stage"] == "Cache2"
+        ):
+            final_pipeline[key]["stage"] = "Cache2" if set_cache_2 else "Cache"
+            set_cache_2 = not set_cache_2
+
+    if "Cache" in final_pipeline["stages"]:
+        final_pipeline["stages"].remove("Cache")
+        final_pipeline["stages"].append("Cache")
+    if "Cache2" in final_pipeline["stages"]:
+        final_pipeline["stages"].remove("Cache2")
+        final_pipeline["stages"].append("Cache2")
+    else:
+        final_pipeline["stages"].append("Cache2")
+
+    # Write the final pipeline
+    with open(args.output, "w") as f:
+        f.write(yaml.dump(final_pipeline, sort_keys=False, default_flow_style=False))
+
+
+if __name__ == "__main__":
+    main()