Lay groundwork for RFS jenkins job

.github/workflows/rfs_pr_e2e_test.yml

@@ -0,0 +1,40 @@
+name: Jenkins
+
+on:
+  push:
+    branches-ignore:
+      - 'backport/**'
+      - 'dependabot/**'
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+env:
+  python-version: '3.11'
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  rfs-e2e-aws-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Determine Github repository and branch
+        id: determine-repo-vars
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
+            branch_name="${{ github.event.pull_request.head.ref }}"
+            pr_repo_url="https://github.com/${{ github.event.pull_request.head.repo.full_name }}.git"
+          else
+            branch_name="${{ github.ref_name }}"
+            pr_repo_url="https://github.com/${{ github.repository }}.git"
+          fi
+          echo "Running jenkins test on repo: $pr_repo_url and branch: $branch_name"
+          echo "branch_name=$branch_name" >> $GITHUB_OUTPUT
+          echo "pr_repo_url=$pr_repo_url" >> $GITHUB_OUTPUT
+      - name: Jenkins Job Trigger and Monitor
+        uses: lewijacn/[email protected]
+        with:
+          jenkins_url: "https://migrations.ci.opensearch.org"
+          job_name: "rfs-default-e2e-test"
+          api_token: "${{ secrets.JENKINS_MIGRATIONS_GENERIC_WEBHOOK_TOKEN }}"
+          job_params: "GIT_REPO_URL=${{ steps.determine-repo-vars.outputs.pr_repo_url }},GIT_BRANCH=${{ steps.determine-repo-vars.outputs.branch_name }}"

jenkins/migrationIntegPipelines/rfsDefaultE2ETestCover.groovy

@@ -0,0 +1,9 @@
+def gitBranch = params.GIT_BRANCH ?: 'main'
+def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git'
+
+library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM(
+        [$class: 'GitSCMSource',
+         remote: "${gitUrl}"])
+
+// Shared library function (location from root: vars/rfsDefaultE2ETest.groovy)
+rfsDefaultE2ETest()

jenkins/migrationIntegPipelines/trafficReplayDefaultE2ETestCover.groovy

@@ -0,0 +1,9 @@
+def gitBranch = params.GIT_BRANCH ?: 'main'
+def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git'
+
+library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM(
+        [$class: 'GitSCMSource',
+         remote: "${gitUrl}"])
+
+// Shared library function (location from root: vars/trafficReplayDefaultE2ETest.groovy)
+trafficReplayDefaultE2ETest()

vars/defaultIntegPipeline.groovy

@@ -2,6 +2,7 @@ def call(Map config = [:]) {
     def sourceContext = config.sourceContext
     def migrationContext = config.migrationContext
     def defaultStageId = config.defaultStageId
+    def jobName = config.jobName
     if(sourceContext == null || sourceContext.isEmpty()){
         throw new RuntimeException("The sourceContext argument must be provided");
     }
@@ -11,11 +12,16 @@ def call(Map config = [:]) {
     if(defaultStageId == null || defaultStageId.isEmpty()){
         throw new RuntimeException("The defaultStageId argument must be provided");
     }
+    if(jobName == null || jobName.isEmpty()){
+        throw new RuntimeException("The jobName argument must be provided");
+    }
     def source_context_id = config.sourceContextId ?: 'source-single-node-ec2'
     def migration_context_id = config.migrationContextId ?: 'migration-default'
     def source_context_file_name = 'sourceJenkinsContext.json'
     def migration_context_file_name = 'migrationJenkinsContext.json'
     def skipCaptureProxyOnNodeSetup = config.skipCaptureProxyOnNodeSetup ?: false
+    def testDir = "/root/lib/integ_test/integ_test"
+    def integTestCommand = config.integTestCommand ?: "${testDir}/replayer_tests.py"
     pipeline {
         agent { label config.workerAgent ?: 'Jenkins-Default-Agent-X64-C5xlarge-Single-Host' }
 
@@ -25,6 +31,27 @@ def call(Map config = [:]) {
             string(name: 'STAGE', defaultValue: "${defaultStageId}", description: 'Stage name for deployment environment')
         }
 
+        options {
+            // Acquire lock on a given deployment stage
+            lock(label: params.STAGE, quantity: 1, variable: 'stage')
+            timeout(time: 3, unit: 'HOURS')
+            buildDiscarder(logRotator(daysToKeepStr: '30'))
+        }
+
+        triggers {
+            GenericTrigger(
+                    genericVariables: [
+                            [key: 'GIT_REPO_URL', value: '$.GIT_REPO_URL'],
+                            [key: 'GIT_BRANCH', value: '$.GIT_BRANCH'],
+                            [key: 'job_name', value: '$.job_name']
+                    ],
+                    tokenCredentialId: 'jenkins-migrations-generic-webhook-token',
+                    causeString: 'Triggered by PR on opensearch-migrations repository',
+                    regexpFilterExpression: "^$jobName\$",
+                    regexpFilterText: "\$job_name",
+            )
+        }
+
         stages {
             stage('Checkout') {
                 steps {
@@ -92,13 +119,14 @@ def call(Map config = [:]) {
                                 if (config.deployStep) {
                                     config.deployStep()
                                 } else {
+                                    echo "Acquired deployment stage: ${stage}"
                                     sh 'sudo usermod -aG docker $USER'
                                     sh 'sudo newgrp docker'
                                     def baseCommand = "sudo --preserve-env ./awsE2ESolutionSetup.sh --source-context-file './$source_context_file_name' " +
                                             "--migration-context-file './$migration_context_file_name' " +
                                             "--source-context-id $source_context_id " +
                                             "--migration-context-id $migration_context_id " +
-                                            "--stage ${params.STAGE} " +
+                                            "--stage ${stage} " +
                                             "--migrations-git-url ${params.GIT_REPO_URL} " +
                                             "--migrations-git-branch ${params.GIT_BRANCH}"
                                     if (skipCaptureProxyOnNodeSetup) {
@@ -127,17 +155,16 @@ def call(Map config = [:]) {
                                 } else {
                                     def time = new Date().getTime()
                                     def uniqueId = "integ_min_${time}_${currentBuild.number}"
-                                    def test_dir = "/root/lib/integ_test/integ_test"
-                                    def test_result_file = "${test_dir}/reports/${uniqueId}/report.xml"
-                                    def command = "pipenv run pytest --log-file=${test_dir}/reports/${uniqueId}/pytest.log " +
-                                            "--junitxml=${test_result_file} ${test_dir}/replayer_tests.py " +
+                                    def test_result_file = "${testDir}/reports/${uniqueId}/report.xml"
+                                    def command = "pipenv run pytest --log-file=${testDir}/reports/${uniqueId}/pytest.log " +
+                                            "--junitxml=${test_result_file} ${integTestCommand} " +
                                             "--unique_id ${uniqueId} " +
                                             "-s"
                                     withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) {
                                         withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", duration: 3600, roleSessionName: 'jenkins-session') {
                                             sh "sudo --preserve-env ./awsRunIntegTests.sh --command '${command}' " +
                                                     "--test-result-file ${test_result_file} " +
-                                                    "--stage ${params.STAGE}"
+                                                    "--stage ${stage}"
                                         }
                                     }
                                 }

vars/rfsDefaultE2ETest.groovy

@@ -0,0 +1,57 @@
+// Note:
+// 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to
+//    allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK.
+
+def call(Map config = [:]) {
+    def sourceContextId = 'source-single-node-ec2'
+    def migrationContextId = 'migration-rfs'
+    def source_cdk_context = """
+        {
+          "source-single-node-ec2": {
+            "suffix": "ec2-source-<STAGE>",
+            "networkStackSuffix": "ec2-source-<STAGE>",
+            "distVersion": "7.10.2",
+            "distributionUrl": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz",
+            "captureProxyEnabled": false,
+            "securityDisabled": true,
+            "minDistribution": false,
+            "cpuArch": "x64",
+            "isInternal": true,
+            "singleNodeCluster": true,
+            "networkAvailabilityZones": 2,
+            "dataNodeCount": 1,
+            "managerNodeCount": 0,
+            "serverAccessType": "ipv4",
+            "restrictServerAccessTo": "0.0.0.0/0"
+          }
+        }
+    """
+    def migration_cdk_context = """
+        {
+          "migration-rfs": {
+            "stage": "<STAGE>",
+            "vpcId": "<VPC_ID>",
+            "engineVersion": "OS_2.11",
+            "domainName": "os-cluster-<STAGE>",
+            "dataNodeCount": 2,
+            "openAccessPolicyEnabled": true,
+            "domainRemovalPolicy": "DESTROY",
+            "artifactBucketRemovalPolicy": "DESTROY",
+            "trafficReplayerServiceEnabled": false,
+            "reindexFromSnapshotServiceEnabled": true,
+            "sourceClusterEndpoint": "<SOURCE_CLUSTER_ENDPOINT>"
+          }
+        }
+    """
+
+    defaultIntegPipeline(
+            sourceContext: source_cdk_context,
+            migrationContext: migration_cdk_context,
+            sourceContextId: sourceContextId,
+            migrationContextId: migrationContextId,
+            defaultStageId: 'rfs-integ',
+            skipCaptureProxyOnNodeSetup: true,
+            jobName: 'rfs-default-e2e-test',
+            integTestCommand: '/root/lib/integ_test/integ_test/backfill_tests.py'
+    )
+}

jenkins/migrationIntegPipelines/ec2SourceE2EPipeline.groovy → vars/trafficReplayDefaultE2ETest.groovy

@@ -2,13 +2,10 @@
 // 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to
 //    allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK.
 
-def sourceContextId = 'source-single-node-ec2'
-def migrationContextId = 'migration-default'
-// These default values should only be used on the initial Jenkins run in order to load parameter options into the UI,
-// all future runs should use the specified parameters
-def gitBranch = params.GIT_BRANCH ?: 'main'
-def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git'
-def source_cdk_context = """
+def call(Map config = [:]) {
+    def sourceContextId = 'source-single-node-ec2'
+    def migrationContextId = 'migration-default'
+    def source_cdk_context = """
     {
       "source-single-node-ec2": {
         "suffix": "ec2-source-<STAGE>",
@@ -29,8 +26,8 @@ def source_cdk_context = """
         "restrictServerAccessTo": "0.0.0.0/0"
       }
     }
-"""
-def migration_cdk_context = """
+    """
+    def migration_cdk_context = """
     {
       "migration-default": {
         "stage": "<STAGE>",
@@ -50,19 +47,17 @@ def migration_cdk_context = """
         "migrationAPIEnabled": true
       }
     }
-"""
+    """
 
-library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM(
-    [$class: 'GitSCMSource',
-    remote: "${gitUrl}"])
-
-defaultIntegPipeline(
-        sourceContext: source_cdk_context,
-        migrationContext: migration_cdk_context,
-        sourceContextId: sourceContextId,
-        migrationContextId: migrationContextId,
-        defaultStageId: 'aws-integ',
-        //deployStep: {
-        //    echo 'Custom Test Step'
-        //}
-)
+    defaultIntegPipeline(
+            sourceContext: source_cdk_context,
+            migrationContext: migration_cdk_context,
+            sourceContextId: sourceContextId,
+            migrationContextId: migrationContextId,
+            defaultStageId: 'aws-integ',
+            jobName: 'traffic-replay-default-e2e-test',
+            //deployStep: {
+            //    echo 'Custom Test Step'
+            //}
+    )
+}