feat(ci): more meta backend agnostic e2e tests in main-cron

ci/docker-compose.yml

@@ -113,7 +113,8 @@ services:
     volumes:
       - ..:/risingwave
 
-  pg-mysql-backend-test-env:
+  # Standard environment for CI, including MySQL and Postgres for metadata.
+  ci-standard-env:
     image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
     depends_on:
       - mysql

ci/workflows/main-cron.yml

@@ -33,7 +33,7 @@ docker-compose-common: &docker-compose-common
     config: ci/docker-compose.yml
     mount-buildkite-agent: true
     propagate-environment: true
-    run: rw-build-env
+    run: ci-standard-env
 
 steps:
   - label: "build"
@@ -93,89 +93,65 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test (release, {{matrix.backend}} backend)"
-    key: "e2e-test-release"
-    <<: *sql-backend
-    command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-test"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.1.0:
-          <<: *docker-compose-common
-          run: pg-mysql-backend-test-env
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
-    retry: *auto-retry
-
-  - label: "slow end-to-end test (release)"
-    key: "slow-e2e-test-release"
-    command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
-      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 8
-    retry: *auto-retry
-
-  - label: "meta backup test (release)"
-    key: "e2e-meta-backup-test-release"
-    command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 45
-    retry: *auto-retry
-
-  - label: "end-to-end test (parallel) (release)"
-    key: "e2e-test-release-parallel"
-    command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "docslt"
-    plugins:
-      - seek-oss/aws-sm#v2.3.1:
-          env:
-            BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - test-collector#v1.0.0:
-          files: "*-junit.xml"
-          format: "junit"
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 13
-    retry: *auto-retry
+  - group: "end-to-end test (release)"
+    steps:
+    - label: "end-to-end test ({{matrix.backend}} backend)"
+      key: "e2e-test-release"
+      <<: *sql-backend
+      command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-test"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+        - "docslt"
+      plugins:
+        - docker-compose#v5.1.0: *docker-compose-common
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 30
+      retry: *auto-retry
+
+    - label: "slow end-to-end test ({{matrix.backend}} backend)"
+      key: "slow-e2e-test-release"
+      <<: *sql-backend
+      command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+        || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+      plugins:
+        - docker-compose#v5.1.0: *docker-compose-common
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 8
+      retry: *auto-retry
+
+    - label: "end-to-end test (parallel, {{matrix.backend}} backend)"
+      key: "e2e-test-release-parallel"
+      <<: *sql-backend
+      command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "docslt"
+      plugins:
+        - seek-oss/aws-sm#v2.3.1:
+            env:
+              BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token
+        - docker-compose#v5.1.0: *docker-compose-common
+        - test-collector#v1.0.0:
+            files: "*-junit.xml"
+            format: "junit"
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 13
+      retry: *auto-retry
 
   - label: "end-to-end test (parallel, in-memory) (release)"
     key: "e2e-test-release-parallel-memory"
@@ -196,62 +172,84 @@ steps:
     timeout_in_minutes: 12
     retry: *auto-retry
 
-  - label: "end-to-end source test (release)"
-    key: "e2e-test-release-source"
-    command: "ci/scripts/e2e-source-test.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-source-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: source-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 15
-    retry: *auto-retry
+  - group: "end-to-end connector test (release)"
+    steps:
+    - label: "end-to-end source test ({{matrix.backend}} backend)"
+      key: "e2e-test-release-source"
+      <<: *sql-backend
+      command: "ci/scripts/e2e-source-test.sh -p ci-release"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-source-tests"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+      plugins:
+        - docker-compose#v5.1.0:
+            <<: *docker-compose-common
+            run: source-test-env
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 15
+      retry: *auto-retry
+
+    - label: "end-to-end sink test ({{matrix.backend}} backend)"
+      key: "e2e-test-release-sink"
+      <<: *sql-backend
+      command: "ci/scripts/e2e-sink-test.sh -p ci-release"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-sink-tests"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+      plugins:
+        - docker-compose#v5.1.0:
+            <<: *docker-compose-common
+            run: sink-test-env
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 35
+      retry: *auto-retry
 
-  - label: "end-to-end sink test (release)"
-    key: "e2e-test-release-sink"
-    command: "ci/scripts/e2e-sink-test.sh -p ci-release"
+  - label: "fuzz test"
+    key: "fuzz-test"
+    command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
+      || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests"
+      || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/
     depends_on:
       - "build"
-      - "build-other"
+      - "build-simulation"
     plugins:
+      - ./ci/plugins/swapfile
       - docker-compose#v5.1.0:
-          run: sink-test-env
+          run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 35
+    timeout_in_minutes: 20
     retry: *auto-retry
 
-  - label: "fuzz test"
-    key: "fuzz-test"
-    command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
+  - label: "meta backup test (release)"
+    key: "e2e-meta-backup-test-release"
+    command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests"
-      || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/
+      || build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
     depends_on:
       - "build"
-      - "build-simulation"
+      - "build-other"
+      - "docslt"
     plugins:
-      - ./ci/plugins/swapfile
       - docker-compose#v5.1.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 20
+    timeout_in_minutes: 45
     retry: *auto-retry
 
   # The timeout should be strictly more than timeout in `pull-request.yml`.

ci/workflows/pull-request.yml

@@ -44,7 +44,7 @@ docker-compose-common: &docker-compose-common
     config: ci/docker-compose.yml
     mount-buildkite-agent: true
     propagate-environment: true
-    run: rw-build-env
+    run: ci-standard-env
 
 steps:
   - label: "check ci image rebuild"
@@ -825,9 +825,7 @@ steps:
       - "build-other"
       - "docslt"
     plugins:
-      - docker-compose#v5.1.0:
-          <<: *docker-compose-common
-          run: pg-mysql-backend-test-env
+      - docker-compose#v5.1.0: *docker-compose-common
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 32
     retry: *auto-retry