ci: parallelize e2e_test/source_inline (#18971)

Signed-off-by: xxchan <[email protected]>

ci/scripts/e2e-source-test.sh

@@ -38,7 +38,8 @@ apt-get -y install jq
 echo "--- e2e, inline test"
 RUST_LOG="debug,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 risedev ci-start ci-inline-source-test
-risedev slt './e2e_test/source_inline/**/*.slt'
+risedev slt './e2e_test/source_inline/**/*.slt' -j16
+risedev slt './e2e_test/source_inline/**/*.slt.serial'
 echo "--- Kill cluster"
 risedev ci-kill
 

e2e_test/source_inline/README.md

@@ -5,6 +5,19 @@ Compared with prior source tests ( `e2e_test/source` ), tests in this directory
 See the [connector development guide](http://risingwavelabs.github.io/risingwave/connector/intro.html#end-to-end-tests) for more information about how to set up the test environment,
 run tests, and write tests.
 
+## Serial Tests
+
+Tests ending with `.slt.serial` can only be run in serial, e.g., it contains `RECOVER` statement which will break other connections, or it has some special `system` commands.
+
+Other tests can be run in parallel.
+
+```bash
+# run all parallel tests
+risedev slt './e2e_test/source_inline/**/*.slt' -j16
+# run all serial tests
+risedev slt './e2e_test/source_inline/**/*.slt.serial'
+```
+
 ## Install Dependencies
 
 Some additional tools are needed to run the `system` commands in tests.

e2e_test/source_inline/kafka/protobuf/alter_source.slt

@@ -1,18 +1,18 @@
 control substitution on
 
 system ok
-rpk topic delete sr_pb_test || true; \
-(rpk sr subject delete 'sr_pb_test-value' && rpk sr subject delete 'sr_pb_test-value' --permanent) || true;
+rpk topic delete pb_alter_source_test || true; \
+(rpk sr subject delete 'pb_alter_source_test-value' && rpk sr subject delete 'pb_alter_source_test-value' --permanent) || true;
 
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 20 user
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 20 user
 
 statement ok
 CREATE SOURCE src_user
 INCLUDE timestamp -- include explicitly here to test a bug found in https://github.com/risingwavelabs/risingwave/pull/17293
 WITH (
     ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
-    topic = 'sr_pb_test',
+    topic = 'pb_alter_source_test',
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
@@ -26,7 +26,7 @@ CREATE MATERIALIZED VIEW mv_user AS SELECT * FROM src_user;
 statement ok
 CREATE TABLE t_user WITH (
     ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
-    topic = 'sr_pb_test',
+    topic = 'pb_alter_source_test',
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
@@ -42,7 +42,7 @@ SELECT age FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 5 user_with_more_fields
 
 sleep 5s
 
@@ -69,7 +69,7 @@ SELECT COUNT(*), MAX(age), MIN(age), SUM(age) FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 5 user_with_more_fields
 
 sleep 5s