From 4f0cc151f3bd0cfe891effb0a91384b101aebd56 Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 11 Oct 2024 16:55:08 +0800 Subject: [PATCH 1/2] test: move more Kafka tests to source_inline --- ci/scripts/deterministic-e2e-test.sh | 4 +- ci/scripts/deterministic-recovery-test.sh | 4 +- ci/scripts/e2e-source-test.sh | 8 +-- .../nexmark/nexmark_endless_part1.slt.part | 46 ----------------- e2e_test/source/basic/ttl_table_with_con.slt | 32 ------------ .../basic => source_inline/kafka}/datagen.slt | 0 .../kafka}/handling_mode.slt | 37 ++++++++------ .../basic => source_inline/kafka}/kinesis.slt | 0 .../kafka}/temporary_kafka_batch.slt | 21 ++++++-- .../kafka/ttl_table_with_con.slt | 47 ++++++++++++++++++ .../nexmark_endless_part1.slt | 0 .../nexmark_endless_part2.slt | 0 .../nexmark_endless_part3.slt | 0 .../nexmark_endless_part4.slt | 0 .../nexmark_endless_part1.slt | 0 .../nexmark_endless_part2.slt | 0 .../nexmark_endless_part3.slt | 0 .../nexmark_endless_part4.slt | 0 .../nexmark_endless_part5.slt | 0 .../nexmark_endless_part6.slt | 0 .../posix_fs}/data/data1.csv | 0 .../posix_fs}/data/data2.csv | 0 .../posix_fs}/posix_fs.slt | 2 +- e2e_test/{source => source_legacy}/README.md | 6 +-- .../cdc/cdc.check.slt | 0 .../cdc/cdc.check_new_rows.slt | 0 .../cdc/cdc.load.slt | 0 .../cdc/cdc.share_stream.slt | 0 .../cdc/cdc.validate.mysql.slt | 0 .../cdc/cdc.validate.postgres.slt | 0 .../cdc/cdc_share_stream_drop.slt | 0 .../cdc/mongodb/mongodb_basic.slt | 0 .../cdc/mysql_cdc.sql | 0 .../cdc/mysql_cdc_insert.sql | 0 .../cdc/mysql_create.sql | 0 .../cdc/mysql_init_data.sql | 0 .../cdc/postgres_cdc.sql | 0 .../cdc/postgres_cdc_insert.sql | 0 .../alter/cdc_backfill_rate_limit.slt | 0 .../cdc_inline/alter/cdc_table_alter.slt | 0 .../cdc_inline/alter/postgres_alter.slt | 0 .../cdc_inline/auto_schema_change_mysql.slt | 0 .../cdc_inline/auto_schema_map_mysql.slt | 0 .../cdc_inline/auto_schema_map_pg.slt | 0 .../cdc_inline/postgres_create_drop.slt | 2 +- .../sql_server_cdc/sql_server_cdc.slt | 0 .../sql_server_cdc/sql_server_cdc_insert.sql | 0 .../sql_server_cdc/sql_server_cdc_prepare.sql | 0 .../basic => source_legacy/kafka}/ddl.slt | 0 .../basic => source_legacy/kafka}/kafka.slt | 0 .../kafka}/kafka_batch.slt | 0 .../kafka}/nosim_kafka.slt | 0 .../kafka}/old_row_format_syntax/datagen.slt | 0 .../kafka}/old_row_format_syntax/ddl.slt | 0 .../kafka}/old_row_format_syntax/kafka.slt | 0 .../old_row_format_syntax/kafka_batch.slt | 2 +- .../source_legacy/kafka/script}/README.md | 0 .../kafka/script}/prepare_ci_kafka.sh | 10 ++-- .../kafka/script}/schema_registry_producer.py | 0 .../source_legacy/kafka/script}/test_data.zip | Bin .../test_data/avro_complex_schema_bin.1 | Bin .../test_data/avro_simple_schema_bin.1 | Bin .../kafka/script}/test_data/bug_bash.1 | 0 .../test_data/canal_json_double_field.1 | 0 .../kafka/script}/test_data/cannal_json.1 | 2 +- .../test_data/debezium_compact_avro_json.1 | 0 .../test_data/debezium_ignore_case_json.1 | 0 .../kafka/script}/test_data/debezium_log.1 | 0 .../test_data/debezium_log_no_schema_field.1 | 0 .../script}/test_data/debezium_mess_key.1 | 0 .../test_data/debezium_mongo_json_customers.1 | 0 ...ium_mongo_json_customers_no_schema_field.1 | 0 .../debezium_non_compact_avro_json.1 | 0 .../kafka/script}/test_data/json_bytea.1 | 0 .../kafka/script}/test_data/json_c.1 | 2 +- .../test_data/json_schema_without_schema.1 | 2 +- .../script}/test_data/kafka_1_csv_topic.1 | 0 .../test_data/kafka_1_partition_mv_topic.1 | 0 .../test_data/kafka_1_partition_topic.1 | 2 +- .../test_data/kafka_2_partition_topic.2 | 2 +- .../test_data/kafka_3_partition_topic.3 | 2 +- .../test_data/kafka_4_partition_topic.4 | 2 +- ...afka_4_partition_topic_generated_columns.1 | 2 +- ...kafka_4_partition_topic_with_100_message.4 | 2 +- .../script}/test_data/kafka_json_schema.1 | 2 +- .../test_data/kafka_source_format_bytes.1 | 2 + .../test_data/kafka_upsert_json_schema.1 | 2 +- .../kafka/script}/test_data/maxwell_json.1 | 2 +- .../kafka/script}/test_data/proto_c_bin.1 | 0 .../kafka/script}/test_data/sink_debezium.1 | 0 .../kafka/script}/test_data/sink_target.1 | 0 .../kafka/script}/test_data/upsert_json.1 | 2 +- .../test_data/upsert_student_avro_json.1 | 0 ...tudent_key_not_subset_of_value_avro_json.1 | 0 .../pulsar/astra-streaming.py | 0 .../pulsar/pulsar.slt | 0 .../pulsar/streamnative-cloud.py | 0 .../json_timestamptz_handling_mode.1 | 4 -- .../test_data/kafka_source_format_bytes.1 | 2 - scripts/source/test_data/weiling.1 | 1 - 100 files changed, 120 insertions(+), 136 deletions(-) delete mode 100644 e2e_test/source/basic/nexmark/nexmark_endless_part1.slt.part delete mode 100644 e2e_test/source/basic/ttl_table_with_con.slt rename e2e_test/{source/basic => source_inline/kafka}/datagen.slt (100%) rename e2e_test/{source/basic => source_inline/kafka}/handling_mode.slt (76%) rename e2e_test/{source/basic => source_inline/kafka}/kinesis.slt (100%) rename e2e_test/{source/basic => source_inline/kafka}/temporary_kafka_batch.slt (74%) create mode 100644 e2e_test/source_inline/kafka/ttl_table_with_con.slt rename e2e_test/{source => source_inline}/nexmark_endless_mvs/nexmark_endless_part1.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_mvs/nexmark_endless_part2.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_mvs/nexmark_endless_part3.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_mvs/nexmark_endless_part4.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_sinks/nexmark_endless_part1.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_sinks/nexmark_endless_part2.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_sinks/nexmark_endless_part3.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_sinks/nexmark_endless_part4.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_sinks/nexmark_endless_part5.slt (100%) rename e2e_test/{source => source_inline}/nexmark_endless_sinks/nexmark_endless_part6.slt (100%) rename e2e_test/{source/opendal => source_inline/posix_fs}/data/data1.csv (100%) rename e2e_test/{source/opendal => source_inline/posix_fs}/data/data2.csv (100%) rename e2e_test/{source/opendal => source_inline/posix_fs}/posix_fs.slt (90%) rename e2e_test/{source => source_legacy}/README.md (72%) rename e2e_test/{source => source_legacy}/cdc/cdc.check.slt (100%) rename e2e_test/{source => source_legacy}/cdc/cdc.check_new_rows.slt (100%) rename e2e_test/{source => source_legacy}/cdc/cdc.load.slt (100%) rename e2e_test/{source => source_legacy}/cdc/cdc.share_stream.slt (100%) rename e2e_test/{source => source_legacy}/cdc/cdc.validate.mysql.slt (100%) rename e2e_test/{source => source_legacy}/cdc/cdc.validate.postgres.slt (100%) rename e2e_test/{source => source_legacy}/cdc/cdc_share_stream_drop.slt (100%) rename e2e_test/{source => source_legacy}/cdc/mongodb/mongodb_basic.slt (100%) rename e2e_test/{source => source_legacy}/cdc/mysql_cdc.sql (100%) rename e2e_test/{source => source_legacy}/cdc/mysql_cdc_insert.sql (100%) rename e2e_test/{source => source_legacy}/cdc/mysql_create.sql (100%) rename e2e_test/{source => source_legacy}/cdc/mysql_init_data.sql (100%) rename e2e_test/{source => source_legacy}/cdc/postgres_cdc.sql (100%) rename e2e_test/{source => source_legacy}/cdc/postgres_cdc_insert.sql (100%) rename e2e_test/{source => source_legacy}/cdc_inline/alter/cdc_backfill_rate_limit.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/alter/cdc_table_alter.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/alter/postgres_alter.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/auto_schema_change_mysql.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/auto_schema_map_mysql.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/auto_schema_map_pg.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/postgres_create_drop.slt (98%) rename e2e_test/{source => source_legacy}/cdc_inline/sql_server_cdc/sql_server_cdc.slt (100%) rename e2e_test/{source => source_legacy}/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql (100%) rename e2e_test/{source => source_legacy}/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql (100%) rename e2e_test/{source/basic => source_legacy/kafka}/ddl.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/kafka.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/kafka_batch.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/nosim_kafka.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/old_row_format_syntax/datagen.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/old_row_format_syntax/ddl.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/old_row_format_syntax/kafka.slt (100%) rename e2e_test/{source/basic => source_legacy/kafka}/old_row_format_syntax/kafka_batch.slt (99%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/README.md (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/prepare_ci_kafka.sh (91%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/schema_registry_producer.py (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data.zip (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/avro_complex_schema_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/avro_simple_schema_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/bug_bash.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/canal_json_double_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/cannal_json.1 (98%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_compact_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_ignore_case_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_log.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_log_no_schema_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_mess_key.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_mongo_json_customers.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_mongo_json_customers_no_schema_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/debezium_non_compact_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/json_bytea.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/json_c.1 (85%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/json_schema_without_schema.1 (95%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_1_csv_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_1_partition_mv_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_1_partition_topic.1 (73%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_2_partition_topic.2 (73%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_3_partition_topic.3 (73%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_4_partition_topic.4 (73%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_4_partition_topic_generated_columns.1 (81%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_4_partition_topic_with_100_message.4 (99%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_json_schema.1 (94%) create mode 100644 e2e_test/source_legacy/kafka/script/test_data/kafka_source_format_bytes.1 rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/kafka_upsert_json_schema.1 (99%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/maxwell_json.1 (88%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/proto_c_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/sink_debezium.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/sink_target.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/upsert_json.1 (99%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/upsert_student_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/kafka/script}/test_data/upsert_student_key_not_subset_of_value_avro_json.1 (100%) rename e2e_test/{source => source_legacy}/pulsar/astra-streaming.py (100%) rename e2e_test/{source => source_legacy}/pulsar/pulsar.slt (100%) rename e2e_test/{source => source_legacy}/pulsar/streamnative-cloud.py (100%) delete mode 100644 scripts/source/test_data/json_timestamptz_handling_mode.1 delete mode 100644 scripts/source/test_data/kafka_source_format_bytes.1 delete mode 100644 scripts/source/test_data/weiling.1 diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh index f93ded7bad4a..6796783af10a 100755 --- a/ci/scripts/deterministic-e2e-test.sh +++ b/ci/scripts/deterministic-e2e-test.sh @@ -11,7 +11,7 @@ download-and-decompress-artifact risingwave_simulation . chmod +x ./risingwave_simulation echo "--- Extract data for Kafka" -pushd ./scripts/source/ +pushd ./e2e_test/source_legacy/kafka/script mkdir -p ./test_data unzip -o test_data.zip -d . popd @@ -39,7 +39,7 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, batch" seq "$TEST_NUM" | parallel './risingwave_simulation ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, kafka source" -seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./scripts/source/test_data ./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log' +seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./e2e_test/source_legacy/kafka/script/test_data ./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, streaming" seq "$TEST_NUM" | parallel './risingwave_simulation -j 16 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/parallel-streaming-{}.log && rm $LOGDIR/parallel-streaming-{}.log' diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index b38adfb3a83d..2bf630ccbc27 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -93,6 +93,6 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, kafka source seq "$TEST_NUM" | parallel './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ ---kafka-datadir=./scripts/source/test_data \ +--kafka-datadir=./e2e_test/source_legacy/kafka/script/test_data \ ${EXTRA_ARGS:-} \ -./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' \ No newline at end of file +./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index f10ee3ebef09..3a5dbb13055c 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -148,13 +148,13 @@ echo "--- Kill cluster" risedev ci-kill export RISINGWAVE_CI=true -echo "--- e2e, ci-kafka-plus-pubsub, kafka and pubsub source" +echo "--- e2e, ci-kafka, legacy kafka tests" export RUST_MIN_STACK=4194304 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ risedev ci-start ci-kafka -./scripts/source/prepare_ci_kafka.sh -risedev slt './e2e_test/source/basic/*.slt' -risedev slt './e2e_test/source/basic/old_row_format_syntax/*.slt' +./e2e_test/source_legacy/kafka/script/prepare_ci_kafka.sh +risedev slt './e2e_test/source_legacy/kafka/*.slt' +risedev slt './e2e_test/source_legacy/kafka/old_row_format_syntax/*.slt' echo "--- Run CH-benCHmark" risedev slt './e2e_test/ch_benchmark/batch/ch_benchmark.slt' diff --git a/e2e_test/source/basic/nexmark/nexmark_endless_part1.slt.part b/e2e_test/source/basic/nexmark/nexmark_endless_part1.slt.part deleted file mode 100644 index 3232f55fe54e..000000000000 --- a/e2e_test/source/basic/nexmark/nexmark_endless_part1.slt.part +++ /dev/null @@ -1,46 +0,0 @@ -include ../../../nexmark/create_sources.slt.part - -include ../../../streaming/nexmark/views/q3.slt.part - -sleep 10s - -query I -select count(*) > 0 from (select * from nexmark_q3 limit 1); ----- -t - -statement ok -drop materialized view nexmark_q3; - -include ../../../nexmark/drop_sources.slt.part - -include ../../../nexmark/create_sources.slt.part -include ../../../streaming/nexmark/views/q4.slt.part - -sleep 10s - -query I -select count(*) > 0 from (select * from nexmark_q4 limit 1); ----- -t - -statement ok -drop materialized view nexmark_q4; - - -include ../../../nexmark/drop_sources.slt.part - -include ../../../nexmark/create_sources.slt.part -include ../../../streaming/nexmark/views/q5.slt.part - -sleep 20s - -query I -select count(*) > 0 from (select * from nexmark_q5 limit 1); ----- -t - -statement ok -drop materialized view nexmark_q5; - -include ../../../nexmark/drop_sources.slt.part diff --git a/e2e_test/source/basic/ttl_table_with_con.slt b/e2e_test/source/basic/ttl_table_with_con.slt deleted file mode 100644 index 6a232bbdee2a..000000000000 --- a/e2e_test/source/basic/ttl_table_with_con.slt +++ /dev/null @@ -1,32 +0,0 @@ -statement ok -create table t (v1 int, v2 varchar) APPEND ONLY with ( - connector = 'kafka', - topic = 'kafka_1_partition_topic', - properties.bootstrap.server = 'message_queue:29092', - scan.startup.mode = 'earliest', - retention_seconds = 5 -) FORMAT PLAIN ENCODE JSON; - -statement ok -flush; - -# Wait enough time to ensure SourceExecutor consumes all Kafka data. -sleep 1s - -query IT rowsort -select * from t ----- -1 1 -2 22 -3 333 -4 4444 - -statement ok -select pg_sleep(10); - -query I -select * from t; ----- - -statement ok -drop table t; \ No newline at end of file diff --git a/e2e_test/source/basic/datagen.slt b/e2e_test/source_inline/kafka/datagen.slt similarity index 100% rename from e2e_test/source/basic/datagen.slt rename to e2e_test/source_inline/kafka/datagen.slt diff --git a/e2e_test/source/basic/handling_mode.slt b/e2e_test/source_inline/kafka/handling_mode.slt similarity index 76% rename from e2e_test/source/basic/handling_mode.slt rename to e2e_test/source_inline/kafka/handling_mode.slt index caf9be9a01fc..6399118c9b03 100644 --- a/e2e_test/source/basic/handling_mode.slt +++ b/e2e_test/source_inline/kafka/handling_mode.slt @@ -1,9 +1,24 @@ +control substitution on + +system ok +rpk topic delete json_timestamptz_handling_mode || true + +system ok +rpk topic create json_timestamptz_handling_mode -p 1 + +system ok +cat <>) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format plain encode json (timestamptz.handling.mode = 'mili'); @@ -13,8 +28,7 @@ create table plain_guess ( "case" varchar, payload struct>) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format plain encode json (timestamptz.handling.mod = 'mili'); @@ -23,8 +37,7 @@ create table plain_milli ( "case" varchar, payload struct>) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format plain encode json (timestamptz.handling.mode = 'milli'); @@ -33,8 +46,7 @@ create table plain_micro ( "case" varchar, payload struct>) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format plain encode json (timestamptz.handling.mode = 'micro'); @@ -43,8 +55,7 @@ create table plain_utc ( "case" varchar, payload struct>) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format plain encode json (timestamptz.handling.mode = 'utc_string'); @@ -53,8 +64,7 @@ create table plain_naive ( "case" varchar, payload struct>) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format plain encode json (timestamptz.handling.mode = 'utc_without_suffix'); @@ -62,8 +72,7 @@ statement ok create table debezium_milli ( "case" varchar, at timestamptz, primary key("case")) with ( - connector = 'kafka', - properties.bootstrap.server='message_queue:29092', + ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, topic = 'json_timestamptz_handling_mode') format debezium encode json (timestamptz.handling.mode = 'milli'); diff --git a/e2e_test/source/basic/kinesis.slt b/e2e_test/source_inline/kafka/kinesis.slt similarity index 100% rename from e2e_test/source/basic/kinesis.slt rename to e2e_test/source_inline/kafka/kinesis.slt diff --git a/e2e_test/source/basic/temporary_kafka_batch.slt b/e2e_test/source_inline/kafka/temporary_kafka_batch.slt similarity index 74% rename from e2e_test/source/basic/temporary_kafka_batch.slt rename to e2e_test/source_inline/kafka/temporary_kafka_batch.slt index 578cc76b47d5..69144a884486 100644 --- a/e2e_test/source/basic/temporary_kafka_batch.slt +++ b/e2e_test/source_inline/kafka/temporary_kafka_batch.slt @@ -1,8 +1,23 @@ +control substitution on + +system ok +rpk topic delete test_temporary_kafka_batch || true + +system ok +rpk topic create test_temporary_kafka_batch -p 1 + +system ok +cat < [!NOTE] > > Please write new tests according to the style in `e2e_test/source_inline`. -> Don't add new tests here. +> DO NOT ADD NEW TESTS HERE! > > See the [connector development guide](http://risingwavelabs.github.io/risingwave/connector/intro.html#end-to-end-tests) for more information about how to test. Test in this directory needs some prior setup. See also `ci/scripts/e2e-source-test.sh`, and `scripts/source` - -## Kafka - -`scripts/source/test_data` contains the data. Filename's convention is `.`. diff --git a/e2e_test/source/cdc/cdc.check.slt b/e2e_test/source_legacy/cdc/cdc.check.slt similarity index 100% rename from e2e_test/source/cdc/cdc.check.slt rename to e2e_test/source_legacy/cdc/cdc.check.slt diff --git a/e2e_test/source/cdc/cdc.check_new_rows.slt b/e2e_test/source_legacy/cdc/cdc.check_new_rows.slt similarity index 100% rename from e2e_test/source/cdc/cdc.check_new_rows.slt rename to e2e_test/source_legacy/cdc/cdc.check_new_rows.slt diff --git a/e2e_test/source/cdc/cdc.load.slt b/e2e_test/source_legacy/cdc/cdc.load.slt similarity index 100% rename from e2e_test/source/cdc/cdc.load.slt rename to e2e_test/source_legacy/cdc/cdc.load.slt diff --git a/e2e_test/source/cdc/cdc.share_stream.slt b/e2e_test/source_legacy/cdc/cdc.share_stream.slt similarity index 100% rename from e2e_test/source/cdc/cdc.share_stream.slt rename to e2e_test/source_legacy/cdc/cdc.share_stream.slt diff --git a/e2e_test/source/cdc/cdc.validate.mysql.slt b/e2e_test/source_legacy/cdc/cdc.validate.mysql.slt similarity index 100% rename from e2e_test/source/cdc/cdc.validate.mysql.slt rename to e2e_test/source_legacy/cdc/cdc.validate.mysql.slt diff --git a/e2e_test/source/cdc/cdc.validate.postgres.slt b/e2e_test/source_legacy/cdc/cdc.validate.postgres.slt similarity index 100% rename from e2e_test/source/cdc/cdc.validate.postgres.slt rename to e2e_test/source_legacy/cdc/cdc.validate.postgres.slt diff --git a/e2e_test/source/cdc/cdc_share_stream_drop.slt b/e2e_test/source_legacy/cdc/cdc_share_stream_drop.slt similarity index 100% rename from e2e_test/source/cdc/cdc_share_stream_drop.slt rename to e2e_test/source_legacy/cdc/cdc_share_stream_drop.slt diff --git a/e2e_test/source/cdc/mongodb/mongodb_basic.slt b/e2e_test/source_legacy/cdc/mongodb/mongodb_basic.slt similarity index 100% rename from e2e_test/source/cdc/mongodb/mongodb_basic.slt rename to e2e_test/source_legacy/cdc/mongodb/mongodb_basic.slt diff --git a/e2e_test/source/cdc/mysql_cdc.sql b/e2e_test/source_legacy/cdc/mysql_cdc.sql similarity index 100% rename from e2e_test/source/cdc/mysql_cdc.sql rename to e2e_test/source_legacy/cdc/mysql_cdc.sql diff --git a/e2e_test/source/cdc/mysql_cdc_insert.sql b/e2e_test/source_legacy/cdc/mysql_cdc_insert.sql similarity index 100% rename from e2e_test/source/cdc/mysql_cdc_insert.sql rename to e2e_test/source_legacy/cdc/mysql_cdc_insert.sql diff --git a/e2e_test/source/cdc/mysql_create.sql b/e2e_test/source_legacy/cdc/mysql_create.sql similarity index 100% rename from e2e_test/source/cdc/mysql_create.sql rename to e2e_test/source_legacy/cdc/mysql_create.sql diff --git a/e2e_test/source/cdc/mysql_init_data.sql b/e2e_test/source_legacy/cdc/mysql_init_data.sql similarity index 100% rename from e2e_test/source/cdc/mysql_init_data.sql rename to e2e_test/source_legacy/cdc/mysql_init_data.sql diff --git a/e2e_test/source/cdc/postgres_cdc.sql b/e2e_test/source_legacy/cdc/postgres_cdc.sql similarity index 100% rename from e2e_test/source/cdc/postgres_cdc.sql rename to e2e_test/source_legacy/cdc/postgres_cdc.sql diff --git a/e2e_test/source/cdc/postgres_cdc_insert.sql b/e2e_test/source_legacy/cdc/postgres_cdc_insert.sql similarity index 100% rename from e2e_test/source/cdc/postgres_cdc_insert.sql rename to e2e_test/source_legacy/cdc/postgres_cdc_insert.sql diff --git a/e2e_test/source/cdc_inline/alter/cdc_backfill_rate_limit.slt b/e2e_test/source_legacy/cdc_inline/alter/cdc_backfill_rate_limit.slt similarity index 100% rename from e2e_test/source/cdc_inline/alter/cdc_backfill_rate_limit.slt rename to e2e_test/source_legacy/cdc_inline/alter/cdc_backfill_rate_limit.slt diff --git a/e2e_test/source/cdc_inline/alter/cdc_table_alter.slt b/e2e_test/source_legacy/cdc_inline/alter/cdc_table_alter.slt similarity index 100% rename from e2e_test/source/cdc_inline/alter/cdc_table_alter.slt rename to e2e_test/source_legacy/cdc_inline/alter/cdc_table_alter.slt diff --git a/e2e_test/source/cdc_inline/alter/postgres_alter.slt b/e2e_test/source_legacy/cdc_inline/alter/postgres_alter.slt similarity index 100% rename from e2e_test/source/cdc_inline/alter/postgres_alter.slt rename to e2e_test/source_legacy/cdc_inline/alter/postgres_alter.slt diff --git a/e2e_test/source/cdc_inline/auto_schema_change_mysql.slt b/e2e_test/source_legacy/cdc_inline/auto_schema_change_mysql.slt similarity index 100% rename from e2e_test/source/cdc_inline/auto_schema_change_mysql.slt rename to e2e_test/source_legacy/cdc_inline/auto_schema_change_mysql.slt diff --git a/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt b/e2e_test/source_legacy/cdc_inline/auto_schema_map_mysql.slt similarity index 100% rename from e2e_test/source/cdc_inline/auto_schema_map_mysql.slt rename to e2e_test/source_legacy/cdc_inline/auto_schema_map_mysql.slt diff --git a/e2e_test/source/cdc_inline/auto_schema_map_pg.slt b/e2e_test/source_legacy/cdc_inline/auto_schema_map_pg.slt similarity index 100% rename from e2e_test/source/cdc_inline/auto_schema_map_pg.slt rename to e2e_test/source_legacy/cdc_inline/auto_schema_map_pg.slt diff --git a/e2e_test/source/cdc_inline/postgres_create_drop.slt b/e2e_test/source_legacy/cdc_inline/postgres_create_drop.slt similarity index 98% rename from e2e_test/source/cdc_inline/postgres_create_drop.slt rename to e2e_test/source_legacy/cdc_inline/postgres_create_drop.slt index ad7425bab3df..f23ff824f249 100644 --- a/e2e_test/source/cdc_inline/postgres_create_drop.slt +++ b/e2e_test/source_legacy/cdc_inline/postgres_create_drop.slt @@ -57,4 +57,4 @@ SELECT * FROM tt1; 1 2023-10-23 10:00:00+00:00 statement ok -drop table tt1; \ No newline at end of file +drop table tt1; diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt b/e2e_test/source_legacy/cdc_inline/sql_server_cdc/sql_server_cdc.slt similarity index 100% rename from e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt rename to e2e_test/source_legacy/cdc_inline/sql_server_cdc/sql_server_cdc.slt diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql b/e2e_test/source_legacy/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql similarity index 100% rename from e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql rename to e2e_test/source_legacy/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql b/e2e_test/source_legacy/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql similarity index 100% rename from e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql rename to e2e_test/source_legacy/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql diff --git a/e2e_test/source/basic/ddl.slt b/e2e_test/source_legacy/kafka/ddl.slt similarity index 100% rename from e2e_test/source/basic/ddl.slt rename to e2e_test/source_legacy/kafka/ddl.slt diff --git a/e2e_test/source/basic/kafka.slt b/e2e_test/source_legacy/kafka/kafka.slt similarity index 100% rename from e2e_test/source/basic/kafka.slt rename to e2e_test/source_legacy/kafka/kafka.slt diff --git a/e2e_test/source/basic/kafka_batch.slt b/e2e_test/source_legacy/kafka/kafka_batch.slt similarity index 100% rename from e2e_test/source/basic/kafka_batch.slt rename to e2e_test/source_legacy/kafka/kafka_batch.slt diff --git a/e2e_test/source/basic/nosim_kafka.slt b/e2e_test/source_legacy/kafka/nosim_kafka.slt similarity index 100% rename from e2e_test/source/basic/nosim_kafka.slt rename to e2e_test/source_legacy/kafka/nosim_kafka.slt diff --git a/e2e_test/source/basic/old_row_format_syntax/datagen.slt b/e2e_test/source_legacy/kafka/old_row_format_syntax/datagen.slt similarity index 100% rename from e2e_test/source/basic/old_row_format_syntax/datagen.slt rename to e2e_test/source_legacy/kafka/old_row_format_syntax/datagen.slt diff --git a/e2e_test/source/basic/old_row_format_syntax/ddl.slt b/e2e_test/source_legacy/kafka/old_row_format_syntax/ddl.slt similarity index 100% rename from e2e_test/source/basic/old_row_format_syntax/ddl.slt rename to e2e_test/source_legacy/kafka/old_row_format_syntax/ddl.slt diff --git a/e2e_test/source/basic/old_row_format_syntax/kafka.slt b/e2e_test/source_legacy/kafka/old_row_format_syntax/kafka.slt similarity index 100% rename from e2e_test/source/basic/old_row_format_syntax/kafka.slt rename to e2e_test/source_legacy/kafka/old_row_format_syntax/kafka.slt diff --git a/e2e_test/source/basic/old_row_format_syntax/kafka_batch.slt b/e2e_test/source_legacy/kafka/old_row_format_syntax/kafka_batch.slt similarity index 99% rename from e2e_test/source/basic/old_row_format_syntax/kafka_batch.slt rename to e2e_test/source_legacy/kafka/old_row_format_syntax/kafka_batch.slt index 44f40b90c08c..ebf63123c5dd 100644 --- a/e2e_test/source/basic/old_row_format_syntax/kafka_batch.slt +++ b/e2e_test/source_legacy/kafka/old_row_format_syntax/kafka_batch.slt @@ -201,4 +201,4 @@ statement ok drop table s8 statement ok -drop source s9 \ No newline at end of file +drop source s9 diff --git a/scripts/source/README.md b/e2e_test/source_legacy/kafka/script/README.md similarity index 100% rename from scripts/source/README.md rename to e2e_test/source_legacy/kafka/script/README.md diff --git a/scripts/source/prepare_ci_kafka.sh b/e2e_test/source_legacy/kafka/script/prepare_ci_kafka.sh similarity index 91% rename from scripts/source/prepare_ci_kafka.sh rename to e2e_test/source_legacy/kafka/script/prepare_ci_kafka.sh index 27c88e6e63af..34d40646861a 100755 --- a/scripts/source/prepare_ci_kafka.sh +++ b/e2e_test/source_legacy/kafka/script/prepare_ci_kafka.sh @@ -4,16 +4,16 @@ set -e SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" -cd "$SCRIPT_PATH/.." || exit 1 -# cwd is /scripts +# SCRIPT_PATH is e2e_test/source_legacy/kafka/script/ +# cwd is e2e_test/source_legacy/kafka/ echo "$SCRIPT_PATH" -source ../.risingwave/config/risedev-env +source ../../../.risingwave/config/risedev-env if [ "$1" == "compress" ]; then echo "Compress test_data/ into test_data.zip" - cd ./source + cd ./script zip_file=test_data.zip if [ -f "$zip_file" ]; then rm "$zip_file" @@ -23,7 +23,7 @@ if [ "$1" == "compress" ]; then fi echo "--- Extract data for Kafka" -cd ./source/ +cd ./script/ mkdir -p ./test_data/ch_benchmark/ unzip -o test_data.zip -d . cd .. diff --git a/scripts/source/schema_registry_producer.py b/e2e_test/source_legacy/kafka/script/schema_registry_producer.py similarity index 100% rename from scripts/source/schema_registry_producer.py rename to e2e_test/source_legacy/kafka/script/schema_registry_producer.py diff --git a/scripts/source/test_data.zip b/e2e_test/source_legacy/kafka/script/test_data.zip similarity index 100% rename from scripts/source/test_data.zip rename to e2e_test/source_legacy/kafka/script/test_data.zip diff --git a/scripts/source/test_data/avro_complex_schema_bin.1 b/e2e_test/source_legacy/kafka/script/test_data/avro_complex_schema_bin.1 similarity index 100% rename from scripts/source/test_data/avro_complex_schema_bin.1 rename to e2e_test/source_legacy/kafka/script/test_data/avro_complex_schema_bin.1 diff --git a/scripts/source/test_data/avro_simple_schema_bin.1 b/e2e_test/source_legacy/kafka/script/test_data/avro_simple_schema_bin.1 similarity index 100% rename from scripts/source/test_data/avro_simple_schema_bin.1 rename to e2e_test/source_legacy/kafka/script/test_data/avro_simple_schema_bin.1 diff --git a/scripts/source/test_data/bug_bash.1 b/e2e_test/source_legacy/kafka/script/test_data/bug_bash.1 similarity index 100% rename from scripts/source/test_data/bug_bash.1 rename to e2e_test/source_legacy/kafka/script/test_data/bug_bash.1 diff --git a/scripts/source/test_data/canal_json_double_field.1 b/e2e_test/source_legacy/kafka/script/test_data/canal_json_double_field.1 similarity index 100% rename from scripts/source/test_data/canal_json_double_field.1 rename to e2e_test/source_legacy/kafka/script/test_data/canal_json_double_field.1 diff --git a/scripts/source/test_data/cannal_json.1 b/e2e_test/source_legacy/kafka/script/test_data/cannal_json.1 similarity index 98% rename from scripts/source/test_data/cannal_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/cannal_json.1 index b3f4055a1320..09d508462bec 100644 --- a/scripts/source/test_data/cannal_json.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/cannal_json.1 @@ -1,4 +1,4 @@ {"data":[{"id":"1","name":"mike","is_adult":"0","balance":"1000.62","reg_time":"2018-01-01 00:00:01","win_rate":"0.65"}],"database":"demo","es":1668673394000,"id":5,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(40)","is_adult":"boolean","balance":"decimal(10,2)","reg_time":"timestamp","win_rate":"double"},"old":null,"pkNames":null,"sql":"","sqlType":{"id":4,"name":12,"is_adult":-6,"balance":3,"reg_time":93,"win_rate":8},"table":"demo","ts":1668673394788,"type":"INSERT"} {"data":[{"id":"2","name":"alice","is_adult":"1","balance":"2000.62","reg_time":"2020-01-01 00:00:01","win_rate":"0.55"}],"database":"demo","es":1668673417000,"id":6,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(40)","is_adult":"boolean","balance":"decimal(10,2)","reg_time":"timestamp","win_rate":"double"},"old":null,"pkNames":null,"sql":"","sqlType":{"id":4,"name":12,"is_adult":-6,"balance":3,"reg_time":93,"win_rate":8},"table":"demo","ts":1668673417265,"type":"INSERT"} {"data":[{"id":"1","name":"mike","is_adult":"0","balance":"1500.62","reg_time":"2018-01-01 00:00:01","win_rate":"0.65"}],"database":"demo","es":1668673476000,"id":7,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(40)","is_adult":"boolean","balance":"decimal(10,2)","reg_time":"timestamp","win_rate":"double"},"old":[{"balance":"1000.62"}],"pkNames":null,"sql":"","sqlType":{"id":4,"name":12,"is_adult":-6,"balance":3,"reg_time":93,"win_rate":8},"table":"demo","ts":1668673476732,"type":"UPDATE"} -{"data":[{"id":"2","name":"alice","is_adult":"1","balance":"2000.62","reg_time":"2020-01-01 00:00:01","win_rate":"0.55"}],"database":"demo","es":1668673493000,"id":8,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(40)","is_adult":"boolean","balance":"decimal(10,2)","reg_time":"timestamp","win_rate":"double"},"old":null,"pkNames":null,"sql":"","sqlType":{"id":4,"name":12,"is_adult":-6,"balance":3,"reg_time":93,"win_rate":8},"table":"demo","ts":1668673493638,"type":"DELETE"} \ No newline at end of file +{"data":[{"id":"2","name":"alice","is_adult":"1","balance":"2000.62","reg_time":"2020-01-01 00:00:01","win_rate":"0.55"}],"database":"demo","es":1668673493000,"id":8,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(40)","is_adult":"boolean","balance":"decimal(10,2)","reg_time":"timestamp","win_rate":"double"},"old":null,"pkNames":null,"sql":"","sqlType":{"id":4,"name":12,"is_adult":-6,"balance":3,"reg_time":93,"win_rate":8},"table":"demo","ts":1668673493638,"type":"DELETE"} diff --git a/scripts/source/test_data/debezium_compact_avro_json.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_compact_avro_json.1 similarity index 100% rename from scripts/source/test_data/debezium_compact_avro_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_compact_avro_json.1 diff --git a/scripts/source/test_data/debezium_ignore_case_json.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_ignore_case_json.1 similarity index 100% rename from scripts/source/test_data/debezium_ignore_case_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_ignore_case_json.1 diff --git a/scripts/source/test_data/debezium_log.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_log.1 similarity index 100% rename from scripts/source/test_data/debezium_log.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_log.1 diff --git a/scripts/source/test_data/debezium_log_no_schema_field.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_log_no_schema_field.1 similarity index 100% rename from scripts/source/test_data/debezium_log_no_schema_field.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_log_no_schema_field.1 diff --git a/scripts/source/test_data/debezium_mess_key.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_mess_key.1 similarity index 100% rename from scripts/source/test_data/debezium_mess_key.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_mess_key.1 diff --git a/scripts/source/test_data/debezium_mongo_json_customers.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_mongo_json_customers.1 similarity index 100% rename from scripts/source/test_data/debezium_mongo_json_customers.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_mongo_json_customers.1 diff --git a/scripts/source/test_data/debezium_mongo_json_customers_no_schema_field.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_mongo_json_customers_no_schema_field.1 similarity index 100% rename from scripts/source/test_data/debezium_mongo_json_customers_no_schema_field.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_mongo_json_customers_no_schema_field.1 diff --git a/scripts/source/test_data/debezium_non_compact_avro_json.1 b/e2e_test/source_legacy/kafka/script/test_data/debezium_non_compact_avro_json.1 similarity index 100% rename from scripts/source/test_data/debezium_non_compact_avro_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/debezium_non_compact_avro_json.1 diff --git a/scripts/source/test_data/json_bytea.1 b/e2e_test/source_legacy/kafka/script/test_data/json_bytea.1 similarity index 100% rename from scripts/source/test_data/json_bytea.1 rename to e2e_test/source_legacy/kafka/script/test_data/json_bytea.1 diff --git a/scripts/source/test_data/json_c.1 b/e2e_test/source_legacy/kafka/script/test_data/json_c.1 similarity index 85% rename from scripts/source/test_data/json_c.1 rename to e2e_test/source_legacy/kafka/script/test_data/json_c.1 index d7d12f151d77..c40b0f286119 100644 --- a/scripts/source/test_data/json_c.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/json_c.1 @@ -1 +1 @@ -{"id": 100, "code": "abc", "timestamp": 1473305798, "xfas": [{"device_model_id": 0, "device_make_id": 200, "ip": "10.0.0.1"}, {"device_model_id": 1, "device_make_id": 400, "ip": "10.0.0.2"}], "contacts": {"phones": ["1xxx", "2xxx"], "emails": ["1xxx", "2xxx"]}, "jsonb": {"blockNumber": 16938734}} \ No newline at end of file +{"id": 100, "code": "abc", "timestamp": 1473305798, "xfas": [{"device_model_id": 0, "device_make_id": 200, "ip": "10.0.0.1"}, {"device_model_id": 1, "device_make_id": 400, "ip": "10.0.0.2"}], "contacts": {"phones": ["1xxx", "2xxx"], "emails": ["1xxx", "2xxx"]}, "jsonb": {"blockNumber": 16938734}} diff --git a/scripts/source/test_data/json_schema_without_schema.1 b/e2e_test/source_legacy/kafka/script/test_data/json_schema_without_schema.1 similarity index 95% rename from scripts/source/test_data/json_schema_without_schema.1 rename to e2e_test/source_legacy/kafka/script/test_data/json_schema_without_schema.1 index 732162bf96ab..26b50c7a5c78 100644 --- a/scripts/source/test_data/json_schema_without_schema.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/json_schema_without_schema.1 @@ -1 +1 @@ -{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}} \ No newline at end of file +{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}} diff --git a/scripts/source/test_data/kafka_1_csv_topic.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_1_csv_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_csv_topic.1 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_1_csv_topic.1 diff --git a/scripts/source/test_data/kafka_1_partition_mv_topic.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_1_partition_mv_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_partition_mv_topic.1 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_1_partition_mv_topic.1 diff --git a/scripts/source/test_data/kafka_1_partition_topic.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_1_partition_topic.1 similarity index 73% rename from scripts/source/test_data/kafka_1_partition_topic.1 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_1_partition_topic.1 index 3fb1b528d5fa..53dc3ed39eb0 100644 --- a/scripts/source/test_data/kafka_1_partition_topic.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_1_partition_topic.1 @@ -1,4 +1,4 @@ {"v1": 1, "v2": "1"} {"v1": 2, "v2": "22"} {"v1": 3, "v2": "333"} -{"v1": 4, "v2": "4444"} \ No newline at end of file +{"v1": 4, "v2": "4444"} diff --git a/scripts/source/test_data/kafka_2_partition_topic.2 b/e2e_test/source_legacy/kafka/script/test_data/kafka_2_partition_topic.2 similarity index 73% rename from scripts/source/test_data/kafka_2_partition_topic.2 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_2_partition_topic.2 index 3fb1b528d5fa..53dc3ed39eb0 100644 --- a/scripts/source/test_data/kafka_2_partition_topic.2 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_2_partition_topic.2 @@ -1,4 +1,4 @@ {"v1": 1, "v2": "1"} {"v1": 2, "v2": "22"} {"v1": 3, "v2": "333"} -{"v1": 4, "v2": "4444"} \ No newline at end of file +{"v1": 4, "v2": "4444"} diff --git a/scripts/source/test_data/kafka_3_partition_topic.3 b/e2e_test/source_legacy/kafka/script/test_data/kafka_3_partition_topic.3 similarity index 73% rename from scripts/source/test_data/kafka_3_partition_topic.3 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_3_partition_topic.3 index 3fb1b528d5fa..53dc3ed39eb0 100644 --- a/scripts/source/test_data/kafka_3_partition_topic.3 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_3_partition_topic.3 @@ -1,4 +1,4 @@ {"v1": 1, "v2": "1"} {"v1": 2, "v2": "22"} {"v1": 3, "v2": "333"} -{"v1": 4, "v2": "4444"} \ No newline at end of file +{"v1": 4, "v2": "4444"} diff --git a/scripts/source/test_data/kafka_4_partition_topic.4 b/e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic.4 similarity index 73% rename from scripts/source/test_data/kafka_4_partition_topic.4 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic.4 index 3fb1b528d5fa..53dc3ed39eb0 100644 --- a/scripts/source/test_data/kafka_4_partition_topic.4 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic.4 @@ -1,4 +1,4 @@ {"v1": 1, "v2": "1"} {"v1": 2, "v2": "22"} {"v1": 3, "v2": "333"} -{"v1": 4, "v2": "4444"} \ No newline at end of file +{"v1": 4, "v2": "4444"} diff --git a/scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic_generated_columns.1 similarity index 81% rename from scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic_generated_columns.1 index 3d8a8ee5a2ff..0ca42dd33141 100644 --- a/scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic_generated_columns.1 @@ -1,3 +1,3 @@ {"v1": 10, "v2": 20} {"v2": 21, "v3": "10"} -{"v1": 0} \ No newline at end of file +{"v1": 0} diff --git a/scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 b/e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic_with_100_message.4 similarity index 99% rename from scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic_with_100_message.4 index aae7a4687ca5..baa131975cd3 100644 --- a/scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_4_partition_topic_with_100_message.4 @@ -97,4 +97,4 @@ {"v1": 96, "v2": "KK6WEX", "v3": [96, 96, 96], "v4": {"v5": 96, "v6": 97}} {"v1": 97, "v2": "XRTK3Y", "v3": [97, 97, 97], "v4": {"v5": 97, "v6": 98}} {"v1": 98, "v2": "ZQ2TCL", "v3": [98, 98, 98], "v4": {"v5": 98, "v6": 99}} -{"v1": 99, "v2": "15UCX7", "v3": [99, 99, 99], "v4": {"v5": 99, "v6": 100}} \ No newline at end of file +{"v1": 99, "v2": "15UCX7", "v3": [99, 99, 99], "v4": {"v5": 99, "v6": 100}} diff --git a/scripts/source/test_data/kafka_json_schema.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_json_schema.1 similarity index 94% rename from scripts/source/test_data/kafka_json_schema.1 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_json_schema.1 index afa96107864c..a33fac01dd9a 100644 --- a/scripts/source/test_data/kafka_json_schema.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_json_schema.1 @@ -1,2 +1,2 @@ {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productId":{"description":"The unique identifier for a product","type":"integer"},"productName":{"description":"Name of the product","type":"string"},"price":{"description":"The price of the product","type":"number","exclusiveMinimum":0},"tags":{"description":"Tags for the product","type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true},"dimensions":{"type":"object","properties":{"length":{"type":"number"},"width":{"type":"number"},"height":{"type":"number"}},"required":["length","width","height"]},"map":{"type":"object","additionalProperties":{"type":"string"}},"notMap":{"type":"object","additionalProperties":{"type":"string"},"properties":{"a":{"type":"string"}}}},"required":["productId","productName","price"]} -{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5},"map":{"foo":"bar"},"notMap":{"a":"b","ignored":"c"}} \ No newline at end of file +{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5},"map":{"foo":"bar"},"notMap":{"a":"b","ignored":"c"}} diff --git a/e2e_test/source_legacy/kafka/script/test_data/kafka_source_format_bytes.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_source_format_bytes.1 new file mode 100644 index 000000000000..25e49a775328 --- /dev/null +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_source_format_bytes.1 @@ -0,0 +1,2 @@ +kk +weawefwef diff --git a/scripts/source/test_data/kafka_upsert_json_schema.1 b/e2e_test/source_legacy/kafka/script/test_data/kafka_upsert_json_schema.1 similarity index 99% rename from scripts/source/test_data/kafka_upsert_json_schema.1 rename to e2e_test/source_legacy/kafka/script/test_data/kafka_upsert_json_schema.1 index 0f96ceb42a5a..7347417a5eda 100644 --- a/scripts/source/test_data/kafka_upsert_json_schema.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/kafka_upsert_json_schema.1 @@ -1,4 +1,4 @@ {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"id":{"description":"The unique identifier for a product","type":"integer"}},"required":["id"]}^{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"id":{"description":"The unique identifier for a product","type":"integer"},"productName":{"description":"Name of the product","type":"string"},"price":{"description":"The price of the product","type":"number","exclusiveMinimum":0},"tags":{"description":"Tags for the product","type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true},"dimensions":{"type":"object","properties":{"length":{"type":"number"},"width":{"type":"number"},"height":{"type":"number"}},"required":["length","width","height"]}},"required":["id","productName","price"]} {"id":1}^{"id":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}} {"id":2}^{"id":2,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}} -{"id":1}^{"id":1,"productName":"An ice sculpture","price":23.0,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}} \ No newline at end of file +{"id":1}^{"id":1,"productName":"An ice sculpture","price":23.0,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}} diff --git a/scripts/source/test_data/maxwell_json.1 b/e2e_test/source_legacy/kafka/script/test_data/maxwell_json.1 similarity index 88% rename from scripts/source/test_data/maxwell_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/maxwell_json.1 index 3c9aff063389..626d82c57092 100644 --- a/scripts/source/test_data/maxwell_json.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/maxwell_json.1 @@ -1,3 +1,3 @@ {"database":"test","table":"t","type":"insert","ts":1666937996,"xid":1171,"commit":true,"data":{"id":1,"name":"tom","is_adult":0,"birthday":"2017-12-31 16:00:01"}} {"database":"test","table":"t","type":"insert","ts":1666938023,"xid":1254,"commit":true,"data":{"id":2,"name":"alex","is_adult":1,"birthday":"1999-12-31 16:00:01"}} -{"database":"test","table":"t","type":"update","ts":1666938068,"xid":1373,"commit":true,"data":{"id":2,"name":"chi","is_adult":1,"birthday":"1999-12-31 16:00:01"},"old":{"name":"alex"}} \ No newline at end of file +{"database":"test","table":"t","type":"update","ts":1666938068,"xid":1373,"commit":true,"data":{"id":2,"name":"chi","is_adult":1,"birthday":"1999-12-31 16:00:01"},"old":{"name":"alex"}} diff --git a/scripts/source/test_data/proto_c_bin.1 b/e2e_test/source_legacy/kafka/script/test_data/proto_c_bin.1 similarity index 100% rename from scripts/source/test_data/proto_c_bin.1 rename to e2e_test/source_legacy/kafka/script/test_data/proto_c_bin.1 diff --git a/scripts/source/test_data/sink_debezium.1 b/e2e_test/source_legacy/kafka/script/test_data/sink_debezium.1 similarity index 100% rename from scripts/source/test_data/sink_debezium.1 rename to e2e_test/source_legacy/kafka/script/test_data/sink_debezium.1 diff --git a/scripts/source/test_data/sink_target.1 b/e2e_test/source_legacy/kafka/script/test_data/sink_target.1 similarity index 100% rename from scripts/source/test_data/sink_target.1 rename to e2e_test/source_legacy/kafka/script/test_data/sink_target.1 diff --git a/scripts/source/test_data/upsert_json.1 b/e2e_test/source_legacy/kafka/script/test_data/upsert_json.1 similarity index 99% rename from scripts/source/test_data/upsert_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/upsert_json.1 index 82866a559124..b0677b8cffb2 100644 --- a/scripts/source/test_data/upsert_json.1 +++ b/e2e_test/source_legacy/kafka/script/test_data/upsert_json.1 @@ -12,4 +12,4 @@ {"ID": 2}^{"ID": 2, "firstName": "Emily", "lastName": "Jackson", "age": 19, "height": 5.4, "weight": 110} {"ID": 3}^{"ID": 3, "firstName": "Noah", "lastName": "Thompson", "age": 21, "height": 6.3, "weight": 195} {"ID": 7}^ -{"ID": 8}^ \ No newline at end of file +{"ID": 8}^ diff --git a/scripts/source/test_data/upsert_student_avro_json.1 b/e2e_test/source_legacy/kafka/script/test_data/upsert_student_avro_json.1 similarity index 100% rename from scripts/source/test_data/upsert_student_avro_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/upsert_student_avro_json.1 diff --git a/scripts/source/test_data/upsert_student_key_not_subset_of_value_avro_json.1 b/e2e_test/source_legacy/kafka/script/test_data/upsert_student_key_not_subset_of_value_avro_json.1 similarity index 100% rename from scripts/source/test_data/upsert_student_key_not_subset_of_value_avro_json.1 rename to e2e_test/source_legacy/kafka/script/test_data/upsert_student_key_not_subset_of_value_avro_json.1 diff --git a/e2e_test/source/pulsar/astra-streaming.py b/e2e_test/source_legacy/pulsar/astra-streaming.py similarity index 100% rename from e2e_test/source/pulsar/astra-streaming.py rename to e2e_test/source_legacy/pulsar/astra-streaming.py diff --git a/e2e_test/source/pulsar/pulsar.slt b/e2e_test/source_legacy/pulsar/pulsar.slt similarity index 100% rename from e2e_test/source/pulsar/pulsar.slt rename to e2e_test/source_legacy/pulsar/pulsar.slt diff --git a/e2e_test/source/pulsar/streamnative-cloud.py b/e2e_test/source_legacy/pulsar/streamnative-cloud.py similarity index 100% rename from e2e_test/source/pulsar/streamnative-cloud.py rename to e2e_test/source_legacy/pulsar/streamnative-cloud.py diff --git a/scripts/source/test_data/json_timestamptz_handling_mode.1 b/scripts/source/test_data/json_timestamptz_handling_mode.1 deleted file mode 100644 index 4ff1440b47f2..000000000000 --- a/scripts/source/test_data/json_timestamptz_handling_mode.1 +++ /dev/null @@ -1,4 +0,0 @@ -{"case":"0 number small","payload":{"after":{"case":"0 number small","at":100},"op":"r"}} -{"case":"1 number recent","payload":{"after":{"case":"1 number recent","at":1712800800123456},"op":"r"}} -{"case":"2 string utc","payload":{"after":{"case":"2 string utc","at":"2024-04-11T02:00:00.654321Z"},"op":"r"}} -{"case":"3 string naive","payload":{"after":{"case":"3 string naive","at":"2024-04-11 02:00:00.234321"},"op":"r"}} diff --git a/scripts/source/test_data/kafka_source_format_bytes.1 b/scripts/source/test_data/kafka_source_format_bytes.1 deleted file mode 100644 index 9477d452c0ca..000000000000 --- a/scripts/source/test_data/kafka_source_format_bytes.1 +++ /dev/null @@ -1,2 +0,0 @@ -kk -weawefwef \ No newline at end of file diff --git a/scripts/source/test_data/weiling.1 b/scripts/source/test_data/weiling.1 deleted file mode 100644 index b0255bb72068..000000000000 --- a/scripts/source/test_data/weiling.1 +++ /dev/null @@ -1 +0,0 @@ -{"v1": 1664267577140960} \ No newline at end of file From 69750ed8279e4964c255db63c20c44838b33e56f Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 11 Oct 2024 21:00:31 +0800 Subject: [PATCH 2/2] update Signed-off-by: xxchan --- ci/scripts/e2e-sink-test.sh | 4 ++-- e2e_test/source_inline/kafka/avro/name_strategy.slt | 6 +++--- e2e_test/source_inline/kafka/handling_mode.slt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh index 5e4685b2fc67..546ec1f1188b 100755 --- a/ci/scripts/e2e-sink-test.sh +++ b/ci/scripts/e2e-sink-test.sh @@ -124,8 +124,8 @@ risedev ci-kill echo "--- e2e, ci-1cn-1fe, nexmark endless" RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ risedev ci-start ci-1cn-1fe -sqllogictest -p 4566 -d dev './e2e_test/source/nexmark_endless_mvs/*.slt' -sqllogictest -p 4566 -d dev './e2e_test/source/nexmark_endless_sinks/*.slt' +sqllogictest -p 4566 -d dev './e2e_test/source_inline/nexmark_endless_mvs/*.slt' +sqllogictest -p 4566 -d dev './e2e_test/source_inline/nexmark_endless_sinks/*.slt' echo "--- Kill cluster" risedev ci-kill diff --git a/e2e_test/source_inline/kafka/avro/name_strategy.slt b/e2e_test/source_inline/kafka/avro/name_strategy.slt index 212c79e6f31a..7dc89660616d 100644 --- a/e2e_test/source_inline/kafka/avro/name_strategy.slt +++ b/e2e_test/source_inline/kafka/avro/name_strategy.slt @@ -25,7 +25,7 @@ create source s1 () with ( # Currently we are abusing this test case to also test data types. system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro" +python3 e2e_test/source_legacy/kafka/script/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro" statement ok CREATE TABLE t_topic ( primary key (rw_key) ) @@ -44,7 +44,7 @@ FORMAT UPSERT ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}'); ## topic: upsert_avro_json-record, key subject: string, value subject: CPLM.OBJ_ATTRIBUTE_VALUE system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro" +python3 e2e_test/source_legacy/kafka/script/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro" statement error key\.message @@ -80,7 +80,7 @@ create table t_record_format_plain () with ( ## key subject: upsert_avro_json-topic-record-string ## value subject: upsert_avro_json-topic-record-CPLM.OBJ_ATTRIBUTE_VALUE system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro" +python3 e2e_test/source_legacy/kafka/script/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro" diff --git a/e2e_test/source_inline/kafka/handling_mode.slt b/e2e_test/source_inline/kafka/handling_mode.slt index 6399118c9b03..40259e130dcb 100644 --- a/e2e_test/source_inline/kafka/handling_mode.slt +++ b/e2e_test/source_inline/kafka/handling_mode.slt @@ -8,7 +8,7 @@ rpk topic create json_timestamptz_handling_mode -p 1 system ok cat <