Merge branch 'main' into 12-23-feat_stream_add_kafka_backill_executor

risingwavelabs · Mar 11, 2024 · fe7ca09 · fe7ca09
2 parents 30bdda3 + d34d7af
commit fe7ca09
Show file tree

Hide file tree

Showing 302 changed files with 6,794 additions and 3,062 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ci/scripts/backwards-compat-test.sh b/ci/scripts/backwards-compat-test.sh
@@ -37,6 +37,8 @@ source backwards-compat-tests/scripts/utils.sh
 ################################### Main
 
 configure_rw() {
+VERSION="$1"
+
 echo "--- Setting up cluster config"
 cat <<EOF > risedev-profiles.user.yml
 full-without-monitoring:
@@ -64,6 +66,10 @@ ENABLE_BUILD_RUST=false
 # Use target/debug for simplicity.
 ENABLE_RELEASE_PROFILE=false
 EOF
+
+if version_le "${VERSION:-}" "1.7.0" ; then
+  echo "ENABLE_ALL_IN_ONE=true" >> risedev-components.user.env
+fi
 }
 
 configure_rw_build() {
@@ -116,7 +122,7 @@ setup_old_cluster() {
 
     echo "--- Start cluster on tag $OLD_VERSION"
     git config --global --add safe.directory /risingwave
-    configure_rw
+    configure_rw $OLD_VERSION
   fi
 }
 
@@ -138,7 +144,10 @@ main() {
   seed_old_cluster "$OLD_VERSION"
 
   setup_new_cluster
-  configure_rw
+  # Assume we use the latest version, so we just set to some large number.
+  # The current $NEW_VERSION as of this change is 1.7.0, so we can't use that.
+  # See: https://github.com/risingwavelabs/risingwave/pull/15448
+  configure_rw "99.99.99"
   validate_new_cluster "$NEW_VERSION"
 }
 

diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -99,4 +99,17 @@ function filter_stack_trace() {
   | sed -E '/  at ...cargo/d' > tmp
   cp tmp "$1"
   rm tmp
-}
+}
+
+get_latest_kafka_version() {
+    local versions=$(curl -s https://downloads.apache.org/kafka/ | grep -Eo 'href="[0-9]+\.[0-9]+\.[0-9]+/"' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
+    # Sort the version numbers and get the latest one
+    local latest_version=$(echo "$versions" | sort -V | tail -n1)
+    echo $latest_version
+}
+
+get_latest_kafka_download_url() {
+    local latest_version=$(get_latest_kafka_version)
+    local download_url="https://downloads.apache.org/kafka/${latest_version}/kafka_2.13-${latest_version}.tgz"
+    echo $download_url
+}
diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh
@@ -49,7 +49,7 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)"
-timeout 7m seq 64 | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+timeout 10m seq 64 | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, e2e extended mode test"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -e 2> $LOGDIR/extended-{}.log && rm $LOGDIR/extended-{}.log'
diff --git a/ci/scripts/gen-flamegraph.sh b/ci/scripts/gen-flamegraph.sh
@@ -97,8 +97,8 @@ install_all() {
   promql --version
 
   echo ">>> Installing Kafka"
-  wget https://downloads.apache.org/kafka/3.4.1/kafka_2.13-3.4.1.tgz
-  tar -zxvf kafka_2.13-3.4.1.tgz
+  wget $(get_latest_kafka_download_url) -O kafka_latest.tgz
+  tar -zxvf kafka_latest.tgz
 
   echo ">>> Installing nexmark bench"
   buildkite-agent artifact download nexmark-server /usr/local/bin
@@ -424,4 +424,4 @@ main() {
   #  popd
 }
 
-main "$@"
+main "$@"
diff --git a/ci/scripts/gen-integration-test-yaml.py b/ci/scripts/gen-integration-test-yaml.py
@@ -16,6 +16,7 @@
     'postgres-sink': ['json'],
     'iceberg-cdc': ['json'],
     'iceberg-sink': ['none'],
+    'iceberg-source': ['none'],
     'twitter': ['json', 'protobuf'],
     'twitter-pulsar': ['json'],
     'debezium-mysql': ['json'],
@@ -33,7 +34,7 @@
     'big-query-sink': ['json'],
     'mindsdb': ['json'],
     'vector': ['json'],
-    'nats': ['json'],
+    'nats': ['json', 'protobuf'],
     'doris-sink': ['json'],
     'starrocks-sink': ['json'],
     'deltalake-sink': ['json'],

diff --git a/ci/scripts/notify.py b/ci/scripts/notify.py
@@ -87,6 +87,8 @@
     "deltalake-sink-json": ["xinhao"],
     "pinot-sink-json": ["yiming"],
     "client-library-none": ["tao"],
+    "nats-json": ["tao"],
+    "nats-protobuf": ["tao"],
 }
 
 def get_failed_tests(get_test_status, test_map):

diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh
@@ -88,7 +88,9 @@ echo "--- e2e, $mode, batch"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 cluster_start
 sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profile}"
-sqllogictest -p 4566 -d dev './e2e_test/background_ddl/basic.slt' --junit "batch-ddl-${profile}"
+if [[ "$mode" != "single-node" ]]; then
+  sqllogictest -p 4566 -d dev './e2e_test/background_ddl/basic.slt' --junit "batch-ddl-${profile}"
+fi
 sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}"
 sqllogictest -p 4566 -d dev './e2e_test/ttl/ttl.slt'
 sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt'

diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
@@ -87,7 +87,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 60
+    timeout_in_minutes: 65
     retry: *auto-retry
 
   - label: "end-to-end test (parallel) (release)"
@@ -409,7 +409,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
+    timeout_in_minutes: 7
     retry: *auto-retry
 
   - label: "end-to-end iceberg sink v2 test (release)"
@@ -428,7 +428,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
+    timeout_in_minutes: 11
     retry: *auto-retry
 
   - label: "e2e java-binding test (release)"
@@ -540,6 +540,28 @@ steps:
     timeout_in_minutes: 25
     retry: *auto-retry
 
+  - label: "S3_v2 source batch read on AWS (json parser)"
+    key: "s3-v2-source-batch-read-check-aws-json-parser"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s 'fs_source_batch.py json'"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-s3-source-tests"
+      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
   - label: "S3_v2 source check on AWS (csv parser)"
     key: "s3-v2-source-check-aws-csv-parser"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s 'fs_source_v2.py csv_without_header'"
@@ -754,7 +776,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 20
+    timeout_in_minutes: 22
     retry: *auto-retry
 
   - label: "e2e standalone binary test"