diff --git a/.bazelrc b/.bazelrc
index 58137d2eec548..736bc18131a91 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -94,11 +94,12 @@ build:debug --@rules_rust//:extra_rustc_flag="-Csplit-debuginfo=unpacked"
# TODO(parkmycar): Enable this for macOS. `toolchains_llvm` defaults to ld64 which
# doesn't support zlib compression.
-build:linux --linkopt="-Wl,--compress-debug-sections=zstd"
-build:linux --@rules_rust//:extra_rustc_flag="-Clink-arg=-Wl,--compress-debug-sections=zstd"
-build:linux --linkopt="-Wl,-O3"
+build:linux --linkopt="-Wl,--compress-debug-sections=zlib"
+build:linux --@rules_rust//:extra_rustc_flag="-Clink-arg=-Wl,--compress-debug-sections=zlib"
+# Specifying "-O2" uses level 6 zlib compression.
+build:linux --linkopt="-Wl,-O2"
build:linux --@rules_rust//:extra_rustc_flag="-Clink-arg=-Wl,-O2"
-build:linux --copt="-gz=zstd"
+build:linux --copt="-gz=zlib"
# Match the DWARF version used by Rust
#
@@ -128,8 +129,7 @@ build --@rules_rust//:extra_rustc_flag="-Csymbol-mangling-version=v0"
build --@rules_rust//:extra_rustc_flag="-Ccodegen-units=64"
# Enabling pipelined builds allows dependent libraries to begin compiling with
# just `.rmeta` instead of the full `.rlib`.
-# TODO: Reenable when fixed, currently still sometimes fails, see for example: https://buildkite.com/materialize/test/builds/93445#0192d426-ff4e-4658-b48a-9473843ae116
-#build --@rules_rust//rust/settings:pipelined_compilation=True
+build --@rules_rust//rust/settings:pipelined_compilation=True
# `cargo check` like config, still experimental!
#
@@ -143,6 +143,8 @@ build:check --output_groups=build_metadata
#
# `/dev/shm` is a RAM backed temporary filesystem, it should speedup sandbox creation.
build:ci --sandbox_base=/dev/shm
+# Always enable verbose failures in CI, makes it easier to debug failures.
+build:ci --verbose_failures
# Release Build Configuration
#
diff --git a/.github/workflows/slack_notify_labeled.yml b/.github/workflows/slack_notify_labeled.yml
index b3c81dd8a138e..b1efb09fc92cc 100644
--- a/.github/workflows/slack_notify_labeled.yml
+++ b/.github/workflows/slack_notify_labeled.yml
@@ -22,7 +22,7 @@
name: Slack Label Notifications
on:
- issues:
+ pull_request:
types:
- labeled
@@ -30,30 +30,12 @@ jobs:
notify:
runs-on: ubuntu-latest
steps:
- - uses: actions-ecosystem/action-slack-notifier@fc778468d09c43a6f4d1b8cccaca59766656996a
- if: ${{ github.event.label.name == 'P1' }}
- with:
- slack_token: ${{ secrets.SLACK_TOKEN }}
- channel: github-p1
- message: |
- `${{ github.event.label.name }}` label has been added to "${{ github.event.issue.title }}" (${{ github.event.issue.html_url }}) (assigned to: ${{ github.event.issue.assignee.login || 'unassigned' }}).
- color: red
- verbose: false
- - uses: actions-ecosystem/action-slack-notifier@fc778468d09c43a6f4d1b8cccaca59766656996a
- if: ${{ github.event.label.name == 'C-bug' }}
- with:
- slack_token: ${{ secrets.SLACK_TOKEN }}
- channel: github-bugs
- message: |
- `${{ github.event.label.name }}` label has been added to "${{ github.event.issue.title }}" (${{ github.event.issue.html_url }}) (assigned to: ${{ github.event.issue.assignee.login || 'unassigned' }}).
- color: red
- verbose: false
- uses: actions-ecosystem/action-slack-notifier@fc778468d09c43a6f4d1b8cccaca59766656996a
if: ${{ github.event.label.name == 'release-blocker' }}
with:
slack_token: ${{ secrets.SLACK_TOKEN }}
channel: release
message: |
- `${{ github.event.label.name }}` label has been added to "${{ github.event.issue.title }}" (${{ github.event.issue.html_url }}) (assigned to: ${{ github.event.issue.assignee.login || 'unassigned' }}).
+ `${{ github.event.label.name }}` label has been added to "${{ github.event.pull_request.title }}" (${{ github.event.pull_request.html_url }}) (assigned to: ${{ github.event.pull_request.assignee.login || 'unassigned' }}).
color: red
verbose: false
diff --git a/.github/workflows/slack_notify_mz_catalog_server.yml b/.github/workflows/slack_notify_mz_catalog_server.yml
deleted file mode 100644
index 0607f78cc82f5..0000000000000
--- a/.github/workflows/slack_notify_mz_catalog_server.yml
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2020 The Actions Ecosystem Authors
-# Modifications Copyright Materialize, Inc. and contributors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Portions of this file are derived from the README examples in the Action
-# Slack Notifier project. The original source code was retrieved on
-# January 5, 2022 from:
-#
-# https://github.com/actions-ecosystem/action-slack-notifier/blob/fc778468d09c43a6f4d1b8cccaca59766656996a/README.md
-
-# Send a notification to the #rnd-mz-catalog-server-council Slack channel when a change
-# is made that adds or modifies an index on the mz_catalog_server cluster, or
-# adds or modifies an object on which those indices depend.
-# Also detects changes to catalog object retention.
-#
-# A notification is sent when all of these conditions are true:
-# * A ready-to-review PR is (re-)opened, or a PR is moved from draft
-# to ready-to-review.
-# * The PR modifies the file 'test/sqllogictest/mz_catalog_server_index_accounting.slt',
-# which is designed to detect new or modified mz_catalog_server indexes, OR
-# * The PR changes the "is_retained_metrics..." field of any catalog object,
-# as detected in the file 'src/catalog/src/builtin.rs'.
-
-name: Slack Mz Catalog Server Council Notifications
-
-on:
- pull_request_target:
- types:
- - opened
- - reopened
- - ready_for_review
- paths:
- - src/catalog/src/builtin.rs
- - test/sqllogictest/mz_catalog_server_index_accounting.slt
-
-jobs:
- notify:
- name: "Notify on mz_catalog_server index changes"
- runs-on: ubuntu-latest
- if: ${{ !github.event.pull_request.draft }}
- steps:
- - name: "Path filter"
- id: filter
- uses: dorny/paths-filter@v2
- with:
- filters: |
- builtin-rs:
- - 'src/catalog/src/builtin.rs'
- index-slt:
- - 'test/sqllogictest/mz_catalog_server_index_accounting.slt'
- - name: Checkout
- uses: actions/checkout@v4
- - name: "Check Retained Metric Changes"
- id: check-retain-metrics
- if: steps.filter.outputs.builtin-rs == 'true'
- run: |
- # Check for the text "is_retained_metrics" modified in builtin.rs in the pull request
- if git diff ${{ github.event.pull_request.base.sha }} -- 'src/catalog/src/builtin.rs' | grep -i 'is_retained_metrics'; then
- echo "changed=true" >> $GITHUB_OUTPUT
- fi
- - name: "Push to Slack"
- if: steps.filter.outputs.index-slt == 'true' || steps.check-retain-metrics.outputs.changed == 'true'
- uses: actions-ecosystem/action-slack-notifier@fc778468d09c43a6f4d1b8cccaca59766656996a
- with:
- slack_token: ${{ secrets.SLACK_TOKEN }}
- channel: rnd-mz-catalog-server-council
- custom_payload: |
- {
- "blocks": [
- {
- "type": "section",
- "text": {
- "type": "mrkdwn",
- "text": "A new builtin index change is ready for review!"
- }
- },
- {
- "type": "section",
- "text": {
- "type": "mrkdwn",
- "text": ${{ toJSON(format('• *PR:* <{0}|{1}>', github.event.pull_request.html_url, github.event.pull_request.title)) }}
- }
- },
- {
- "type": "section",
- "text": {
- "type": "mrkdwn",
- "text": "• *Author:* <${{ github.event.pull_request.user.html_url }}|${{ github.event.pull_request.user.login }}>"
- }
- }
- ]
- }
diff --git a/.github/workflows/slack_notify_sql_parser.yml b/.github/workflows/slack_notify_sql_parser.yml
index e051420c819fc..08e30d4e9e30e 100644
--- a/.github/workflows/slack_notify_sql_parser.yml
+++ b/.github/workflows/slack_notify_sql_parser.yml
@@ -39,6 +39,7 @@ on:
- "src/sql-parser/**"
- "src/sql-lexer/**"
- "src/catalog/src/builtin.rs"
+ - test/sqllogictest/mz_catalog_server_index_accounting.slt
jobs:
notify:
@@ -60,8 +61,20 @@ jobs:
- 'src/catalog/src/builtin.rs'
- '!**/Cargo.toml'
- '!**/BUILD.bazel'
+ index-slt:
+ - 'test/sqllogictest/mz_catalog_server_index_accounting.slt'
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: "Check Retained Metric Changes"
+ id: check-retain-metrics
+ if: steps.filter.outputs.builtin-rs == 'true'
+ run: |
+ # Check for the text "is_retained_metrics" modified in builtin.rs in the pull request
+ if git diff ${{ github.event.pull_request.base.sha }} -- 'src/catalog/src/builtin.rs' | grep -i 'is_retained_metrics'; then
+ echo "changed=true" >> $GITHUB_OUTPUT
+ fi
- name: "Push to Slack"
- if: steps.filter.outputs.sql-parser == 'true' || steps.filter.outputs.system-catalog == 'true'
+ if: steps.filter.outputs.sql-parser == 'true' || steps.filter.outputs.system-catalog == 'true' || steps.filter.outputs.index-slt == 'true'
uses: actions-ecosystem/action-slack-notifier@fc778468d09c43a6f4d1b8cccaca59766656996a
with:
slack_token: ${{ secrets.SLACK_TOKEN }}
diff --git a/Cargo.lock b/Cargo.lock
index 64b662c22657a..cd5757ce37c06 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4340,7 +4340,7 @@ dependencies = [
[[package]]
name = "mz-balancerd"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"async-trait",
@@ -4484,7 +4484,7 @@ dependencies = [
[[package]]
name = "mz-catalog-debug"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"clap",
@@ -4640,7 +4640,7 @@ dependencies = [
[[package]]
name = "mz-clusterd"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"axum",
@@ -4920,7 +4920,7 @@ dependencies = [
[[package]]
name = "mz-environmentd"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"askama",
@@ -5377,7 +5377,7 @@ dependencies = [
[[package]]
name = "mz-materialized"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"mz-clusterd",
"mz-environmentd",
@@ -5569,7 +5569,7 @@ dependencies = [
[[package]]
name = "mz-orchestratord"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"async-trait",
@@ -5733,7 +5733,7 @@ dependencies = [
[[package]]
name = "mz-persist-client"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"arrayvec 0.7.4",
@@ -6818,7 +6818,7 @@ dependencies = [
[[package]]
name = "mz-testdrive"
-version = "0.126.0-dev.0"
+version = "0.127.0-dev.0"
dependencies = [
"anyhow",
"arrow",
@@ -11089,6 +11089,8 @@ dependencies = [
"globset",
"hashbrown 0.14.5",
"hyper 0.14.27",
+ "hyper 1.4.1",
+ "hyper-util",
"indexmap 1.9.1",
"insta",
"k8s-openapi",
diff --git a/LICENSE b/LICENSE
index 7fd5683bed33c..660056dc3942f 100644
--- a/LICENSE
+++ b/LICENSE
@@ -13,7 +13,7 @@ Business Source License 1.1
Licensor: Materialize, Inc.
-Licensed Work: Materialize Version 20241126
+Licensed Work: Materialize Version 20241204
The Licensed Work is © 2024 Materialize, Inc.
Additional Use Grant: Within a single installation of Materialize, you
@@ -32,7 +32,7 @@ Additional Use Grant: Within a single installation of Materialize, you
whose definitions are controlled by such third
parties.
-Change Date: November 26, 2028
+Change Date: December 04, 2028
Change License: Apache License, Version 2.0
diff --git a/bin/ci-annotate-errors b/bin/ci-annotate-errors
index 10bf5473a8b83..b15f1ebda6a98 100755
--- a/bin/ci-annotate-errors
+++ b/bin/ci-annotate-errors
@@ -25,6 +25,7 @@ else
# keep only the remaining first line because there is a further line containing the mzcompose result summary
HOSTNAME=$(echo "$OUTPUT" | head -n 1)
else
+ # TODO: Remove when database-issues#8592 is fixed
echo "Failed to get cloud hostname, using fallback value"
HOSTNAME="7vifiksqeftxc6ld3r6zvc8n2.lb.us-east-1.aws.materialize.cloud"
fi
diff --git a/bin/ci-builder b/bin/ci-builder
index e2aa654ee5ab1..75410e60d0792 100755
--- a/bin/ci-builder
+++ b/bin/ci-builder
@@ -103,7 +103,7 @@ fi
# *inside* this image. See materialize.git.expand_globs in the Python code for
# details on this computation.
files=$(cat \
- <(git diff --name-only -z 4b825dc642cb6eb9a060e54bf8d69288fbee4904 ci/builder .bazelversion) \
+ <(git diff --name-only -z 4b825dc642cb6eb9a060e54bf8d69288fbee4904 ci/builder .bazelversion WORKSPACE) \
<(git ls-files --others --exclude-standard -z ci/builder) \
| LC_ALL=C sort -z \
| xargs -0 "$shasum")
diff --git a/bin/gen-completion b/bin/gen-completion
index b0a260dd1296d..b98699fb98244 100755
--- a/bin/gen-completion
+++ b/bin/gen-completion
@@ -50,7 +50,7 @@ case "$cmd" in
for name in "${python_autocompletions[@]}"; do
for shell in "${supported_shells[@]}"; do
if [ ! -f "$directory/../misc/completions/$shell/_$name" ]; then
- printf "missing shell completion scripts. try running \`bin/completion generate\` and checking in the changes\n"
+ printf "missing shell completion scripts. try running \`bin/gen-completion generate\` and checking in the changes\n"
exit 1
fi
@@ -58,7 +58,7 @@ case "$cmd" in
existing=$("$shasum" < "$directory"/../misc/completions/"$shell"/_"$name")
if [[ "$latest" != "$existing" ]]; then
- printf "shell completion scripts have uncommitted changes. try running \`bin/completion generate\` and checking in the changes\n"
+ printf "shell completion scripts have uncommitted changes. try running \`bin/gen-completion generate\` and checking in the changes\n"
exit 1
fi
done
diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile
index f7926057d30b7..9d172eba6b0fa 100644
--- a/ci/builder/Dockerfile
+++ b/ci/builder/Dockerfile
@@ -203,10 +203,10 @@ RUN mkdir rust \
&& cargo install --root /usr/local --version ="0.9.28" --locked cargo-hakari \
&& cargo install --root /usr/local --version "=0.9.72" --locked cargo-nextest \
&& cargo install --root /usr/local --version "=0.6.11" --locked cargo-llvm-cov \
- && cargo install --root /usr/local --version "=0.1.50" --features=vendored-openssl cargo-udeps \
- && cargo install --root /usr/local --version "=0.2.15" --no-default-features --features=s3,openssl/vendored sccache \
- && cargo install --root /usr/local --version "=0.3.6" cargo-binutils \
- && cargo install --root /usr/local --version "=0.13.0" wasm-pack
+ && cargo install --root /usr/local --version "=0.1.50" --locked --features=vendored-openssl cargo-udeps \
+ && cargo install --root /usr/local --version "=0.2.15" --locked --no-default-features --features=s3,openssl/vendored sccache \
+ && cargo install --root /usr/local --version "=0.3.6" --locked cargo-binutils \
+ && cargo install --root /usr/local --version "=0.13.0" --locked wasm-pack
# Link the system lld into the cross-compiling sysroot.
@@ -273,7 +273,7 @@ RUN arch_bazel=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm64/" -e "s/amd64/x86_64
&& if [ "$arch_bazel" = x86_64 ]; then echo '48ea0ff9d397a48add6369c261c5a4431fe6d5d5348cfb81411782fb80c388d3 /usr/local/bin/bazel' | sha256sum --check; fi \
&& chmod +x /usr/local/bin/bazel
-# Install KinD, kubectl, helm & helm-docs
+# Install KinD, kubectl, helm, helm-docs & terraform
RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.14.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
&& chmod +x /usr/local/bin/kind \
@@ -305,6 +305,13 @@ RUN mkdir -p /usr/local/share/helm/plugins/unittest \
&& tar -xf helm-unittest.tar.gz -C /usr/local/share/helm/plugins/unittest \
&& rm helm-unittest.tar.gz
+RUN curl -fsSL https://releases.hashicorp.com/terraform/1.9.8/terraform_1.9.8_linux_$ARCH_GO.zip > terraform.zip \
+ && if [ $ARCH_GO = amd64 ]; then echo '186e0145f5e5f2eb97cbd785bc78f21bae4ef15119349f6ad4fa535b83b10df8 terraform.zip' | sha256sum --check; fi \
+ && if [ $ARCH_GO = arm64 ]; then echo 'f85868798834558239f6148834884008f2722548f84034c9b0f62934b2d73ebb terraform.zip' | sha256sum --check; fi \
+ && unzip terraform.zip terraform -d /usr/local/bin \
+ && chmod +x /usr/local/bin/terraform \
+ && rm terraform.zip
+
# Hardcode some known SSH hosts, or else SSH will ask whether the host is
# trustworthy on the first connection.
diff --git a/ci/builder/requirements.txt b/ci/builder/requirements.txt
index 3b409f05a70b8..aa2d83960c6c4 100644
--- a/ci/builder/requirements.txt
+++ b/ci/builder/requirements.txt
@@ -19,7 +19,7 @@ ipywidgets==8.1.5
junit-xml==1.9
junitparser==3.2.0
jupyterlab==4.3.0
-jupyter-black==0.3.4
+jupyter-black==0.4.0
kubernetes==25.3.0
kubernetes-stubs==22.6.0.post1
launchdarkly-api==17.0.0
@@ -29,8 +29,8 @@ networkx==3.4.2
networkx-stubs==0.0.1
numpy==1.26.4
pandas==2.2.3
-pandas-stubs==2.2.3.241009
-parameterized==0.8.1
+pandas-stubs==2.2.3.241126
+parameterized==0.9.0
paramiko==3.5.0
pdoc==15.0.0
# We can revert back to standard pg8000 versions once https://github.com/tlocke/pg8000/pull/161 is released
@@ -40,7 +40,7 @@ psutil==6.1.0
psycopg==3.2.3
psycopg-binary==3.2.3
pydantic==2.8.2
-pyelftools==0.29
+pyelftools==0.31
pyjwt==2.9.0
PyMySQL==1.1.1
pytest==8.3.3
@@ -51,7 +51,7 @@ ruamel.yaml==0.18.6
ruff==0.0.292
scipy==1.14.1
semver==3.0.0
-shtab==1.5.8
+shtab==1.7.1
sqlparse==0.5.0
toml==0.10.2
twine==5.1.1
@@ -71,5 +71,5 @@ fastavro==1.9.4
websocket-client==1.8.0
pyarrow-stubs==17.12
pyarrow==18.0.0
-minio==7.2.10
+minio==7.2.12
zstandard==0.23.0
diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py
index 5c8f6cd0c7dd2..d0cd70cb7214b 100644
--- a/ci/mkpipeline.py
+++ b/ci/mkpipeline.py
@@ -386,12 +386,29 @@ def switch_jobs_to_aws(pipeline: Any, priority: int) -> None:
def visit(config: Any) -> None:
if "agents" in config:
agent = config["agents"].get("queue", None)
- if agent == "hetzner-aarch64-4cpu-8gb":
+ if agent in ("hetzner-aarch64-4cpu-8gb", "hetzner-aarch64-2cpu-4gb"):
config["agents"]["queue"] = "linux-aarch64-small"
if agent == "hetzner-aarch64-8cpu-16gb":
config["agents"]["queue"] = "linux-aarch64"
if agent == "hetzner-aarch64-16cpu-32gb":
config["agents"]["queue"] = "linux-aarch64-medium"
+ if agent in (
+ "hetzner-x86-64-4cpu-8gb",
+ "hetzner-x86-64-2cpu-4gb",
+ "hetzner-x86-64-dedi-2cpu-8gb",
+ ):
+ config["agents"]["queue"] = "linux-x86-64-small"
+ if agent in ("hetzner-x86-64-8cpu-16gb", "hetzner-x86-64-dedi-4cpu-16gb"):
+ config["agents"]["queue"] = "linux-x86-64"
+ if agent in ("hetzner-x86-64-16cpu-32gb", "hetzner-x86-64-dedi-8cpu-32gb"):
+ config["agents"]["queue"] = "linux-x86-64-medium"
+ if agent == "hetzner-x86-64-dedi-16cpu-64gb":
+ config["agents"]["queue"] = "linux-x86-64-large"
+ if agent in (
+ "hetzner-x86-64-dedi-32cpu-128gb",
+ "hetzner-x86-64-dedi-48cpu-192gb",
+ ):
+ config["agents"]["queue"] = "builder-linux-x86_64"
for config in pipeline["steps"]:
if "trigger" in config or "wait" in config:
diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml
index aceac595f4423..d8bebe9313d24 100644
--- a/ci/nightly/pipeline.template.yml
+++ b/ci/nightly/pipeline.template.yml
@@ -73,8 +73,7 @@ steps:
composition: cargo-test
args: [--miri-full]
agents:
- # TODO(def-) Switch back to Hetzner
- queue: builder-linux-aarch64-mem
+ queue: hetzner-aarch64-16cpu-32gb
sanitizer: skip
- group: Benchmarks
@@ -82,11 +81,11 @@ steps:
steps:
- id: feature-benchmark
label: "Feature benchmark against merge base or 'latest' %N"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 720
parallelism: 8
agents:
- queue: linux-aarch64-medium
+ queue: hetzner-x86-64-dedi-8cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: feature-benchmark
@@ -96,10 +95,11 @@ steps:
- common-ancestor
- id: scalability-benchmark-dml-dql
label: "Scalability benchmark (read & write) against merge base or 'latest'"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 240
agents:
- queue: linux-aarch64
+ # Larger instance is more stable in performance
+ queue: hetzner-x86-64-dedi-8cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: scalability
@@ -116,10 +116,11 @@ steps:
- "256"
- id: scalability-benchmark-ddl
label: "Scalability benchmark (DDL) against merge base or 'latest'"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 1200
agents:
- queue: linux-aarch64
+ # Larger instance is more stable in performance
+ queue: hetzner-x86-64-dedi-16cpu-64gb
plugins:
- ./ci/plugins/mzcompose:
composition: scalability
@@ -140,10 +141,11 @@ steps:
- "64"
- id: scalability-benchmark-connection
label: "Scalability benchmark (connection) against merge base or 'latest'"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 180
agents:
- queue: linux-aarch64-medium
+ # Larger instance is more stable in performance
+ queue: hetzner-x86-64-dedi-16cpu-64gb
plugins:
- ./ci/plugins/mzcompose:
composition: scalability
@@ -162,10 +164,11 @@ steps:
- "2048"
- id: parallel-benchmark
label: "Parallel Benchmark"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 120
agents:
- queue: linux-aarch64-medium
+ # Larger instance is more stable in performance
+ queue: hetzner-x86-64-dedi-16cpu-64gb
plugins:
- ./ci/plugins/mzcompose:
composition: parallel-benchmark
@@ -540,7 +543,8 @@ steps:
plugins:
- ./ci/plugins/mzcompose:
composition: zippy
- args: [--scenario=KafkaSources, --actions=10000, --cockroach-tag=latest, --max-execution-time=30m]
+ # TODO: Reenable --cockroach-tag=latest when https://github.com/cockroachdb/cockroach/issues/136678 is fixed
+ args: [--scenario=KafkaSources, --actions=10000, --max-execution-time=30m]
- id: zippy-alter-connection
label: "Zippy w/ alter connection"
@@ -821,61 +825,49 @@ steps:
composition: platform-checks
args: [--scenario=UpgradeEntireMzFourVersions, "--seed=$BUILDKITE_JOB_ID"]
- - id: checks-0dt-restart-entire-mz
- label: "Checks 0dt restart of the entire Mz"
- depends_on: build-aarch64
- timeout_in_minutes: 240
- agents:
- # TODO(def-): Switch back to hetzner
- queue: linux-aarch64-large
- plugins:
- - ./ci/plugins/mzcompose:
- composition: platform-checks
- args: [--scenario=ZeroDowntimeRestartEntireMz, "--seed=$BUILDKITE_JOB_ID"]
-
- id: checks-0dt-restart-entire-mz-forced-migrations
- label: "Checks 0dt restart of the entire Mz with forced migrations"
+ label: "Checks 0dt restart of the entire Mz with forced migrations %N"
depends_on: build-aarch64
- timeout_in_minutes: 240
+ timeout_in_minutes: 60
+ parallelism: 2
agents:
- # TODO(def-): Switch back to hetzner
- queue: linux-aarch64-large
+ queue: hetzner-aarch64-16cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: platform-checks
args: [--scenario=ZeroDowntimeRestartEntireMzForcedMigrations, "--seed=$BUILDKITE_JOB_ID"]
- id: checks-0dt-upgrade-entire-mz
- label: "Checks 0dt upgrade, whole-Mz restart"
+ label: "Checks 0dt upgrade, whole-Mz restart %N"
depends_on: build-aarch64
- timeout_in_minutes: 240
+ timeout_in_minutes: 60
+ parallelism: 2
agents:
- # TODO(def-): Switch back to hetzner
- queue: linux-aarch64-large
+ queue: hetzner-aarch64-16cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: platform-checks
args: [--scenario=ZeroDowntimeUpgradeEntireMz, "--seed=$BUILDKITE_JOB_ID"]
- id: checks-0dt-upgrade-entire-mz-two-versions
- label: "Checks 0dt upgrade across two versions"
+ label: "Checks 0dt upgrade across two versions %N"
depends_on: build-aarch64
- timeout_in_minutes: 240
+ timeout_in_minutes: 60
+ parallelism: 2
agents:
- # TODO(def-): Switch back to hetzner
- queue: linux-aarch64-large
+ queue: hetzner-aarch64-16cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: platform-checks
args: [--scenario=ZeroDowntimeUpgradeEntireMzTwoVersions, "--seed=$BUILDKITE_JOB_ID"]
- id: checks-0dt-upgrade-entire-mz-four-versions
- label: "Checks 0dt upgrade across four versions"
+ label: "Checks 0dt upgrade across four versions %N"
depends_on: build-aarch64
- timeout_in_minutes: 240
+ timeout_in_minutes: 60
+ parallelism: 2
agents:
- # TODO(def-): Switch back to hetzner
- queue: linux-aarch64-large
+ queue: hetzner-aarch64-16cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: platform-checks
@@ -883,11 +875,10 @@ steps:
- id: checks-0dt-bump-version
label: "Checks 0dt upgrade to a bumped version"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 240
agents:
- # TODO(def-): Switch back to hetzner
- queue: builder-linux-aarch64-mem
+ queue: hetzner-x86-64-dedi-32cpu-128gb
plugins:
- ./ci/plugins/mzcompose:
composition: platform-checks
@@ -1102,31 +1093,49 @@ steps:
- ./ci/plugins/mzcompose:
composition: launchdarkly
- - id: cloud-canary
- label: "Canary Deploy in Staging Cloud"
- depends_on: build-aarch64
- timeout_in_minutes: 1200
- concurrency: 1
- concurrency_group: 'cloud-canary'
- agents:
- queue: linux-aarch64-small
- plugins:
- - ./ci/plugins/mzcompose:
- composition: cloud-canary
- branches: "main v*.*"
+ - group: E2E
+ key: e2e
+ steps:
+ - id: cloud-canary
+ label: "Canary Deploy in Staging Cloud"
+ depends_on: build-aarch64
+ timeout_in_minutes: 1200
+ concurrency: 1
+ concurrency_group: 'cloud-canary'
+ agents:
+ queue: linux-aarch64-small
+ plugins:
+ - ./ci/plugins/mzcompose:
+ composition: cloud-canary
+ branches: "main v*.*"
- - id: mz-e2e
- label: "Mz E2E Test"
- depends_on: build-aarch64
- timeout_in_minutes: 1200
- concurrency: 1
- concurrency_group: 'mz-e2e'
- agents:
- # Requires real Mz access, CONFLUENT_CLOUD_DEVEX_KAFKA_USERNAME, etc.
- queue: linux-aarch64-small
- plugins:
- - ./ci/plugins/mzcompose:
- composition: mz-e2e
+ - id: mz-e2e
+ label: "Mz E2E Test"
+ depends_on: build-aarch64
+ timeout_in_minutes: 1200
+ concurrency: 1
+ concurrency_group: 'mz-e2e'
+ agents:
+ # Requires real Mz access, CONFLUENT_CLOUD_DEVEX_KAFKA_USERNAME, etc.
+ queue: linux-aarch64-small
+ plugins:
+ - ./ci/plugins/mzcompose:
+ composition: mz-e2e
+
+ - id: terraform-aws
+ label: "Terraform + Helm Chart E2E on AWS"
+ artifact_paths: [test/terraform/aws/terraform.tfstate]
+ depends_on: build-aarch64
+ timeout_in_minutes: 1200
+ concurrency: 1
+ concurrency_group: 'terraform'
+ agents:
+ queue: linux-aarch64-small
+ plugins:
+ - ./ci/plugins/scratch-aws-access: ~
+ - ./ci/plugins/mzcompose:
+ composition: terraform
+ branches: "main v*.*"
- group: "Output consistency"
key: output-consistency
@@ -1180,7 +1189,7 @@ steps:
depends_on: build-aarch64
timeout_in_minutes: 58
agents:
- queue: linux-aarch64-small
+ queue: hetzner-aarch64-4cpu-8gb
plugins:
- ./ci/plugins/mzcompose:
composition: feature-flag-consistency
@@ -1314,6 +1323,7 @@ steps:
- id: rqg-window-functions
label: "RQG window functions workload"
depends_on: build-aarch64
+ skip: "flaky until database-issues#8366 is fixed"
timeout_in_minutes: 45
agents:
queue: hetzner-aarch64-4cpu-8gb
@@ -1517,12 +1527,11 @@ steps:
- id: parallel-workload-0dt
label: "Parallel Workload (0dt deploy)"
- depends_on: build-aarch64
+ depends_on: build-x86_64
artifact_paths: [parallel-workload-queries.log.zst]
timeout_in_minutes: 90
agents:
- # TODO(def-): Switch back to hetzner
- queue: linux-aarch64-large
+ queue: hetzner-x86-64-dedi-16cpu-64gb
plugins:
- ./ci/plugins/mzcompose:
composition: parallel-workload
diff --git a/ci/plugins/cloudtest/hooks/command b/ci/plugins/cloudtest/hooks/command
index 6216f5a2c2c3f..b8405dda6311d 100644
--- a/ci/plugins/cloudtest/hooks/command
+++ b/ci/plugins/cloudtest/hooks/command
@@ -42,6 +42,7 @@ ci_collapsed_heading ":docker: Purging all existing docker containers and volume
sudo systemctl restart docker
docker ps --all --quiet | xargs --no-run-if-empty docker rm --force --volumes
killall -9 clusterd || true # There might be remaining processes from a previous cargo-test run
+rm -rf ~/.kube # Remove potential state from E2E Terraform tests
ci_collapsed_heading "kind: Increase system limits..."
sudo sysctl fs.inotify.max_user_watches=524288
diff --git a/ci/plugins/mzcompose/hooks/pre-exit b/ci/plugins/mzcompose/hooks/pre-exit
index 8ba86668a1ec5..5153e85197d18 100755
--- a/ci/plugins/mzcompose/hooks/pre-exit
+++ b/ci/plugins/mzcompose/hooks/pre-exit
@@ -85,7 +85,7 @@ timeout 300 buildkite-agent artifact upload "$artifacts_str" || true
bin/ci-builder run stable bin/ci-annotate-errors --test-cmd="$(cat test_cmd)" --test-desc="$(cat test_desc)" "${artifacts[@]}" > ci-annotate-errors.log || CI_ANNOTATE_ERRORS_RESULT=$?
buildkite-agent artifact upload "ci-annotate-errors.log"
-if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ]; then
+if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ]; then
echo "+++ services.log is empty, failing"
exit 1
fi
@@ -139,10 +139,6 @@ fi
ci_unimportant_heading ":docker: Cleaning up after mzcompose"
-run() {
- bin/ci-builder run stable bin/mzcompose --mz-quiet --find "$BUILDKITE_PLUGIN_MZCOMPOSE_COMPOSITION" "$@"
-}
-
# docker-compose kill may fail attempting to kill containers
# that have just exited on their own because of the
# "shared-fate" mechanism employed by Mz clusters
diff --git a/ci/plugins/scratch-aws-access/README.md b/ci/plugins/scratch-aws-access/README.md
index b654508212e38..8e72e51bc07ed 100644
--- a/ci/plugins/scratch-aws-access/README.md
+++ b/ci/plugins/scratch-aws-access/README.md
@@ -7,7 +7,7 @@ scratch AWS account. The following environment variables are set:
* `AWS_SECRET_ACCESS_KEY`
* `AWS_SESSION_TOKEN`
-The credentials are valid for one hour.
+The credentials are valid for 12 hours.
## Example
diff --git a/ci/release-qualification/pipeline.template.yml b/ci/release-qualification/pipeline.template.yml
index f003862a33e91..0701846c5dc18 100644
--- a/ci/release-qualification/pipeline.template.yml
+++ b/ci/release-qualification/pipeline.template.yml
@@ -166,11 +166,11 @@ steps:
steps:
- id: feature-benchmark-scale-plus-one
label: "Feature benchmark against 'common-ancestor' with --scale=+1 %N"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 2880
parallelism: 8
agents:
- queue: linux-aarch64-medium
+ queue: hetzner-x86-64-dedi-16cpu-64gb
plugins:
- ./ci/plugins/mzcompose:
composition: feature-benchmark
@@ -178,10 +178,10 @@ steps:
- id: long-parallel-benchmark
label: "Long Parallel Benchmark"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 1200
agents:
- queue: linux-aarch64-medium
+ queue: hetzner-x86-64-dedi-8cpu-32gb
plugins:
- ./ci/plugins/mzcompose:
composition: parallel-benchmark
@@ -438,10 +438,9 @@ steps:
args: [--scenario=KillClusterdStorage, "--seed=$BUILDKITE_JOB_ID"]
- id: checks-restart-source-postgres
- label: "Checks + restart source Postgres %N"
+ label: "Checks + restart source Postgres"
depends_on: build-aarch64
timeout_in_minutes: 180
- parallelism: 2
agents:
queue: hetzner-aarch64-16cpu-32gb
plugins:
@@ -474,6 +473,18 @@ steps:
composition: platform-checks
args: [--scenario=DropCreateDefaultReplica, "--seed=$BUILDKITE_JOB_ID"]
+ - id: checks-0dt-restart-entire-mz
+ label: "Checks 0dt restart of the entire Mz %N"
+ depends_on: build-aarch64
+ timeout_in_minutes: 60
+ parallelism: 2
+ agents:
+ queue: hetzner-aarch64-16cpu-32gb
+ plugins:
+ - ./ci/plugins/mzcompose:
+ composition: platform-checks
+ args: [--scenario=ZeroDowntimeRestartEntireMz, "--seed=$BUILDKITE_JOB_ID"]
+
- id: limits
label: "Product limits (finding new limits) %N"
depends_on: build-aarch64
diff --git a/ci/test/docs-widgets/package-lock.json b/ci/test/docs-widgets/package-lock.json
index bbbd8411d7c81..a6e64ec126234 100644
--- a/ci/test/docs-widgets/package-lock.json
+++ b/ci/test/docs-widgets/package-lock.json
@@ -11,9 +11,9 @@
}
},
"node_modules/@esbuild/aix-ppc64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz",
- "integrity": "sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
+ "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==",
"cpu": [
"ppc64"
],
@@ -26,9 +26,9 @@
}
},
"node_modules/@esbuild/android-arm": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.2.tgz",
- "integrity": "sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz",
+ "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==",
"cpu": [
"arm"
],
@@ -41,9 +41,9 @@
}
},
"node_modules/@esbuild/android-arm64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz",
- "integrity": "sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz",
+ "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==",
"cpu": [
"arm64"
],
@@ -56,9 +56,9 @@
}
},
"node_modules/@esbuild/android-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.2.tgz",
- "integrity": "sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz",
+ "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==",
"cpu": [
"x64"
],
@@ -71,9 +71,9 @@
}
},
"node_modules/@esbuild/darwin-arm64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz",
- "integrity": "sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz",
+ "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==",
"cpu": [
"arm64"
],
@@ -86,9 +86,9 @@
}
},
"node_modules/@esbuild/darwin-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz",
- "integrity": "sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz",
+ "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==",
"cpu": [
"x64"
],
@@ -101,9 +101,9 @@
}
},
"node_modules/@esbuild/freebsd-arm64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz",
- "integrity": "sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz",
+ "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==",
"cpu": [
"arm64"
],
@@ -116,9 +116,9 @@
}
},
"node_modules/@esbuild/freebsd-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz",
- "integrity": "sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz",
+ "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==",
"cpu": [
"x64"
],
@@ -131,9 +131,9 @@
}
},
"node_modules/@esbuild/linux-arm": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz",
- "integrity": "sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz",
+ "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==",
"cpu": [
"arm"
],
@@ -146,9 +146,9 @@
}
},
"node_modules/@esbuild/linux-arm64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz",
- "integrity": "sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz",
+ "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==",
"cpu": [
"arm64"
],
@@ -161,9 +161,9 @@
}
},
"node_modules/@esbuild/linux-ia32": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz",
- "integrity": "sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz",
+ "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==",
"cpu": [
"ia32"
],
@@ -176,9 +176,9 @@
}
},
"node_modules/@esbuild/linux-loong64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz",
- "integrity": "sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz",
+ "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==",
"cpu": [
"loong64"
],
@@ -191,9 +191,9 @@
}
},
"node_modules/@esbuild/linux-mips64el": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz",
- "integrity": "sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz",
+ "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==",
"cpu": [
"mips64el"
],
@@ -206,9 +206,9 @@
}
},
"node_modules/@esbuild/linux-ppc64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz",
- "integrity": "sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz",
+ "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==",
"cpu": [
"ppc64"
],
@@ -221,9 +221,9 @@
}
},
"node_modules/@esbuild/linux-riscv64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz",
- "integrity": "sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz",
+ "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==",
"cpu": [
"riscv64"
],
@@ -236,9 +236,9 @@
}
},
"node_modules/@esbuild/linux-s390x": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz",
- "integrity": "sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz",
+ "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==",
"cpu": [
"s390x"
],
@@ -251,9 +251,9 @@
}
},
"node_modules/@esbuild/linux-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz",
- "integrity": "sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz",
+ "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==",
"cpu": [
"x64"
],
@@ -266,9 +266,9 @@
}
},
"node_modules/@esbuild/netbsd-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz",
- "integrity": "sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz",
+ "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==",
"cpu": [
"x64"
],
@@ -281,9 +281,9 @@
}
},
"node_modules/@esbuild/openbsd-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz",
- "integrity": "sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz",
+ "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==",
"cpu": [
"x64"
],
@@ -296,9 +296,9 @@
}
},
"node_modules/@esbuild/sunos-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz",
- "integrity": "sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz",
+ "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==",
"cpu": [
"x64"
],
@@ -311,9 +311,9 @@
}
},
"node_modules/@esbuild/win32-arm64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz",
- "integrity": "sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz",
+ "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==",
"cpu": [
"arm64"
],
@@ -326,9 +326,9 @@
}
},
"node_modules/@esbuild/win32-ia32": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz",
- "integrity": "sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz",
+ "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==",
"cpu": [
"ia32"
],
@@ -341,9 +341,9 @@
}
},
"node_modules/@esbuild/win32-x64": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz",
- "integrity": "sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz",
+ "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==",
"cpu": [
"x64"
],
@@ -372,9 +372,9 @@
"integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
},
"node_modules/@rollup/rollup-android-arm-eabi": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.13.2.tgz",
- "integrity": "sha512-3XFIDKWMFZrMnao1mJhnOT1h2g0169Os848NhhmGweEcfJ4rCi+3yMCOLG4zA61rbJdkcrM/DjVZm9Hg5p5w7g==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.26.0.tgz",
+ "integrity": "sha512-gJNwtPDGEaOEgejbaseY6xMFu+CPltsc8/T+diUTTbOQLqD+bnrJq9ulH6WD69TqwqWmrfRAtUv30cCFZlbGTQ==",
"cpu": [
"arm"
],
@@ -384,9 +384,9 @@
]
},
"node_modules/@rollup/rollup-android-arm64": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.13.2.tgz",
- "integrity": "sha512-GdxxXbAuM7Y/YQM9/TwwP+L0omeE/lJAR1J+olu36c3LqqZEBdsIWeQ91KBe6nxwOnb06Xh7JS2U5ooWU5/LgQ==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.26.0.tgz",
+ "integrity": "sha512-YJa5Gy8mEZgz5JquFruhJODMq3lTHWLm1fOy+HIANquLzfIOzE9RA5ie3JjCdVb9r46qfAQY/l947V0zfGJ0OQ==",
"cpu": [
"arm64"
],
@@ -396,9 +396,9 @@
]
},
"node_modules/@rollup/rollup-darwin-arm64": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.13.2.tgz",
- "integrity": "sha512-mCMlpzlBgOTdaFs83I4XRr8wNPveJiJX1RLfv4hggyIVhfB5mJfN4P8Z6yKh+oE4Luz+qq1P3kVdWrCKcMYrrA==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.26.0.tgz",
+ "integrity": "sha512-ErTASs8YKbqTBoPLp/kA1B1Um5YSom8QAc4rKhg7b9tyyVqDBlQxy7Bf2wW7yIlPGPg2UODDQcbkTlruPzDosw==",
"cpu": [
"arm64"
],
@@ -408,9 +408,9 @@
]
},
"node_modules/@rollup/rollup-darwin-x64": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.13.2.tgz",
- "integrity": "sha512-yUoEvnH0FBef/NbB1u6d3HNGyruAKnN74LrPAfDQL3O32e3k3OSfLrPgSJmgb3PJrBZWfPyt6m4ZhAFa2nZp2A==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.26.0.tgz",
+ "integrity": "sha512-wbgkYDHcdWW+NqP2mnf2NOuEbOLzDblalrOWcPyY6+BRbVhliavon15UploG7PpBRQ2bZJnbmh8o3yLoBvDIHA==",
"cpu": [
"x64"
],
@@ -419,10 +419,46 @@
"darwin"
]
},
+ "node_modules/@rollup/rollup-freebsd-arm64": {
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.26.0.tgz",
+ "integrity": "sha512-Y9vpjfp9CDkAG4q/uwuhZk96LP11fBz/bYdyg9oaHYhtGZp7NrbkQrj/66DYMMP2Yo/QPAsVHkV891KyO52fhg==",
+ "cpu": [
+ "arm64"
+ ],
+ "optional": true,
+ "os": [
+ "freebsd"
+ ]
+ },
+ "node_modules/@rollup/rollup-freebsd-x64": {
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.26.0.tgz",
+ "integrity": "sha512-A/jvfCZ55EYPsqeaAt/yDAG4q5tt1ZboWMHEvKAH9Zl92DWvMIbnZe/f/eOXze65aJaaKbL+YeM0Hz4kLQvdwg==",
+ "cpu": [
+ "x64"
+ ],
+ "optional": true,
+ "os": [
+ "freebsd"
+ ]
+ },
"node_modules/@rollup/rollup-linux-arm-gnueabihf": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.13.2.tgz",
- "integrity": "sha512-GYbLs5ErswU/Xs7aGXqzc3RrdEjKdmoCrgzhJWyFL0r5fL3qd1NPcDKDowDnmcoSiGJeU68/Vy+OMUluRxPiLQ==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.26.0.tgz",
+ "integrity": "sha512-paHF1bMXKDuizaMODm2bBTjRiHxESWiIyIdMugKeLnjuS1TCS54MF5+Y5Dx8Ui/1RBPVRE09i5OUlaLnv8OGnA==",
+ "cpu": [
+ "arm"
+ ],
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@rollup/rollup-linux-arm-musleabihf": {
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.26.0.tgz",
+ "integrity": "sha512-cwxiHZU1GAs+TMxvgPfUDtVZjdBdTsQwVnNlzRXC5QzIJ6nhfB4I1ahKoe9yPmoaA/Vhf7m9dB1chGPpDRdGXg==",
"cpu": [
"arm"
],
@@ -432,9 +468,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm64-gnu": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.13.2.tgz",
- "integrity": "sha512-L1+D8/wqGnKQIlh4Zre9i4R4b4noxzH5DDciyahX4oOz62CphY7WDWqJoQ66zNR4oScLNOqQJfNSIAe/6TPUmQ==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.26.0.tgz",
+ "integrity": "sha512-4daeEUQutGRCW/9zEo8JtdAgtJ1q2g5oHaoQaZbMSKaIWKDQwQ3Yx0/3jJNmpzrsScIPtx/V+1AfibLisb3AMQ==",
"cpu": [
"arm64"
],
@@ -444,9 +480,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm64-musl": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.13.2.tgz",
- "integrity": "sha512-tK5eoKFkXdz6vjfkSTCupUzCo40xueTOiOO6PeEIadlNBkadH1wNOH8ILCPIl8by/Gmb5AGAeQOFeLev7iZDOA==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.26.0.tgz",
+ "integrity": "sha512-eGkX7zzkNxvvS05ROzJ/cO/AKqNvR/7t1jA3VZDi2vRniLKwAWxUr85fH3NsvtxU5vnUUKFHKh8flIBdlo2b3Q==",
"cpu": [
"arm64"
],
@@ -456,11 +492,11 @@
]
},
"node_modules/@rollup/rollup-linux-powerpc64le-gnu": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.13.2.tgz",
- "integrity": "sha512-zvXvAUGGEYi6tYhcDmb9wlOckVbuD+7z3mzInCSTACJ4DQrdSLPNUeDIcAQW39M3q6PDquqLWu7pnO39uSMRzQ==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.26.0.tgz",
+ "integrity": "sha512-Odp/lgHbW/mAqw/pU21goo5ruWsytP7/HCC/liOt0zcGG0llYWKrd10k9Fj0pdj3prQ63N5yQLCLiE7HTX+MYw==",
"cpu": [
- "ppc64le"
+ "ppc64"
],
"optional": true,
"os": [
@@ -468,9 +504,9 @@
]
},
"node_modules/@rollup/rollup-linux-riscv64-gnu": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.13.2.tgz",
- "integrity": "sha512-C3GSKvMtdudHCN5HdmAMSRYR2kkhgdOfye4w0xzyii7lebVr4riCgmM6lRiSCnJn2w1Xz7ZZzHKuLrjx5620kw==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.26.0.tgz",
+ "integrity": "sha512-MBR2ZhCTzUgVD0OJdTzNeF4+zsVogIR1U/FsyuFerwcqjZGvg2nYe24SAHp8O5sN8ZkRVbHwlYeHqcSQ8tcYew==",
"cpu": [
"riscv64"
],
@@ -480,9 +516,9 @@
]
},
"node_modules/@rollup/rollup-linux-s390x-gnu": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.13.2.tgz",
- "integrity": "sha512-l4U0KDFwzD36j7HdfJ5/TveEQ1fUTjFFQP5qIt9gBqBgu1G8/kCaq5Ok05kd5TG9F8Lltf3MoYsUMw3rNlJ0Yg==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.26.0.tgz",
+ "integrity": "sha512-YYcg8MkbN17fMbRMZuxwmxWqsmQufh3ZJFxFGoHjrE7bv0X+T6l3glcdzd7IKLiwhT+PZOJCblpnNlz1/C3kGQ==",
"cpu": [
"s390x"
],
@@ -492,9 +528,9 @@
]
},
"node_modules/@rollup/rollup-linux-x64-gnu": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.13.2.tgz",
- "integrity": "sha512-xXMLUAMzrtsvh3cZ448vbXqlUa7ZL8z0MwHp63K2IIID2+DeP5iWIT6g1SN7hg1VxPzqx0xZdiDM9l4n9LRU1A==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.26.0.tgz",
+ "integrity": "sha512-ZuwpfjCwjPkAOxpjAEjabg6LRSfL7cAJb6gSQGZYjGhadlzKKywDkCUnJ+KEfrNY1jH5EEoSIKLCb572jSiglA==",
"cpu": [
"x64"
],
@@ -504,9 +540,9 @@
]
},
"node_modules/@rollup/rollup-linux-x64-musl": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.13.2.tgz",
- "integrity": "sha512-M/JYAWickafUijWPai4ehrjzVPKRCyDb1SLuO+ZyPfoXgeCEAlgPkNXewFZx0zcnoIe3ay4UjXIMdXQXOZXWqA==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.26.0.tgz",
+ "integrity": "sha512-+HJD2lFS86qkeF8kNu0kALtifMpPCZU80HvwztIKnYwym3KnA1os6nsX4BGSTLtS2QVAGG1P3guRgsYyMA0Yhg==",
"cpu": [
"x64"
],
@@ -516,9 +552,9 @@
]
},
"node_modules/@rollup/rollup-win32-arm64-msvc": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.13.2.tgz",
- "integrity": "sha512-2YWwoVg9KRkIKaXSh0mz3NmfurpmYoBBTAXA9qt7VXk0Xy12PoOP40EFuau+ajgALbbhi4uTj3tSG3tVseCjuA==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.26.0.tgz",
+ "integrity": "sha512-WUQzVFWPSw2uJzX4j6YEbMAiLbs0BUysgysh8s817doAYhR5ybqTI1wtKARQKo6cGop3pHnrUJPFCsXdoFaimQ==",
"cpu": [
"arm64"
],
@@ -528,9 +564,9 @@
]
},
"node_modules/@rollup/rollup-win32-ia32-msvc": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.13.2.tgz",
- "integrity": "sha512-2FSsE9aQ6OWD20E498NYKEQLneShWes0NGMPQwxWOdws35qQXH+FplabOSP5zEe1pVjurSDOGEVCE2agFwSEsw==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.26.0.tgz",
+ "integrity": "sha512-D4CxkazFKBfN1akAIY6ieyOqzoOoBV1OICxgUblWxff/pSjCA2khXlASUx7mK6W1oP4McqhgcCsu6QaLj3WMWg==",
"cpu": [
"ia32"
],
@@ -540,9 +576,9 @@
]
},
"node_modules/@rollup/rollup-win32-x64-msvc": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.13.2.tgz",
- "integrity": "sha512-7h7J2nokcdPePdKykd8wtc8QqqkqxIrUz7MHj6aNr8waBRU//NLDVnNjQnqQO6fqtjrtCdftpbTuOKAyrAQETQ==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.26.0.tgz",
+ "integrity": "sha512-2x8MO1rm4PGEP0xWbubJW5RtbNLk3puzAMaLQd3B3JHVw4KcHlmXcO+Wewx9zCoo7EUFiMlu/aZbCJ7VjMzAag==",
"cpu": [
"x64"
],
@@ -557,9 +593,9 @@
"integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="
},
"node_modules/@types/estree": {
- "version": "1.0.5",
- "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz",
- "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw=="
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
+ "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw=="
},
"node_modules/@vitest/expect": {
"version": "1.4.0",
@@ -822,9 +858,9 @@
}
},
"node_modules/esbuild": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz",
- "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==",
+ "version": "0.21.5",
+ "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz",
+ "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==",
"hasInstallScript": true,
"bin": {
"esbuild": "bin/esbuild"
@@ -833,29 +869,29 @@
"node": ">=12"
},
"optionalDependencies": {
- "@esbuild/aix-ppc64": "0.20.2",
- "@esbuild/android-arm": "0.20.2",
- "@esbuild/android-arm64": "0.20.2",
- "@esbuild/android-x64": "0.20.2",
- "@esbuild/darwin-arm64": "0.20.2",
- "@esbuild/darwin-x64": "0.20.2",
- "@esbuild/freebsd-arm64": "0.20.2",
- "@esbuild/freebsd-x64": "0.20.2",
- "@esbuild/linux-arm": "0.20.2",
- "@esbuild/linux-arm64": "0.20.2",
- "@esbuild/linux-ia32": "0.20.2",
- "@esbuild/linux-loong64": "0.20.2",
- "@esbuild/linux-mips64el": "0.20.2",
- "@esbuild/linux-ppc64": "0.20.2",
- "@esbuild/linux-riscv64": "0.20.2",
- "@esbuild/linux-s390x": "0.20.2",
- "@esbuild/linux-x64": "0.20.2",
- "@esbuild/netbsd-x64": "0.20.2",
- "@esbuild/openbsd-x64": "0.20.2",
- "@esbuild/sunos-x64": "0.20.2",
- "@esbuild/win32-arm64": "0.20.2",
- "@esbuild/win32-ia32": "0.20.2",
- "@esbuild/win32-x64": "0.20.2"
+ "@esbuild/aix-ppc64": "0.21.5",
+ "@esbuild/android-arm": "0.21.5",
+ "@esbuild/android-arm64": "0.21.5",
+ "@esbuild/android-x64": "0.21.5",
+ "@esbuild/darwin-arm64": "0.21.5",
+ "@esbuild/darwin-x64": "0.21.5",
+ "@esbuild/freebsd-arm64": "0.21.5",
+ "@esbuild/freebsd-x64": "0.21.5",
+ "@esbuild/linux-arm": "0.21.5",
+ "@esbuild/linux-arm64": "0.21.5",
+ "@esbuild/linux-ia32": "0.21.5",
+ "@esbuild/linux-loong64": "0.21.5",
+ "@esbuild/linux-mips64el": "0.21.5",
+ "@esbuild/linux-ppc64": "0.21.5",
+ "@esbuild/linux-riscv64": "0.21.5",
+ "@esbuild/linux-s390x": "0.21.5",
+ "@esbuild/linux-x64": "0.21.5",
+ "@esbuild/netbsd-x64": "0.21.5",
+ "@esbuild/openbsd-x64": "0.21.5",
+ "@esbuild/sunos-x64": "0.21.5",
+ "@esbuild/win32-arm64": "0.21.5",
+ "@esbuild/win32-ia32": "0.21.5",
+ "@esbuild/win32-x64": "0.21.5"
}
},
"node_modules/estree-walker": {
@@ -1250,9 +1286,9 @@
}
},
"node_modules/picocolors": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
- "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ=="
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
+ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="
},
"node_modules/pkg-types": {
"version": "1.0.3",
@@ -1265,9 +1301,9 @@
}
},
"node_modules/postcss": {
- "version": "8.4.38",
- "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz",
- "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==",
+ "version": "8.4.49",
+ "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz",
+ "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==",
"funding": [
{
"type": "opencollective",
@@ -1284,8 +1320,8 @@
],
"dependencies": {
"nanoid": "^3.3.7",
- "picocolors": "^1.0.0",
- "source-map-js": "^1.2.0"
+ "picocolors": "^1.1.1",
+ "source-map-js": "^1.2.1"
},
"engines": {
"node": "^10 || ^12 || >=14"
@@ -1333,11 +1369,11 @@
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
},
"node_modules/rollup": {
- "version": "4.13.2",
- "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.13.2.tgz",
- "integrity": "sha512-MIlLgsdMprDBXC+4hsPgzWUasLO9CE4zOkj/u6j+Z6j5A4zRY+CtiXAdJyPtgCsc42g658Aeh1DlrdVEJhsL2g==",
+ "version": "4.26.0",
+ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.26.0.tgz",
+ "integrity": "sha512-ilcl12hnWonG8f+NxU6BlgysVA0gvY2l8N0R84S1HcINbW20bvwuCngJkkInV6LXhwRpucsW5k1ovDwEdBVrNg==",
"dependencies": {
- "@types/estree": "1.0.5"
+ "@types/estree": "1.0.6"
},
"bin": {
"rollup": "dist/bin/rollup"
@@ -1347,21 +1383,24 @@
"npm": ">=8.0.0"
},
"optionalDependencies": {
- "@rollup/rollup-android-arm-eabi": "4.13.2",
- "@rollup/rollup-android-arm64": "4.13.2",
- "@rollup/rollup-darwin-arm64": "4.13.2",
- "@rollup/rollup-darwin-x64": "4.13.2",
- "@rollup/rollup-linux-arm-gnueabihf": "4.13.2",
- "@rollup/rollup-linux-arm64-gnu": "4.13.2",
- "@rollup/rollup-linux-arm64-musl": "4.13.2",
- "@rollup/rollup-linux-powerpc64le-gnu": "4.13.2",
- "@rollup/rollup-linux-riscv64-gnu": "4.13.2",
- "@rollup/rollup-linux-s390x-gnu": "4.13.2",
- "@rollup/rollup-linux-x64-gnu": "4.13.2",
- "@rollup/rollup-linux-x64-musl": "4.13.2",
- "@rollup/rollup-win32-arm64-msvc": "4.13.2",
- "@rollup/rollup-win32-ia32-msvc": "4.13.2",
- "@rollup/rollup-win32-x64-msvc": "4.13.2",
+ "@rollup/rollup-android-arm-eabi": "4.26.0",
+ "@rollup/rollup-android-arm64": "4.26.0",
+ "@rollup/rollup-darwin-arm64": "4.26.0",
+ "@rollup/rollup-darwin-x64": "4.26.0",
+ "@rollup/rollup-freebsd-arm64": "4.26.0",
+ "@rollup/rollup-freebsd-x64": "4.26.0",
+ "@rollup/rollup-linux-arm-gnueabihf": "4.26.0",
+ "@rollup/rollup-linux-arm-musleabihf": "4.26.0",
+ "@rollup/rollup-linux-arm64-gnu": "4.26.0",
+ "@rollup/rollup-linux-arm64-musl": "4.26.0",
+ "@rollup/rollup-linux-powerpc64le-gnu": "4.26.0",
+ "@rollup/rollup-linux-riscv64-gnu": "4.26.0",
+ "@rollup/rollup-linux-s390x-gnu": "4.26.0",
+ "@rollup/rollup-linux-x64-gnu": "4.26.0",
+ "@rollup/rollup-linux-x64-musl": "4.26.0",
+ "@rollup/rollup-win32-arm64-msvc": "4.26.0",
+ "@rollup/rollup-win32-ia32-msvc": "4.26.0",
+ "@rollup/rollup-win32-x64-msvc": "4.26.0",
"fsevents": "~2.3.2"
}
},
@@ -1422,9 +1461,9 @@
}
},
"node_modules/source-map-js": {
- "version": "1.2.0",
- "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz",
- "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==",
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
+ "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
"engines": {
"node": ">=0.10.0"
}
@@ -1543,13 +1582,13 @@
}
},
"node_modules/vite": {
- "version": "5.2.7",
- "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.7.tgz",
- "integrity": "sha512-k14PWOKLI6pMaSzAuGtT+Cf0YmIx12z9YGon39onaJNy8DLBfBJrzg9FQEmkAM5lpHBZs9wksWAsyF/HkpEwJA==",
+ "version": "5.4.11",
+ "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz",
+ "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==",
"dependencies": {
- "esbuild": "^0.20.1",
- "postcss": "^8.4.38",
- "rollup": "^4.13.0"
+ "esbuild": "^0.21.3",
+ "postcss": "^8.4.43",
+ "rollup": "^4.20.0"
},
"bin": {
"vite": "bin/vite.js"
@@ -1568,6 +1607,7 @@
"less": "*",
"lightningcss": "^1.21.0",
"sass": "*",
+ "sass-embedded": "*",
"stylus": "*",
"sugarss": "*",
"terser": "^5.4.0"
@@ -1585,6 +1625,9 @@
"sass": {
"optional": true
},
+ "sass-embedded": {
+ "optional": true
+ },
"stylus": {
"optional": true
},
diff --git a/ci/test/lint-main/checks/check-new-line.sh b/ci/test/lint-main/checks/check-new-line.sh
index 20801af950855..2c5a6b90d6236 100755
--- a/ci/test/lint-main/checks/check-new-line.sh
+++ b/ci/test/lint-main/checks/check-new-line.sh
@@ -22,7 +22,7 @@ files=$(git_files "$@")
# Only binary files are permitted to omit a trailing newline. If you're here to
# exclude a text file that is missing its trailing newline, like an SVG, add
# a trailing newline to the text file instead.
-newline_files=$(grep -vE '\.(png|jpe?g|pb|avro|ico|so)$' <<< "$files")
+newline_files=$(grep -vE '(_scratch|\.(png|jpe?g|pb|avro|ico|so))$' <<< "$files")
try xargs misc/lint/trailing-newline.sh <<< "$newline_files"
try xargs git --no-pager diff --check "$(git_empty_tree)" <<< "$newline_files"
diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml
index bfc9db1845e66..a002ebfe8960c 100644
--- a/ci/test/pipeline.template.yml
+++ b/ci/test/pipeline.template.yml
@@ -170,7 +170,7 @@ steps:
depends_on: []
timeout_in_minutes: 20
agents:
- queue: linux-aarch64-small
+ queue: hetzner-aarch64-4cpu-8gb
coverage: skip
sanitizer: skip
@@ -640,7 +640,7 @@ steps:
- id: feature-benchmark-kafka-only
label: "Feature benchmark (Kafka only)"
- depends_on: build-aarch64
+ depends_on: build-x86_64
timeout_in_minutes: 45
plugins:
- ./ci/plugins/mzcompose:
@@ -654,7 +654,7 @@ steps:
]
coverage: skip
agents:
- queue: linux-aarch64-small
+ queue: hetzner-x86-64-dedi-4cpu-16gb
# Fast tests closer to the end, doesn't matter as much if they have to wait
# for an agent
diff --git a/doc/user/assets/sass/_layout.scss b/doc/user/assets/sass/_layout.scss
index c94d5b86395c1..b337bd0185fdb 100644
--- a/doc/user/assets/sass/_layout.scss
+++ b/doc/user/assets/sass/_layout.scss
@@ -31,7 +31,7 @@
// https://weblog.west-wind.com/posts/2016/Feb/15/Flexbox-Containers-PRE-tags-and-managing-Overflow
min-width: 0;
max-width: 840px;
- padding: var(--xx-small) var(--small);
+ padding: var(--large) var(--small);
@media (max-width: 1260px) {
flex: 2.5;
@@ -154,7 +154,7 @@ table.inline-headings {
margin-left: auto;
position: sticky;
top: var(--nav-height);
- padding: var(--x-small) var(--nano);
+ padding: var(--large) var(--small);
overflow-y: auto;
height: 100vh;
display: flex;
diff --git a/doc/user/config.toml b/doc/user/config.toml
index cda412e9a614c..2a2fc09384741 100644
--- a/doc/user/config.toml
+++ b/doc/user/config.toml
@@ -7,6 +7,8 @@ disableKinds = ['taxonomy']
[params]
repo = "//github.com/MaterializeInc/materialize"
+bannerMessage = "Need to run Materialize in your own private or public cloud? Get early access to **self-managed Materialize**!"
+bannerLink="https://materialize.com/blog/self-managed/?utm_medium=docs-banner&utm_source=documentation&utm_campaign=FY25_Blogs&sf_campaign=701TR00000NoNFXYA3&utm_term=&utm_content="
[frontmatter]
publishDate = ["publishDate"]
diff --git a/doc/user/content/ingest-data/mysql/amazon-aurora.md b/doc/user/content/ingest-data/mysql/amazon-aurora.md
index a0b0064db4a19..eb288d4d7d4fd 100644
--- a/doc/user/content/ingest-data/mysql/amazon-aurora.md
+++ b/doc/user/content/ingest-data/mysql/amazon-aurora.md
@@ -8,8 +8,6 @@ menu:
identifier: "mysql-amazon-aurora"
---
-{{< public-preview />}}
-
This page shows you how to stream data from [Amazon Aurora MySQL](https://aws.amazon.com/rds/aurora/)
to Materialize using the [MySQL source](/sql/create-source/mysql/).
diff --git a/doc/user/content/ingest-data/mysql/amazon-rds.md b/doc/user/content/ingest-data/mysql/amazon-rds.md
index 30874c55f29de..32b40f894f292 100644
--- a/doc/user/content/ingest-data/mysql/amazon-rds.md
+++ b/doc/user/content/ingest-data/mysql/amazon-rds.md
@@ -8,8 +8,6 @@ menu:
identifier: "mysql-amazon-rds"
---
-{{< public-preview />}}
-
This page shows you how to stream data from [Amazon RDS for MySQL](https://aws.amazon.com/rds/mysql/)
to Materialize using the [MySQL source](/sql/create-source/mysql).
diff --git a/doc/user/content/ingest-data/mysql/azure-db.md b/doc/user/content/ingest-data/mysql/azure-db.md
index ed4ca6a2c8956..ee1417500daa9 100644
--- a/doc/user/content/ingest-data/mysql/azure-db.md
+++ b/doc/user/content/ingest-data/mysql/azure-db.md
@@ -8,8 +8,6 @@ menu:
indentifier: "mysql-azure-db"
---
-{{< public-preview />}}
-
This page shows you how to stream data from [Azure DB for MySQL](https://azure.microsoft.com/en-us/products/MySQL)
to Materialize using the [MySQL source](/sql/create-source/mysql/).
diff --git a/doc/user/content/ingest-data/mysql/google-cloud-sql.md b/doc/user/content/ingest-data/mysql/google-cloud-sql.md
index 8fd54244cc67a..131e0e8692b40 100644
--- a/doc/user/content/ingest-data/mysql/google-cloud-sql.md
+++ b/doc/user/content/ingest-data/mysql/google-cloud-sql.md
@@ -8,8 +8,6 @@ menu:
identifier: "mysql-google-cloudsql"
---
-{{< public-preview />}}
-
This page shows you how to stream data from [Google Cloud SQL for MySQL](https://cloud.google.com/sql/MySQL)
to Materialize using the[MySQL source](/sql/create-source/mysql/).
diff --git a/doc/user/content/ingest-data/mysql/self-hosted.md b/doc/user/content/ingest-data/mysql/self-hosted.md
index bd63c401719d1..49bd9d493a5a0 100644
--- a/doc/user/content/ingest-data/mysql/self-hosted.md
+++ b/doc/user/content/ingest-data/mysql/self-hosted.md
@@ -8,8 +8,6 @@ menu:
identifier: "mysql-self-hosted"
---
-{{< public-preview />}}
-
This page shows you how to stream data from a self-hosted MySQL database to
Materialize using the [MySQL source](/sql/create-source/mysql/).
diff --git a/doc/user/content/releases/previews.md b/doc/user/content/releases/previews.md
index fce5b037223ff..4bc88bfcc40cf 100644
--- a/doc/user/content/releases/previews.md
+++ b/doc/user/content/releases/previews.md
@@ -76,12 +76,6 @@ For more information, see:
- [`CREATE CLUSTER`](/sql/create-cluster).
- [`ALTER CLUSTER`](/sql/alter-cluster).
-### COMMENT ON
-
-Materialize adds the `COMMENT ON` command to allow commenting on an object.
-
-For more information, see [`COMMENT ON`](/sql/comment-on).
-
### Support for Fivetran
Materialize adds support for Fivetran to sync data into Materialize.
@@ -186,18 +180,6 @@ your environment.
Overview](/images/health-dashboard.png "Health Dashboard in the Materialize
Console Environment Overview")
-### Native connectors for MySQL
-
-Materialize provides native connectors for MySQL as a source.
-
-For more information, see:
-
-- [Ingest data: MySQL](/ingest-data/mysql/)
-
-- [Create Source: MySQL](/sql/create-source/mysql/)
-
-- [Create Connection: MySQL](/sql/create-connection/#mysql)
-
### AWS Connection for IAM authentication
Materialize supports the use of an AWS Connection to perform:
diff --git a/doc/user/content/releases/v0.126.md b/doc/user/content/releases/v0.126.md
index 67654adabf387..4134e217d5d63 100644
--- a/doc/user/content/releases/v0.126.md
+++ b/doc/user/content/releases/v0.126.md
@@ -2,6 +2,7 @@
title: "Materialize v0.126"
date: 2024-12-04
released: false
+patch: 2
_build:
render: never
---
diff --git a/doc/user/content/releases/v0.127.md b/doc/user/content/releases/v0.127.md
new file mode 100644
index 0000000000000..9a88d6a25d92e
--- /dev/null
+++ b/doc/user/content/releases/v0.127.md
@@ -0,0 +1,7 @@
+---
+title: "Materialize v0.127"
+date: 2024-12-18
+released: false
+_build:
+ render: never
+---
diff --git a/doc/user/content/sql/comment-on.md b/doc/user/content/sql/comment-on.md
index 17c92e78a0099..1f2d2923d176c 100644
--- a/doc/user/content/sql/comment-on.md
+++ b/doc/user/content/sql/comment-on.md
@@ -6,8 +6,6 @@ menu:
parent: 'commands'
---
-{{< public-preview />}}
-
`COMMENT ON ...` adds or updates the comment of an object.
## Syntax
diff --git a/doc/user/content/sql/create-connection.md b/doc/user/content/sql/create-connection.md
index f42705dbef693..b4acc4089deb3 100644
--- a/doc/user/content/sql/create-connection.md
+++ b/doc/user/content/sql/create-connection.md
@@ -625,8 +625,6 @@ CREATE CONNECTION csr_ssh TO CONFLUENT SCHEMA REGISTRY (
### MySQL
-{{< public-preview />}}
-
A MySQL connection establishes a link to a [MySQL] server. You can use
MySQL connections to create [sources](/sql/create-source/mysql).
diff --git a/doc/user/content/sql/create-source/mysql.md b/doc/user/content/sql/create-source/mysql.md
index ca98a60f0e570..bd089132246cd 100644
--- a/doc/user/content/sql/create-source/mysql.md
+++ b/doc/user/content/sql/create-source/mysql.md
@@ -10,8 +10,6 @@ menu:
weight: 20
---
-{{< public-preview />}}
-
{{% create-source/intro %}}
Materialize supports MySQL (5.7+) as a real-time data source. To connect to a
MySQL database, you first need to tweak its configuration to enable
diff --git a/doc/user/content/sql/system-catalog/mz_internal.md b/doc/user/content/sql/system-catalog/mz_internal.md
index 1d309845e539a..8c8bc7e86f3b8 100644
--- a/doc/user/content/sql/system-catalog/mz_internal.md
+++ b/doc/user/content/sql/system-catalog/mz_internal.md
@@ -68,6 +68,7 @@ granted the [`mz_monitor` role](/manage/access-control/manage-roles#builtin-role
| `finished_at` | [`timestamp with time zone`] | The wall-clock time at which the statement finished executing. |
| `finished_status` | [`text`] | The final status of the statement (e.g., `success`, `canceled`, `error`, or `aborted`). `aborted` means that Materialize exited before the statement finished executing. |
| `error_message` | [`text`] | The error message, if the statement failed. |
+| `result_size` | [`bigint`] | The size in bytes of the result, for statements that return rows. |
| `rows_returned` | [`bigint`] | The number of rows returned, for statements that return rows. |
| `execution_strategy` | [`text`] | For `SELECT` queries, the strategy for executing the query. `constant` means computed in the control plane without the involvement of a cluster, `fast-path` means read by a cluster directly from an in-memory index, and `standard` means computed by a temporary dataflow. |
| `transaction_id` | [`uint8`] | The ID of the transaction that the statement was part of. Note that transaction IDs are only unique per session. |
@@ -288,14 +289,14 @@ The `mz_pending_cluster_replicas` table lists the replicas that were created dur
## `mz_comments`
-The `mz_comments` table stores optional comments (descriptions) for objects in the database.
+The `mz_comments` table stores optional comments (i.e., descriptions) for objects in the database.
| Field | Type | Meaning |
| -------------- |-------------| -------- |
| `id` | [`text`] | The ID of the object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). |
| `object_type` | [`text`] | The type of object the comment is associated with. |
-| `object_sub_id`| [`integer`] | For a comment on a column of a relation, this is the column number. For all other object types this column is `NULL`. |
+| `object_sub_id`| [`integer`] | For a comment on a column of a relation, the column number. `NULL` for other object types. |
| `comment` | [`text`] | The comment itself. |
## `mz_compute_dependencies`
@@ -420,7 +421,7 @@ usage. For example:
the the reference documentation for [query optimization](/transform-data/optimization/#indexes)
instead.
- If a view is depended on by multiple objects that use very selective filters,
- or a multiple projections that can be pushed into or even beyond the view,
+ or multiple projections that can be pushed into or even beyond the view,
adding an index may increase resource usage.
- If an index has been created to [enable delta joins](/transform-data/optimization/#optimize-multi-way-joins-with-delta-joins),
removing it may lead to lower memory utilization, but the delta join
@@ -1176,6 +1177,7 @@ and cannot be changed by users), the latter is used instead.
| `finished_at` | [`timestamp with time zone`] | The time at which execution ended. |
| `finished_status` | [`text`] | `'success'`, `'error'`, `'canceled'`, or `'aborted'`. `'aborted'` means that the database restarted (e.g., due to a crash or planned maintenance) before the query finished. |
| `error_message` | [`text`] | The error returned when executing the statement, or `NULL` if it was successful, canceled or aborted. |
+| `result_size` | [`bigint`] | The size in bytes of the result, for statements that return rows. |
| `rows_returned` | [`int8`] | The number of rows returned by the statement, if it finished successfully and was of a kind of statement that can return rows, or `NULL` otherwise. |
| `execution_strategy` | [`text`] | `'standard'`, `'fast-path'` `'constant'`, or `NULL`. `'standard'` means a dataflow was built on a cluster to compute the result. `'fast-path'` means a cluster read the result from an existing arrangement. `'constant'` means the result was computed in the serving layer, without involving a cluster. |
-->
diff --git a/doc/user/content/sql/system-catalog/mz_introspection.md b/doc/user/content/sql/system-catalog/mz_introspection.md
index ade7e4602a985..46836ff521e16 100644
--- a/doc/user/content/sql/system-catalog/mz_introspection.md
+++ b/doc/user/content/sql/system-catalog/mz_introspection.md
@@ -222,6 +222,19 @@ The `mz_dataflow_channel_operators` view associates [dataflow] channels with the
+## `mz_dataflow_global_ids`
+
+The `mz_dataflow_global_ids` view associates [dataflow] ids with global ids (ids of the form `u8` or `t5`).
+
+
+
+| Field | Type | Meaning |
+|------------- | ------- | -------- |
+| `id` | [`uint8`] | The dataflow ID. |
+| `global_id` | [`text`] | A global ID associated with that dataflow. |
+
+
+
## `mz_dataflow_operators`
The `mz_dataflow_operators` view describes the [dataflow] operators in the system.
@@ -292,6 +305,29 @@ through a hierarchical scheme for either aggregation or Top K computations.
| `savings` | [`numeric`] | A conservative estimate of the amount of memory in bytes to be saved by applying the hint. |
| `hint` | [`double precision`] | The hint value that will eliminate `to_cut` levels from the region's hierarchy. |
+## `mz_lir_mapping`
+
+The `mz_lir_mapping` view describes the low-level internal representation (LIR) plan that corresponds to global ids.
+LIR is a higher-level representation than dataflows; this view is used for profiling and debugging indices and materialized views.
+Note that LIR is not a stable interface and may change at any time.
+In particular, you should not attempt to parse `operator` descriptions.
+LIR nodes are implemented by zero or more dataflow operators with sequential ids.
+We use the range `[operator_id_start, operator_id_end)` to record this information.
+If an LIR node was implemented without any dataflow operators, `operator_id_start` will be equal to `operator_id_end`.
+
+
+| Field | Type | Meaning
+| --------- | -------- | -----------
+| global_id | [`text`] | The global ID.
+| lir_id | [`uint8`] | The LIR node ID.
+| operator | [`text`] | The LIR operator, in the format `OperatorName INPUTS [OPTIONS]`.
+| parent_lir_id | [`uint8`] | The parent of this LIR node. May be `NULL`.
+| nesting | [`uint2`] | The nesting level of this LIR node.
+| operator_id_start | [`uint8`] | The first dataflow operator ID implementing this LIR operator (inclusive).
+| operator_id_end | [`uint8`] | The first dataflow operator ID _after_ this LIR operator (exclusive).
+
+
+
## `mz_message_counts`
The `mz_message_counts` view describes the messages and message batches sent and received over the [dataflow] channels in the system.
diff --git a/doc/user/data/sql_funcs.yml b/doc/user/data/sql_funcs.yml
index 28083fa3044cf..321d6f52a1907 100644
--- a/doc/user/data/sql_funcs.yml
+++ b/doc/user/data/sql_funcs.yml
@@ -37,7 +37,7 @@
- signature: 'bool_or(x: T) -> T'
description: _NULL_ if all values of `x` are _NULL_, otherwise true if any values of `x` are true, otherwise false.
- - signature: 'count(x: T) -> int'
+ - signature: 'count(x: T) -> bigint'
description: Number of non-_NULL_ inputs.
- signature: jsonb_agg(expression) -> jsonb
diff --git a/doc/user/layouts/partials/banner.html b/doc/user/layouts/partials/banner.html
index cdd43268a68bd..9ed5997f1ad67 100644
--- a/doc/user/layouts/partials/banner.html
+++ b/doc/user/layouts/partials/banner.html
@@ -4,7 +4,7 @@
{{ if .Site.Params.bannerLink }}
{{ end }}
- {{ .Site.Params.bannerMessage }}
+ {{ .Site.Params.bannerMessage | markdownify }}
{{ if .Site.Params.bannerLink}} {{ end }}
diff --git a/doc/user/layouts/partials/toc.html b/doc/user/layouts/partials/toc.html
index 07ad5a0ea17c9..629f5ef996d09 100644
--- a/doc/user/layouts/partials/toc.html
+++ b/doc/user/layouts/partials/toc.html
@@ -59,7 +59,7 @@ On this page
const headings = $(".content h2, .content h3");
let offsets = headings.toArray().map((h) => ({
id: h.id,
- offset: h.offsetTop - 65, // change to 75 if banner is displayed
+ offset: h.offsetTop - 75, // 75 if banner is displayed, 65 otherwise
}));
const cutoff = $(document).height() - $(window).height() - SLOP;
const firstBad = offsets.findIndex((o) => o.offset > cutoff);
diff --git a/misc/completions/bash/_scratch b/misc/completions/bash/_scratch
index 435f56ab678c4..012b7bdfef9be 100644
--- a/misc/completions/bash/_scratch
+++ b/misc/completions/bash/_scratch
@@ -71,7 +71,7 @@ _shtab_replace_nonword() {
# set default values (called for the initial parser & any subparsers)
_set_parser_defaults() {
local subparsers_var="${prefix}_subparsers[@]"
- sub_parsers=${!subparsers_var}
+ sub_parsers=${!subparsers_var-}
local current_option_strings_var="${prefix}_option_strings[@]"
current_option_strings=${!current_option_strings_var}
@@ -88,19 +88,19 @@ _set_new_action() {
current_action="${prefix}_$(_shtab_replace_nonword $1)"
local current_action_compgen_var=${current_action}_COMPGEN
- current_action_compgen="${!current_action_compgen_var}"
+ current_action_compgen="${!current_action_compgen_var-}"
local current_action_choices_var="${current_action}_choices[@]"
- current_action_choices="${!current_action_choices_var}"
+ current_action_choices="${!current_action_choices_var-}"
local current_action_nargs_var="${current_action}_nargs"
- if [ -n "${!current_action_nargs_var}" ]; then
+ if [ -n "${!current_action_nargs_var-}" ]; then
current_action_nargs="${!current_action_nargs_var}"
else
current_action_nargs=1
fi
- current_action_args_start_index=$(( $word_index + 1 ))
+ current_action_args_start_index=$(( $word_index + 1 - $pos_only ))
current_action_is_positional=$2
}
@@ -114,10 +114,20 @@ _set_new_action() {
# ${!x} -> ${hello} -> "world"
_shtab_scratch() {
local completing_word="${COMP_WORDS[COMP_CWORD]}"
+ local completed_positional_actions
+ local current_action
+ local current_action_args_start_index
+ local current_action_choices
+ local current_action_compgen
+ local current_action_is_positional
+ local current_action_nargs
+ local current_option_strings
+ local sub_parsers
COMPREPLY=()
- prefix=_shtab_scratch
- word_index=0
+ local prefix=_shtab_scratch
+ local word_index=0
+ local pos_only=0 # "--" delimeter not encountered yet
_set_parser_defaults
word_index=1
@@ -126,26 +136,30 @@ _shtab_scratch() {
while [ $word_index -ne $COMP_CWORD ]; do
local this_word="${COMP_WORDS[$word_index]}"
- if [[ -n $sub_parsers && " ${sub_parsers[@]} " =~ " ${this_word} " ]]; then
- # valid subcommand: add it to the prefix & reset the current action
- prefix="${prefix}_$(_shtab_replace_nonword $this_word)"
- _set_parser_defaults
- fi
-
- if [[ " ${current_option_strings[@]} " =~ " ${this_word} " ]]; then
- # a new action should be acquired (due to recognised option string or
- # no more input expected from current action);
- # the next positional action can fill in here
- _set_new_action $this_word false
- fi
-
- if [[ "$current_action_nargs" != "*" ]] && \
- [[ "$current_action_nargs" != "+" ]] && \
- [[ "$current_action_nargs" != *"..." ]] && \
- (( $word_index + 1 - $current_action_args_start_index >= \
- $current_action_nargs )); then
- $current_action_is_positional && let "completed_positional_actions += 1"
- _set_new_action "pos_${completed_positional_actions}" true
+ if [[ $pos_only = 1 || " $this_word " != " -- " ]]; then
+ if [[ -n $sub_parsers && " ${sub_parsers[@]} " == *" ${this_word} "* ]]; then
+ # valid subcommand: add it to the prefix & reset the current action
+ prefix="${prefix}_$(_shtab_replace_nonword $this_word)"
+ _set_parser_defaults
+ fi
+
+ if [[ " ${current_option_strings[@]} " == *" ${this_word} "* ]]; then
+ # a new action should be acquired (due to recognised option string or
+ # no more input expected from current action);
+ # the next positional action can fill in here
+ _set_new_action $this_word false
+ fi
+
+ if [[ "$current_action_nargs" != "*" ]] && \
+ [[ "$current_action_nargs" != "+" ]] && \
+ [[ "$current_action_nargs" != *"..." ]] && \
+ (( $word_index + 1 - $current_action_args_start_index - $pos_only >= \
+ $current_action_nargs )); then
+ $current_action_is_positional && let "completed_positional_actions += 1"
+ _set_new_action "pos_${completed_positional_actions}" true
+ fi
+ else
+ pos_only=1 # "--" delimeter encountered
fi
let "word_index+=1"
@@ -153,7 +167,7 @@ _shtab_scratch() {
# Generate the completions
- if [[ "${completing_word}" == -* ]]; then
+ if [[ $pos_only = 0 && "${completing_word}" == -* ]]; then
# optional argument started: use option strings
COMPREPLY=( $(compgen -W "${current_option_strings[*]}" -- "${completing_word}") )
else
diff --git a/misc/completions/zsh/_scratch b/misc/completions/zsh/_scratch
index c61b1042f0b78..2e113ae62166f 100644
--- a/misc/completions/zsh/_scratch
+++ b/misc/completions/zsh/_scratch
@@ -51,7 +51,7 @@ _shtab_scratch_destroy_options=(
_shtab_scratch_forward_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
- ":The ID of the instance to connect to:"
+ ":The ID of the instance to connect to, or \'mine\' to specify your only live instance:"
"(*)::The remote ports to forward locally:"
)
@@ -69,17 +69,17 @@ _shtab_scratch_mine_options=(
_shtab_scratch_push_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
"--rev[The git rev to checkout]:rev:"
- ":The ID of the instance to connect to:"
+ ":The ID of the instance to connect to, or \'mine\' to specify your only live instance:"
)
_shtab_scratch_sftp_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
- ":The ID of the instance to connect to:"
+ ":The ID of the instance to connect to, or \'mine\' to specify your only live instance:"
)
_shtab_scratch_ssh_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
- ":The ID of the instance to connect to:"
+ ":The ID of the instance to connect to, or \'mine\' to specify your only live instance:"
"(*)::The command to run via SSH, if any:"
)
@@ -90,7 +90,7 @@ _shtab_scratch() {
if ((${_shtab_scratch_options[(I)${(q)one_or_more}*]} + ${_shtab_scratch_options[(I)${(q)remainder}*]} == 0)); then # noqa: E501
_shtab_scratch_options+=(': :_shtab_scratch_commands' '*::: :->scratch')
fi
- _arguments -C $_shtab_scratch_options
+ _arguments -C -s $_shtab_scratch_options
case $state in
scratch)
@@ -98,15 +98,15 @@ _shtab_scratch() {
(( CURRENT += 1 ))
curcontext="${curcontext%:*:*}:_shtab_scratch-$line[1]:"
case $line[1] in
- completion) _arguments -C $_shtab_scratch_completion_options ;;
- create) _arguments -C $_shtab_scratch_create_options ;;
- destroy) _arguments -C $_shtab_scratch_destroy_options ;;
- forward) _arguments -C $_shtab_scratch_forward_options ;;
- login) _arguments -C $_shtab_scratch_login_options ;;
- mine) _arguments -C $_shtab_scratch_mine_options ;;
- push) _arguments -C $_shtab_scratch_push_options ;;
- sftp) _arguments -C $_shtab_scratch_sftp_options ;;
- ssh) _arguments -C $_shtab_scratch_ssh_options ;;
+ completion) _arguments -C -s $_shtab_scratch_completion_options ;;
+ create) _arguments -C -s $_shtab_scratch_create_options ;;
+ destroy) _arguments -C -s $_shtab_scratch_destroy_options ;;
+ forward) _arguments -C -s $_shtab_scratch_forward_options ;;
+ login) _arguments -C -s $_shtab_scratch_login_options ;;
+ mine) _arguments -C -s $_shtab_scratch_mine_options ;;
+ push) _arguments -C -s $_shtab_scratch_push_options ;;
+ sftp) _arguments -C -s $_shtab_scratch_sftp_options ;;
+ ssh) _arguments -C -s $_shtab_scratch_ssh_options ;;
esac
esac
}
@@ -114,4 +114,12 @@ _shtab_scratch() {
typeset -A opt_args
-_shtab_scratch "$@"
+
+if [[ $zsh_eval_context[-1] == eval ]]; then
+ # eval/source/. command, register function for later
+ compdef _shtab_scratch -N scratch
+else
+ # autoload from fpath, call function directly
+ _shtab_scratch "$@"
+fi
+
diff --git a/misc/helm-charts/operator/Chart.yaml b/misc/helm-charts/operator/Chart.yaml
index bbfd5c18970b3..1786b350c83b1 100644
--- a/misc/helm-charts/operator/Chart.yaml
+++ b/misc/helm-charts/operator/Chart.yaml
@@ -1,4 +1,3 @@
----
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
@@ -12,7 +11,7 @@ apiVersion: v2
name: materialize-operator
description: Materialize Kubernetes Operator Helm Chart
type: application
-version: 25.1.0-beta.1
-appVersion: v0.125.2
+version: v25.1.0-beta.1
+appVersion: v0.127.0-dev.0
icon: https://materialize.com/favicon.ico
home: https://materialize.com
diff --git a/misc/helm-charts/operator/README.md b/misc/helm-charts/operator/README.md
index fcfa5077213eb..01e4b903e465b 100644
--- a/misc/helm-charts/operator/README.md
+++ b/misc/helm-charts/operator/README.md
@@ -1,6 +1,6 @@
# Materialize Kubernetes Operator Helm Chart
-  
+  
Materialize Kubernetes Operator Helm Chart
@@ -118,9 +118,10 @@ The following table lists the configurable parameters of the Materialize operato
| `networkPolicies.ingress.cidrs[0]` | | ``"0.0.0.0/0"`` |
| `networkPolicies.ingress.enabled` | | ``false`` |
| `networkPolicies.internal.enabled` | | ``false`` |
-| `observability.enabled` | | ``false`` |
+| `observability.enabled` | | ``true`` |
| `observability.podMetrics.enabled` | | ``false`` |
-| `observability.prometheus.enabled` | | ``false`` |
+| `observability.prometheus.scrapeAnnotations.enabled` | | ``true`` |
+| `operator.args.enableInternalStatementLogging` | | ``true`` |
| `operator.args.startupLogFilter` | | ``"INFO,mz_orchestratord=TRACE"`` |
| `operator.cloudProvider.providers.aws.accountID` | | ``""`` |
| `operator.cloudProvider.providers.aws.enabled` | | ``false`` |
@@ -252,12 +253,13 @@ The following table lists the configurable parameters of the Materialize operato
| `operator.features.createConsole` | | ``true`` |
| `operator.image.pullPolicy` | | ``"IfNotPresent"`` |
| `operator.image.repository` | | ``"materialize/orchestratord"`` |
-| `operator.image.tag` | | ``"v0.125.2"`` |
+| `operator.image.tag` | | ``"v0.127.0-dev.0"`` |
| `operator.nodeSelector` | | ``{}`` |
| `operator.resources.limits.memory` | | ``"512Mi"`` |
| `operator.resources.requests.cpu` | | ``"100m"`` |
| `operator.resources.requests.memory` | | ``"512Mi"`` |
| `rbac.create` | | ``true`` |
+| `rbac.enabled` | | ``true`` |
| `serviceAccount.create` | | ``true`` |
| `serviceAccount.name` | | ``"orchestratord"`` |
| `storage.storageClass.allowVolumeExpansion` | | ``false`` |
@@ -269,12 +271,16 @@ The following table lists the configurable parameters of the Materialize operato
| `storage.storageClass.provisioner` | | ``""`` |
| `storage.storageClass.reclaimPolicy` | | ``"Delete"`` |
| `storage.storageClass.volumeBindingMode` | | ``"WaitForFirstConsumer"`` |
+| `telemetry.enabled` | | ``true`` |
+| `telemetry.segmentApiKey` | | ``"hMWi3sZ17KFMjn2sPWo9UJGpOQqiba4A"`` |
+| `telemetry.segmentClientSide` | | ``true`` |
+| `tls.defaultCertificateSpecs` | | ``{}`` |
Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example:
```shell
helm install my-materialize-operator \
- --set operator.image.tag=v0.125.2 \
+ --set operator.image.tag=v0.127.0-dev.0 \
materialize/materialize-operator
```
@@ -309,7 +315,7 @@ metadata:
name: 12345678-1234-1234-1234-123456789012
namespace: materialize-environment
spec:
- environmentdImageRef: materialize/environmentd:v0.125.2
+ environmentdImageRef: materialize/environmentd:v0.127.0-dev.0
backendSecretName: materialize-backend
environmentdResourceRequirements:
limits:
@@ -392,7 +398,7 @@ Or check the `Chart.yaml` file in the `misc/helm-charts/operator` directory:
apiVersion: v2
name: materialize-operator
# ...
-version: 25.1.0-beta.1
+version: v25.1.0-beta.1
appVersion: v0.125.2 # Use this version for your Materialize instances
```
diff --git a/misc/helm-charts/operator/README.md.gotmpl b/misc/helm-charts/operator/README.md.gotmpl
index 51903d14e4587..567c9fff6faad 100644
--- a/misc/helm-charts/operator/README.md.gotmpl
+++ b/misc/helm-charts/operator/README.md.gotmpl
@@ -245,7 +245,7 @@ Or check the `Chart.yaml` file in the `misc/helm-charts/operator` directory:
apiVersion: v2
name: materialize-operator
# ...
-version: 25.1.0-beta.1
+version: v25.1.0-beta.1
appVersion: v0.125.2 # Use this version for your Materialize instances
```
diff --git a/misc/helm-charts/operator/templates/clusterrole.yaml b/misc/helm-charts/operator/templates/clusterrole.yaml
index 3ef7aa9318206..531b74053e676 100644
--- a/misc/helm-charts/operator/templates/clusterrole.yaml
+++ b/misc/helm-charts/operator/templates/clusterrole.yaml
@@ -109,5 +109,16 @@ rules:
verbs:
- get
- list
+- apiGroups: ["cert-manager.io"]
+ resources:
+ - certificates
+ verbs:
+ - create
+ - update
+ - patch
+ - delete
+ - get
+ - list
+ - watch
{{- end }}
diff --git a/misc/helm-charts/operator/templates/deployment.yaml b/misc/helm-charts/operator/templates/deployment.yaml
index b39701d873abc..8ffc32e196663 100644
--- a/misc/helm-charts/operator/templates/deployment.yaml
+++ b/misc/helm-charts/operator/templates/deployment.yaml
@@ -42,6 +42,9 @@ spec:
{{- range $key, $value := include "materialize-operator.selectorLabels" . | fromYaml }}
- "--orchestratord-pod-selector-labels={{ $key }}={{ $value }}"
{{- end }}
+ {{- if .Values.operator.args.enableInternalStatementLogging }}
+ - "--enable-internal-statement-logging"
+ {{- end }}
{{/* AWS Configuration */}}
{{- if eq .Values.operator.cloudProvider.type "aws" }}
@@ -70,6 +73,11 @@ spec:
- "--console-image-tag-map={{ $key }}={{ $value }}"
{{- end }}
+ {{/* Authentication */}}
+ {{- if not .Values.rbac.enabled }}
+ - "--disable-authentication"
+ {{- end }}
+
{{/* Cluster Configuration */}}
{{- if .Values.operator.clusters }}
{{- if .Values.operator.clusters.sizes }}
@@ -123,12 +131,24 @@ spec:
{{- end }}
{{- end }}
{{- end }}
-
+ {{- if .Values.tls.defaultCertificateSpecs }}
+ - '--default-certificate-specs={{ toJson .Values.tls.defaultCertificateSpecs }}'
+ {{- end }}
{{/* Observability */}}
{{- if .Values.observability.enabled }}
{{- if .Values.observability.podMetrics.enabled }}
- "--collect-pod-metrics"
{{- end }}
+ {{- if .Values.observability.prometheus.scrapeAnnotations.enabled }}
+ - "--enable-prometheus-scrape-annotations"
+ {{- end }}
+ {{- end }}
+ {{/* Telemetry */}}
+ {{- if .Values.telemetry.enabled }}
+ - "--segment-api-key={{ .Values.telemetry.segmentApiKey }}"
+ {{- if .Values.telemetry.segmentClientSide }}
+ - "--segment-client-side"
+ {{- end }}
{{- end }}
resources:
{{- toYaml .Values.operator.resources | nindent 10 }}
diff --git a/misc/helm-charts/operator/tests/deployment_test.yaml b/misc/helm-charts/operator/tests/deployment_test.yaml
index a244fd4dac9e6..4e794687d3be0 100644
--- a/misc/helm-charts/operator/tests/deployment_test.yaml
+++ b/misc/helm-charts/operator/tests/deployment_test.yaml
@@ -9,155 +9,155 @@
suite: test deployment
templates:
- - deployment.yaml
+- deployment.yaml
tests:
- - it: should create a deployment
- asserts:
- - isKind:
- of: Deployment
- - equal:
- path: spec.template.spec.containers[0].image
- value: materialize/orchestratord:v0.125.2
- - equal:
- path: spec.template.spec.containers[0].imagePullPolicy
- value: IfNotPresent
- - equal:
- path: spec.template.spec.serviceAccountName
- value: orchestratord
+- it: should create a deployment
+ asserts:
+ - isKind:
+ of: Deployment
+ - equal:
+ path: spec.template.spec.containers[0].image
+ value: materialize/orchestratord:v0.127.0-dev.0
+ - equal:
+ path: spec.template.spec.containers[0].imagePullPolicy
+ value: IfNotPresent
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: orchestratord
- - it: should set ephemeral volume class when storage class is configured
- set:
- storage.storageClass.name: "my-storage-class"
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--ephemeral-volume-class=my-storage-class"
+- it: should set ephemeral volume class when storage class is configured
+ set:
+ storage.storageClass.name: "my-storage-class"
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--ephemeral-volume-class=my-storage-class"
- - it: should not set ephemeral volume class when storage class is not configured
- set:
- storage.storageClass.name: ""
- asserts:
- - notContains:
- path: spec.template.spec.containers[0].args
- content: "--ephemeral-volume-class"
+- it: should not set ephemeral volume class when storage class is not configured
+ set:
+ storage.storageClass.name: ""
+ asserts:
+ - notContains:
+ path: spec.template.spec.containers[0].args
+ content: "--ephemeral-volume-class"
- - it: should set correct base arguments
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--startup-log-filter=INFO,mz_orchestratord=TRACE"
+- it: should set correct base arguments
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--startup-log-filter=INFO,mz_orchestratord=TRACE"
- - it: should set resources correctly
- asserts:
- - equal:
- path: spec.template.spec.containers[0].resources.requests.cpu
- value: 100m
- - equal:
- path: spec.template.spec.containers[0].resources.requests.memory
- value: 512Mi
- - equal:
- path: spec.template.spec.containers[0].resources.limits.memory
- value: 512Mi
+- it: should set resources correctly
+ asserts:
+ - equal:
+ path: spec.template.spec.containers[0].resources.requests.cpu
+ value: 100m
+ - equal:
+ path: spec.template.spec.containers[0].resources.requests.memory
+ value: 512Mi
+ - equal:
+ path: spec.template.spec.containers[0].resources.limits.memory
+ value: 512Mi
- - it: should set disk limit to 0 when no storage class is configured
- set:
- storage.storageClass.name: ""
- asserts:
- - matchRegex:
- path: spec.template.spec.containers[0].args[9] # Index of the environmentd-cluster-replica-sizes argument
- pattern: disk_limit":"0"
- - matchRegex:
- path: spec.template.spec.containers[0].args[9]
- pattern: is_cc":true
+- it: should set disk limit to 0 when no storage class is configured
+ set:
+ storage.storageClass.name: ""
+ asserts:
+ - matchRegex:
+ path: spec.template.spec.containers[0].args[10] # Index of the environmentd-cluster-replica-sizes argument
+ pattern: disk_limit":"0"
+ - matchRegex:
+ path: spec.template.spec.containers[0].args[10]
+ pattern: is_cc":true
- - it: should have a cluster with disk limit to 1552MiB when storage class is configured
- set:
- storage.storageClass.name: "my-storage-class"
- asserts:
- - matchRegex:
- path: spec.template.spec.containers[0].args[9]
- pattern: disk_limit":"1552MiB"
- - matchRegex:
- path: spec.template.spec.containers[0].args[9]
- pattern: is_cc":true
+- it: should have a cluster with disk limit to 1552MiB when storage class is configured
+ set:
+ storage.storageClass.name: "my-storage-class"
+ asserts:
+ - matchRegex:
+ path: spec.template.spec.containers[0].args[10]
+ pattern: disk_limit":"1552MiB"
+ - matchRegex:
+ path: spec.template.spec.containers[0].args[10]
+ pattern: is_cc":true
- - it: should configure for AWS provider correctly
- set:
- operator.cloudProvider.type: "aws"
- operator.cloudProvider.region: "us-east-1"
- operator.cloudProvider.providers.aws:
- enabled: true
- accountID: "123456789012"
- iam:
- roles:
- environment: "arn:aws:iam::123456789012:role/env-role"
- connection: "arn:aws:iam::123456789012:role/conn-role"
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--cloud-provider=aws"
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--region=us-east-1"
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--aws-account-id=123456789012"
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--environmentd-iam-role-arn=arn:aws:iam::123456789012:role/env-role"
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--environmentd-connection-role-arn=arn:aws:iam::123456789012:role/conn-role"
+- it: should configure for AWS provider correctly
+ set:
+ operator.cloudProvider.type: "aws"
+ operator.cloudProvider.region: "us-east-1"
+ operator.cloudProvider.providers.aws:
+ enabled: true
+ accountID: "123456789012"
+ iam:
+ roles:
+ environment: "arn:aws:iam::123456789012:role/env-role"
+ connection: "arn:aws:iam::123456789012:role/conn-role"
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--cloud-provider=aws"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--region=us-east-1"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--aws-account-id=123456789012"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--environmentd-iam-role-arn=arn:aws:iam::123456789012:role/env-role"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--environmentd-connection-role-arn=arn:aws:iam::123456789012:role/conn-role"
- - it: should configure for generic provider correctly
- set:
- operator.cloudProvider.type: "generic"
- operator.cloudProvider.region: "local"
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--cloud-provider=generic"
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--region=local"
- - notContains:
- path: spec.template.spec.containers[0].args
- content: "--aws-account-id"
+- it: should configure for generic provider correctly
+ set:
+ operator.cloudProvider.type: "generic"
+ operator.cloudProvider.region: "local"
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--cloud-provider=generic"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--region=local"
+ - notContains:
+ path: spec.template.spec.containers[0].args
+ content: "--aws-account-id"
# Feature Flag Tests
- - it: should enable balancer creation when configured
- set:
- operator.features.createBalancers: true
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--create-balancers"
+- it: should enable balancer creation when configured
+ set:
+ operator.features.createBalancers: true
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--create-balancers"
- - it: should not enable balancer creation when disabled
- set:
- operator.features.createBalancers: false
- asserts:
- - notContains:
- path: spec.template.spec.containers[0].args
- content: "--create-balancers"
+- it: should not enable balancer creation when disabled
+ set:
+ operator.features.createBalancers: false
+ asserts:
+ - notContains:
+ path: spec.template.spec.containers[0].args
+ content: "--create-balancers"
- - it: should enable console creation when configured
- set:
- operator.features.createConsole: true
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--create-console"
+- it: should enable console creation when configured
+ set:
+ operator.features.createConsole: true
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--create-console"
- - it: should configure console image tag map override correctly
- set:
- operator.features.consoleImageTagMapOverride:
- "v0.125.0": "25.1.0"
- "v0.126.0": "25.2.0"
- asserts:
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--console-image-tag-map=v0.125.0=25.1.0"
- - contains:
- path: spec.template.spec.containers[0].args
- content: "--console-image-tag-map=v0.126.0=25.2.0"
+- it: should configure console image tag map override correctly
+ set:
+ operator.features.consoleImageTagMapOverride:
+ "v0.125.0": "25.1.0"
+ "v0.126.0": "25.2.0"
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--console-image-tag-map=v0.125.0=25.1.0"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--console-image-tag-map=v0.126.0=25.2.0"
diff --git a/misc/helm-charts/operator/values.yaml b/misc/helm-charts/operator/values.yaml
index c97c34d05b4c5..145bc9b50e013 100644
--- a/misc/helm-charts/operator/values.yaml
+++ b/misc/helm-charts/operator/values.yaml
@@ -13,13 +13,14 @@ operator:
# The Docker repository for the operator image
repository: materialize/orchestratord
# The tag/version of the operator image to be used
- tag: v0.125.2
+ tag: v0.127.0-dev.0
# Policy for pulling the image: "IfNotPresent" avoids unnecessary re-pulling of images
pullPolicy: IfNotPresent
args:
# Log filtering settings for startup logs
startupLogFilter: "INFO,mz_orchestratord=TRACE"
+ enableInternalStatementLogging: true
features:
# Flag to indicate whether to create balancerd pods for the environments
@@ -214,6 +215,7 @@ clusterd:
# RBAC (Role-Based Access Control) settings
rbac:
+ enabled: true
# Whether to create necessary RBAC roles and bindings
create: true
@@ -226,15 +228,21 @@ serviceAccount:
# Observability settings (disabled in this case)
observability:
- enabled: false
+ enabled: true
podMetrics:
# Whether to enable the pod metrics scraper which populates the
# Environment Overview Monitoring tab in the web console (requires
# metrics-server to be installed)
enabled: false
prometheus:
- # Whether to enable Prometheus integration for monitoring (disabled here)
- enabled: false
+ scrapeAnnotations:
+ # Whether to annotate pods with common keys used for prometheus scraping.
+ enabled: true
+
+telemetry:
+ enabled: true
+ segmentApiKey: hMWi3sZ17KFMjn2sPWo9UJGpOQqiba4A
+ segmentClientSide: true
# Network policies configuration
networkPolicies:
@@ -248,12 +256,31 @@ networkPolicies:
ingress:
enabled: false
cidrs:
- - 0.0.0.0/0
+ - 0.0.0.0/0
# egress from Materialize pods to sources and sinks
egress:
enabled: false
cidrs:
- - 0.0.0.0/0
+ - 0.0.0.0/0
+
+tls:
+ defaultCertificateSpecs: {}
+ #balancerdExternal:
+ # dnsNames:
+ # - balancerd
+ # issuerRef:
+ # name: dns01
+ # kind: ClusterIssuer
+ #consoleExternal:
+ # dnsNames:
+ # - console
+ # issuerRef:
+ # name: dns01
+ # kind: ClusterIssuer
+ #internal:
+ # issuerRef:
+ # name: dns01
+ # kind: ClusterIssuer
# Namespace configuration
namespace:
diff --git a/misc/helm-charts/testing/environmentd.yaml b/misc/helm-charts/testing/environmentd.yaml
index 431ae370a9dc9..7b1310f1d6e15 100644
--- a/misc/helm-charts/testing/environmentd.yaml
+++ b/misc/helm-charts/testing/environmentd.yaml
@@ -1,4 +1,3 @@
----
# Source: materialize-environmentd/templates/secret.yaml
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
@@ -29,5 +28,21 @@ metadata:
name: 12345678-1234-1234-1234-123456789012
namespace: materialize-environment
spec:
- environmentdImageRef: materialize/environmentd:v0.125.2
+ environmentdImageRef: materialize/environmentd:v0.127.0-dev.0
backendSecretName: materialize-backend
+ #balancerdExternalCertificateSpec:
+ # dnsNames:
+ # - balancerd
+ # issuerRef:
+ # name: dns01
+ # kind: ClusterIssuer
+ #consoleExternalCertificateSpec:
+ # dnsNames:
+ # - console
+ # issuerRef:
+ # name: dns01
+ # kind: ClusterIssuer
+ #internalCertificateSpec:
+ # issuerRef:
+ # name: intermediate-ca
+ # kind: Issuer
diff --git a/misc/python/materialize/buildkite_insights/costs/extract_stats.py b/misc/python/materialize/buildkite_insights/costs/extract_stats.py
index d33b29c5885e3..ad457e7f46d68 100755
--- a/misc/python/materialize/buildkite_insights/costs/extract_stats.py
+++ b/misc/python/materialize/buildkite_insights/costs/extract_stats.py
@@ -65,9 +65,12 @@
"x86-64-4cpu-8gb": 0.0113,
"x86-64-8cpu-16gb": 0.0273,
"x86-64-16cpu-32gb": 0.0540,
- "x86-64-16cpu-64gb": 0.1546,
- "x86-64-32cpu-128gb": 0.3085,
- "x86-64-48cpu-192gb": 0.4623,
+ "x86-64-dedi-2cpu-8gb": 0.0200,
+ "x86-64-dedi-4cpu-16gb": 0.0392,
+ "x86-64-dedi-8cpu-32gb": 0.0777,
+ "x86-64-dedi-16cpu-64gb": 0.1546,
+ "x86-64-dedi-32cpu-128gb": 0.3085,
+ "x86-64-dedi-48cpu-192gb": 0.4623,
"x86-64": 0, # local experiments
}
diff --git a/misc/python/materialize/cli/ci_annotate_errors.py b/misc/python/materialize/cli/ci_annotate_errors.py
index fdd7cd8986a6e..eee396590bff8 100644
--- a/misc/python/materialize/cli/ci_annotate_errors.py
+++ b/misc/python/materialize/cli/ci_annotate_errors.py
@@ -143,7 +143,7 @@
# Will print a separate panic line which will be handled and contains the relevant information (new style)
| internal\ error:\ unexpected\ panic\ during\ query\ optimization
# redpanda INFO logging
- | larger\ sizes\ prevent\ running\ out\ of\ memory
+ | [Ll]arger\ sizes\ prevent\ running\ out\ of\ memory
# Old versions won't support new parameters
| (platform-checks|legacy-upgrade|upgrade-matrix|feature-benchmark)-materialized-.* \| .*cannot\ load\ unknown\ system\ parameter\ from\ catalog\ storage
# Fencing warnings are OK in fencing/0dt tests
diff --git a/misc/python/materialize/cli/scratch/forward.py b/misc/python/materialize/cli/scratch/forward.py
index 8c23977ef9b12..327132b8fb3ed 100644
--- a/misc/python/materialize/cli/scratch/forward.py
+++ b/misc/python/materialize/cli/scratch/forward.py
@@ -9,21 +9,22 @@
import argparse
-import boto3
-
from materialize.cli.scratch import check_required_vars
-from materialize.scratch import mssh
+from materialize.scratch import get_instance, mssh
def configure_parser(parser: argparse.ArgumentParser) -> None:
check_required_vars()
- parser.add_argument("instance", help="The ID of the instance to connect to")
+ parser.add_argument(
+ "instance",
+ help="The ID of the instance to connect to, or 'mine' to specify your only live instance",
+ )
parser.add_argument("ports", nargs="*", help="The remote ports to forward locally")
def run(args: argparse.Namespace) -> None:
- instance = boto3.resource("ec2").Instance(args.instance)
+ instance = get_instance(args.instance)
ssh_args = []
for port in args.ports:
ssh_args.extend(["-L", f"{port}:127.0.0.1:{port}"])
diff --git a/misc/python/materialize/cli/scratch/push.py b/misc/python/materialize/cli/scratch/push.py
index 9cd4f236d8657..fbe3ded3544ab 100644
--- a/misc/python/materialize/cli/scratch/push.py
+++ b/misc/python/materialize/cli/scratch/push.py
@@ -9,20 +9,21 @@
import argparse
-import boto3
-
from materialize.cli.scratch import check_required_vars
-from materialize.scratch import mkrepo
+from materialize.scratch import get_instance, mkrepo
def configure_parser(parser: argparse.ArgumentParser) -> None:
check_required_vars()
- parser.add_argument("instance", help="The ID of the instance to connect to")
+ parser.add_argument(
+ "instance",
+ help="The ID of the instance to connect to, or 'mine' to specify your only live instance",
+ )
parser.add_argument("--rev", help="The git rev to checkout", default="HEAD")
def run(args: argparse.Namespace) -> None:
- instance = boto3.resource("ec2").Instance(args.instance)
+ instance = get_instance(args.instance)
mkrepo(instance, args.rev, init=False, force=True)
diff --git a/misc/python/materialize/cli/scratch/sftp.py b/misc/python/materialize/cli/scratch/sftp.py
index 7cb747b04d760..3c05fd96304bd 100644
--- a/misc/python/materialize/cli/scratch/sftp.py
+++ b/misc/python/materialize/cli/scratch/sftp.py
@@ -9,18 +9,19 @@
import argparse
-import boto3
-
from materialize.cli.scratch import check_required_vars
-from materialize.scratch import msftp
+from materialize.scratch import get_instance, msftp
def configure_parser(parser: argparse.ArgumentParser) -> None:
check_required_vars()
- parser.add_argument("instance", help="The ID of the instance to connect to")
+ parser.add_argument(
+ "instance",
+ help="The ID of the instance to connect to, or 'mine' to specify your only live instance",
+ )
def run(args: argparse.Namespace) -> None:
- instance = boto3.resource("ec2").Instance(args.instance)
+ instance = get_instance(args.instance)
msftp(instance)
diff --git a/misc/python/materialize/cli/scratch/ssh.py b/misc/python/materialize/cli/scratch/ssh.py
index fa60512647f20..97d10b315dac2 100644
--- a/misc/python/materialize/cli/scratch/ssh.py
+++ b/misc/python/materialize/cli/scratch/ssh.py
@@ -9,21 +9,22 @@
import argparse
-import boto3
-
from materialize.cli.scratch import check_required_vars
-from materialize.scratch import mssh
+from materialize.scratch import get_instance, mssh
def configure_parser(parser: argparse.ArgumentParser) -> None:
check_required_vars()
- parser.add_argument("instance", help="The ID of the instance to connect to")
+ parser.add_argument(
+ "instance",
+ help="The ID of the instance to connect to, or 'mine' to specify your only live instance",
+ )
parser.add_argument("command", nargs="*", help="The command to run via SSH, if any")
def run(args: argparse.Namespace) -> None:
# SSH will join together multiple arguments with spaces, so we don't lose
# anything by doing the same.
- instance = boto3.resource("ec2").Instance(args.instance)
+ instance = get_instance(args.instance)
mssh(instance, " ".join(args.command))
diff --git a/misc/python/materialize/cloudtest/k8s/redpanda.py b/misc/python/materialize/cloudtest/k8s/redpanda.py
index 013ccdc1765cf..621c446eae852 100644
--- a/misc/python/materialize/cloudtest/k8s/redpanda.py
+++ b/misc/python/materialize/cloudtest/k8s/redpanda.py
@@ -24,6 +24,7 @@
from materialize.cloudtest.k8s.api.k8s_deployment import K8sDeployment
from materialize.cloudtest.k8s.api.k8s_resource import K8sResource
from materialize.cloudtest.k8s.api.k8s_service import K8sService
+from materialize.mzcompose.services.redpanda import REDPANDA_VERSION
class RedpandaDeployment(K8sDeployment):
@@ -31,7 +32,7 @@ def __init__(self, namespace: str, apply_node_selectors: bool) -> None:
super().__init__(namespace)
container = V1Container(
name="redpanda",
- image="vectorized/redpanda:v24.2.2",
+ image=f"redpandadata/redpanda:{REDPANDA_VERSION}",
command=[
"/usr/bin/rpk",
"redpanda",
diff --git a/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py b/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py
index c05691fd04bbf..5aceb2154dcc4 100644
--- a/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py
+++ b/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py
@@ -239,6 +239,10 @@ def init(self) -> Action:
def benchmark(self) -> MeasurementSource:
return Td(
f"""
+$ postgres-connect name=mz_system url=postgres://mz_system:materialize@${{testdrive.materialize-internal-sql-addr}}
+$ postgres-execute connection=mz_system
+ALTER SYSTEM SET max_result_size = 17179869184;
+
> DROP TABLE IF EXISTS t1;
> CREATE TABLE t1 (f1 INTEGER)
@@ -311,7 +315,10 @@ def benchmark(self) -> MeasurementSource:
class InsertMultiRow(DML):
- """Measure the time it takes for a single multi-row INSERT statement to return."""
+ """Measure the time it takes for a single multi-row INSERT statement to return.
+ When `sequence_insert` calls `constant_optimizer`, it should be able to reach a constant. Otherwise, we run the full
+ logical optimizer, which makes this test show a regression.
+ """
SCALE = 4 # FATAL: request larger than 2.0 MB
@@ -334,7 +341,7 @@ def benchmark(self) -> MeasurementSource:
class Update(DML):
"""Measure the time it takes for an UPDATE statement to return to client"""
- SCALE = 6 # TODO: Increase scale when database-issues#8766 is fixed
+ SCALE = 7
def init(self) -> list[Action]:
return [
@@ -461,6 +468,10 @@ def init(self) -> Action:
def benchmark(self) -> MeasurementSource:
return Td(
f"""
+$ postgres-connect name=mz_system url=postgres://mz_system:materialize@${{testdrive.materialize-internal-sql-addr}}
+$ postgres-execute connection=mz_system
+ALTER SYSTEM SET max_result_size = 17179869184;
+
> DROP TABLE IF EXISTS t1;
> CREATE TABLE t1 (f1 INTEGER)
diff --git a/misc/python/materialize/feature_benchmark/scenarios/concurrency.py b/misc/python/materialize/feature_benchmark/scenarios/concurrency.py
index e33e2030e17bb..5fc82ded8dbde 100644
--- a/misc/python/materialize/feature_benchmark/scenarios/concurrency.py
+++ b/misc/python/materialize/feature_benchmark/scenarios/concurrency.py
@@ -10,6 +10,7 @@
from materialize.feature_benchmark.action import Action, TdAction
from materialize.feature_benchmark.measurement_source import MeasurementSource, Td
from materialize.feature_benchmark.scenario import Scenario
+from materialize.feature_benchmark.scenario_version import ScenarioVersion
class Concurrency(Scenario):
@@ -22,6 +23,9 @@ class ParallelIngestion(Concurrency):
SOURCES = 10
FIXED_SCALE = True # Disk slowness in CRDB leading to CRDB going down
+ def version(self) -> ScenarioVersion:
+ return ScenarioVersion.create(1, 1, 0)
+
def shared(self) -> Action:
return TdAction(
self.schema()
@@ -72,7 +76,7 @@ def benchmark(self) -> MeasurementSource:
create_indexes = "\n".join(
[
f"""
-> CREATE DEFAULT INDEX ON s{s}
+> CREATE DEFAULT INDEX ON s{s}_tbl
"""
for s in sources
]
diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py
index 482068ce95e3b..dcc325ba862cf 100644
--- a/misc/python/materialize/mzcompose/__init__.py
+++ b/misc/python/materialize/mzcompose/__init__.py
@@ -146,6 +146,8 @@ def get_default_system_parameters(
"persist_schema_register": (
"false" if version < MzVersion.parse_mz("v0.111.0-dev") else "true"
),
+ # 16 MiB - large enough to avoid a big perf hit, small enough to get more coverage...
+ "persist_blob_target_size": "16777216",
"persist_schema_require": "true",
"persist_stats_audit_percent": "100",
"persist_txn_tables": "lazy", # removed, but keep value for older versions
diff --git a/misc/python/materialize/mzcompose/service.py b/misc/python/materialize/mzcompose/service.py
index ce937a221758d..416926be5542b 100644
--- a/misc/python/materialize/mzcompose/service.py
+++ b/misc/python/materialize/mzcompose/service.py
@@ -171,6 +171,9 @@ class ServiceConfig(TypedDict, total=False):
stop_grace_period: str | None
"""Time to wait when stopping a container."""
+ network_mode: str | None
+ """Network mode."""
+
class Service:
"""A Docker Compose service in a `Composition`.
diff --git a/misc/python/materialize/mzcompose/services/cockroach.py b/misc/python/materialize/mzcompose/services/cockroach.py
index fdc9e866f8034..946ec16093d46 100644
--- a/misc/python/materialize/mzcompose/services/cockroach.py
+++ b/misc/python/materialize/mzcompose/services/cockroach.py
@@ -21,6 +21,7 @@
class Cockroach(Service):
+ # TODO: Bump version to >= v24.3.0 when https://github.com/cockroachdb/cockroach/issues/136678 is fixed
DEFAULT_COCKROACH_TAG = "v24.2.0"
def __init__(
diff --git a/misc/python/materialize/mzcompose/services/redpanda.py b/misc/python/materialize/mzcompose/services/redpanda.py
index 873d9818e9eb3..67c0859ae9a73 100644
--- a/misc/python/materialize/mzcompose/services/redpanda.py
+++ b/misc/python/materialize/mzcompose/services/redpanda.py
@@ -13,7 +13,7 @@
ServiceConfig,
)
-REDPANDA_VERSION = "v24.2.7"
+REDPANDA_VERSION = "v24.3.1"
class Redpanda(Service):
@@ -27,7 +27,7 @@ def __init__(
ports: list[int] | None = None,
) -> None:
if image is None:
- image = f"vectorized/redpanda:{version}"
+ image = f"redpandadata/redpanda:{version}"
if ports is None:
ports = [9092, 8081]
@@ -37,7 +37,7 @@ def __init__(
aliases = ["kafka", "schema-registry"]
# Most of these options are simply required when using Redpanda in Docker.
- # See: https://vectorized.io/docs/quick-start-docker/#Single-command-for-a-1-node-cluster
+ # See: https://docs.redpanda.com/current/get-started/quick-start/#Single-command-for-a-1-node-cluster
# The `enable_transactions` and `enable_idempotence` feature flags enable
# features Materialize requires that are present by default in Apache Kafka
# but not in Redpanda.
diff --git a/misc/python/materialize/mzcompose/services/testdrive.py b/misc/python/materialize/mzcompose/services/testdrive.py
index d6d6b36450527..1781ae8ae2fb6 100644
--- a/misc/python/materialize/mzcompose/services/testdrive.py
+++ b/misc/python/materialize/mzcompose/services/testdrive.py
@@ -15,6 +15,7 @@
from materialize.mzcompose import DEFAULT_MZ_VOLUMES, cluster_replica_size_map
from materialize.mzcompose.service import (
Service,
+ ServiceConfig,
ServiceDependency,
)
from materialize.mzcompose.services.postgres import METADATA_STORE
@@ -59,6 +60,7 @@ def __init__(
metadata_store: str = METADATA_STORE,
stop_grace_period: str = "120s",
cluster_replica_size: dict[str, dict[str, Any]] | None = None,
+ network_mode: str | None = None,
) -> None:
depends_graph: dict[str, ServiceDependency] = {}
@@ -174,16 +176,20 @@ def __init__(
entrypoint.extend(entrypoint_extra)
+ config: ServiceConfig = {
+ "depends_on": depends_graph,
+ "mzbuild": mzbuild,
+ "entrypoint": entrypoint,
+ "environment": environment,
+ "volumes": volumes,
+ "propagate_uid_gid": propagate_uid_gid,
+ "init": True,
+ "stop_grace_period": stop_grace_period,
+ }
+ if network_mode:
+ config["network_mode"] = network_mode
+
super().__init__(
name=name,
- config={
- "depends_on": depends_graph,
- "mzbuild": mzbuild,
- "entrypoint": entrypoint,
- "environment": environment,
- "volumes": volumes,
- "propagate_uid_gid": propagate_uid_gid,
- "init": True,
- "stop_grace_period": stop_grace_period,
- },
+ config=config,
)
diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py
index fc89c14e2ac25..d892875e46dbb 100644
--- a/misc/python/materialize/parallel_workload/action.py
+++ b/misc/python/materialize/parallel_workload/action.py
@@ -994,6 +994,10 @@ def __init__(
BOOLEAN_FLAG_VALUES = ["TRUE", "FALSE"]
self.flags_with_values: dict[str, list[str]] = dict()
+ self.flags_with_values["persist_blob_target_size"] = (
+ # 1 MiB, 16 MiB, 128 MiB
+ ["1048576", "16777216", "134217728"]
+ )
for flag in ["catalog", "source", "snapshot", "txn"]:
self.flags_with_values[f"persist_use_critical_since_{flag}"] = (
BOOLEAN_FLAG_VALUES
diff --git a/misc/python/materialize/scratch.py b/misc/python/materialize/scratch.py
index 5cde3e3d18459..5ce8f8ef0c8fe 100644
--- a/misc/python/materialize/scratch.py
+++ b/misc/python/materialize/scratch.py
@@ -24,6 +24,7 @@
from mypy_boto3_ec2.literals import InstanceTypeType
from mypy_boto3_ec2.service_resource import Instance
from mypy_boto3_ec2.type_defs import (
+ FilterTypeDef,
InstanceNetworkInterfaceSpecificationTypeDef,
InstanceTypeDef,
RunInstancesRequestRequestTypeDef,
@@ -367,6 +368,32 @@ def whoami() -> str:
return boto3.client("sts").get_caller_identity()["UserId"].split(":")[1]
+def get_instance(name: str) -> Instance:
+ """
+ Get an instance by instance id. The special name 'mine' resolves to a
+ unique running owned instance, if there is one; otherwise the name is
+ assumed to be an instance id.
+ :param name: The instance id or the special case 'mine'.
+ :return: The instance to which the name refers.
+ """
+ if name == "mine":
+ filters: list[FilterTypeDef] = [
+ {"Name": "tag:LaunchedBy", "Values": [whoami()]},
+ {"Name": "instance-state-name", "Values": ["pending", "running"]},
+ ]
+ instances = [i for i in boto3.resource("ec2").instances.filter(Filters=filters)]
+ if not instances:
+ raise RuntimeError("can't understand 'mine': no owned instance?")
+ if len(instances) > 1:
+ raise RuntimeError(
+ f"can't understand 'mine': too many owned instances ({', '.join(i.id for i in instances)})"
+ )
+ instance = instances[0]
+ say(f"understanding 'mine' as unique owned instance {instance.id}")
+ return instance
+ return boto3.resource("ec2").Instance(name)
+
+
def get_instances_by_tag(k: str, v: str) -> list[InstanceTypeDef]:
return [
i
diff --git a/misc/python/materialize/test_analytics/config/test_analytics_db_config.py b/misc/python/materialize/test_analytics/config/test_analytics_db_config.py
index c7965d9648250..4a4cb34cdc852 100644
--- a/misc/python/materialize/test_analytics/config/test_analytics_db_config.py
+++ b/misc/python/materialize/test_analytics/config/test_analytics_db_config.py
@@ -9,7 +9,7 @@
import os
-from materialize import buildkite
+from materialize import buildkite, ui
from materialize.mz_env_util import get_cloud_hostname
from materialize.mzcompose.composition import Composition
from materialize.test_analytics.config.mz_db_config import MzDbConfig
@@ -20,7 +20,12 @@ def create_test_analytics_config(c: Composition) -> MzDbConfig:
app_password = os.getenv("PRODUCTION_ANALYTICS_APP_PASSWORD")
if app_password is not None:
- hostname = get_cloud_hostname(c, app_password=app_password)
+ try:
+ hostname = get_cloud_hostname(c, app_password=app_password)
+ except ui.CommandFailureCausedUIError as e:
+ # TODO: Remove when database-issues#8592 is fixed
+ print(f"Failed to get cloud hostname ({e}), using fallback value")
+ hostname = "7vifiksqeftxc6ld3r6zvc8n2.lb.us-east-1.aws.materialize.cloud"
else:
hostname = "unknown"
diff --git a/misc/python/materialize/version_ancestor_overrides.py b/misc/python/materialize/version_ancestor_overrides.py
index 8f29fcf21dfd9..59fafdf775602 100644
--- a/misc/python/materialize/version_ancestor_overrides.py
+++ b/misc/python/materialize/version_ancestor_overrides.py
@@ -28,6 +28,28 @@ def get_ancestor_overrides_for_performance_regressions(
min_ancestor_mz_version_per_commit = dict()
+ if scenario_class_name in ("KafkaUpsertUnique", "ParallelIngestion"):
+ # PR#30617 (storage/kafka: use separate consumer for metadata probing)
+ # adds 1s of delay to Kafka source startup
+ min_ancestor_mz_version_per_commit[
+ "9f7b634e6824f73d0effcdfa86c2b8b1642a4784"
+ ] = MzVersion.parse_mz("v0.127.0")
+ if scenario_class_name == "InsertMultiRow":
+ # PR#30622 (Refactor how we run FoldConstants) increases wallclock
+ min_ancestor_mz_version_per_commit[
+ "a558d6bdc4b29abf79457eaba52914a0d6c805b7"
+ ] = MzVersion.parse_mz("v0.127.0")
+ if scenario_class_name == "CrossJoin":
+ # PR#26745 (compute: MV sink refresh) increases wallclock
+ min_ancestor_mz_version_per_commit[
+ "9485261eae85fb7f12a2884fdac464a68798dc8b"
+ ] = MzVersion.parse_mz("v0.126.0")
+ if "OptbenchTPCH" in scenario_class_name:
+ # PR#30602 (Replace ColumnKnowledge with EquivalencePropagation) increases wallclock
+ min_ancestor_mz_version_per_commit[
+ "1bd45336f8335b3487153beb7ce57f6391a7cf9c"
+ ] = MzVersion.parse_mz("v0.126.0")
+
if "OptbenchTPCH" in scenario_class_name:
# PR#30506 (Remove NonNullable transform) increases wallclock
min_ancestor_mz_version_per_commit[
diff --git a/src/adapter/src/catalog.rs b/src/adapter/src/catalog.rs
index bccf6580692bf..844f4741f1bee 100644
--- a/src/adapter/src/catalog.rs
+++ b/src/adapter/src/catalog.rs
@@ -518,14 +518,52 @@ impl Catalog {
Fut: Future