From c24f3b412a29559e3e0c216c29d5b58ee7ca9340 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Wed, 13 Dec 2023 19:18:54 +0530 Subject: [PATCH 01/19] Runtime: otel fix for db close leaks (#3690) * otel fix for db close leaks * comment when to revert * fix test case - wal doesn't exist * otel pkgs upgrade and code changes * deployment not found fix --- admin/client/client.go | 3 +- admin/database/postgres/postgres.go | 2 +- admin/server/auth/middleware.go | 2 +- admin/server/server.go | 4 +- cli/cmd/project/refresh.go | 4 + go.mod | 73 +++++----- go.sum | 154 ++++++++++----------- runtime/client/client.go | 3 +- runtime/drivers/duckdb/duckdb.go | 48 +------ runtime/drivers/duckdb/duckdb_test.go | 1 - runtime/pkg/observability/middleware.go | 11 -- runtime/pkg/observability/observability.go | 2 +- runtime/server/auth/interceptors.go | 2 +- runtime/server/server.go | 4 +- 14 files changed, 128 insertions(+), 185 deletions(-) diff --git a/admin/client/client.go b/admin/client/client.go index cc7384013f7..c2c6d8caf9c 100644 --- a/admin/client/client.go +++ b/admin/client/client.go @@ -28,8 +28,7 @@ func New(adminHost, bearerToken, userAgent string) (*Client, error) { opts := []grpc.DialOption{ grpc.WithUserAgent(userAgent), - grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()), - grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()), + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), } if uri.Scheme == "http" { diff --git a/admin/database/postgres/postgres.go b/admin/database/postgres/postgres.go index 9449d08b56a..2e2441a78ca 100644 --- a/admin/database/postgres/postgres.go +++ b/admin/database/postgres/postgres.go @@ -11,7 +11,7 @@ import ( "github.com/jackc/pgconn" "github.com/jmoiron/sqlx" "github.com/rilldata/rill/admin/database" - semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" // Load postgres driver _ "github.com/jackc/pgx/v4/stdlib" diff --git a/admin/server/auth/middleware.go b/admin/server/auth/middleware.go index 574517a9c87..8e1f7ac315b 100644 --- a/admin/server/auth/middleware.go +++ b/admin/server/auth/middleware.go @@ -9,7 +9,7 @@ import ( grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" - semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/admin/server/server.go b/admin/server/server.go index e71a55accf0..501761a572a 100644 --- a/admin/server/server.go +++ b/admin/server/server.go @@ -27,6 +27,7 @@ import ( "github.com/rilldata/rill/runtime/pkg/ratelimit" runtimeauth "github.com/rilldata/rill/runtime/server/auth" "github.com/rs/cors" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.uber.org/zap" "google.golang.org/grpc" @@ -121,7 +122,6 @@ func (s *Server) ServeGRPC(ctx context.Context) error { server := grpc.NewServer( grpc.ChainStreamInterceptor( middleware.TimeoutStreamServerInterceptor(timeoutSelector), - observability.TracingStreamServerInterceptor(), observability.LoggingStreamServerInterceptor(s.logger), errorMappingStreamServerInterceptor(), grpc_auth.StreamServerInterceptor(checkUserAgent), @@ -131,7 +131,6 @@ func (s *Server) ServeGRPC(ctx context.Context) error { ), grpc.ChainUnaryInterceptor( middleware.TimeoutUnaryServerInterceptor(timeoutSelector), - observability.TracingUnaryServerInterceptor(), observability.LoggingUnaryServerInterceptor(s.logger), errorMappingUnaryServerInterceptor(), grpc_auth.UnaryServerInterceptor(checkUserAgent), @@ -139,6 +138,7 @@ func (s *Server) ServeGRPC(ctx context.Context) error { s.authenticator.UnaryServerInterceptor(), grpc_auth.UnaryServerInterceptor(s.checkRateLimit), ), + grpc.StatsHandler(otelgrpc.NewServerHandler()), ) adminv1.RegisterAdminServiceServer(server, s) diff --git a/cli/cmd/project/refresh.go b/cli/cmd/project/refresh.go index a67be9611db..e45c31feac5 100644 --- a/cli/cmd/project/refresh.go +++ b/cli/cmd/project/refresh.go @@ -47,6 +47,10 @@ func RefreshCmd(ch *cmdutil.Helper) *cobra.Command { return err } + if resp.ProdDeployment == nil { + return fmt.Errorf("no production deployment found for project %q", project) + } + _, err = client.TriggerRefreshSources(ctx, &adminv1.TriggerRefreshSourcesRequest{DeploymentId: resp.ProdDeployment.Id, Sources: source}) if err != nil { return fmt.Errorf("failed to trigger refresh: %w", err) diff --git a/go.mod b/go.mod index 9bb75f4bc28..ef490e9266c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/MicahParks/keyfunc v1.9.0 github.com/NYTimes/gziphandler v1.1.1 - github.com/XSAM/otelsql v0.23.0 + github.com/XSAM/otelsql v0.26.0 github.com/alicebob/miniredis v2.5.0+incompatible github.com/apache/arrow/go/v14 v14.0.1 github.com/apache/calcite-avatica-go/v5 v5.2.0 @@ -39,7 +39,7 @@ require ( github.com/gorilla/securecookie v1.1.1 github.com/gorilla/sessions v1.2.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/golang-lru v0.6.0 github.com/jackc/pgconn v1.14.0 @@ -52,7 +52,7 @@ require ( github.com/marcboeker/go-duckdb v1.5.4 github.com/mazznoer/csscolorparser v0.1.3 github.com/mitchellh/mapstructure v1.5.0 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.17.0 github.com/redis/go-redis/v9 v9.0.2 github.com/rs/cors v1.9.0 github.com/snowflakedb/gosnowflake v1.7.0 @@ -61,18 +61,18 @@ require ( github.com/stretchr/testify v1.8.4 github.com/testcontainers/testcontainers-go v0.19.0 github.com/xuri/excelize/v2 v2.7.1 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 - go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 - go.opentelemetry.io/otel v1.16.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 - go.opentelemetry.io/otel/exporters/prometheus v0.39.0 - go.opentelemetry.io/otel/metric v1.16.0 - go.opentelemetry.io/otel/sdk v1.16.0 - go.opentelemetry.io/otel/sdk/metric v0.39.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 + go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1 + go.opentelemetry.io/otel v1.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 + go.opentelemetry.io/otel/exporters/prometheus v0.44.0 + go.opentelemetry.io/otel/metric v1.21.0 + go.opentelemetry.io/otel/sdk v1.21.0 + go.opentelemetry.io/otel/sdk/metric v1.21.0 + go.opentelemetry.io/otel/trace v1.21.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.25.0 go.uber.org/zap/exp v0.1.0 @@ -80,9 +80,9 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/oauth2 v0.13.0 golang.org/x/sync v0.4.0 - golang.org/x/sys v0.14.0 + golang.org/x/sys v0.15.0 google.golang.org/api v0.149.0 - google.golang.org/grpc v1.59.0 + google.golang.org/grpc v1.60.0 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/square/go-jose.v2 v2.6.0 @@ -109,17 +109,18 @@ require ( github.com/jackc/pgtype v1.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect ) require ( - cloud.google.com/go v0.110.8 // indirect + cloud.google.com/go v0.110.10 // indirect cloud.google.com/go/bigquery v1.57.1 - cloud.google.com/go/compute v1.23.1 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -163,13 +164,13 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -217,7 +218,6 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect @@ -235,9 +235,9 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect @@ -254,19 +254,17 @@ require ( github.com/yuin/gopher-lua v1.1.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect - golang.org/x/crypto v0.15.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + golang.org/x/crypto v0.16.0 // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/term v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect @@ -290,3 +288,6 @@ replace github.com/marcboeker/go-duckdb v1.5.4 => github.com/rilldata/go-duckdb replace github.com/snowflakedb/gosnowflake v1.7.0 => github.com/esevastyanov/gosnowflake v0.0.0-20231129090721-012eca4e3448 exclude modernc.org/sqlite v1.18.1 + +// revert to original SDK once this PR is merged and released : https://github.com/XSAM/otelsql/pull/199 +replace github.com/XSAM/otelsql v0.26.0 => github.com/rilldata/otelsql v0.0.0-20231213102114-31132d178f10 diff --git a/go.sum b/go.sum index bc36df33e64..c2ebbf3131f 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -172,8 +172,8 @@ cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvj cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -198,8 +198,8 @@ cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOX cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= -cloud.google.com/go/datacatalog v1.18.1 h1:xJp9mZrc2HPaoxIz3sP9pCmf/impifweQ/yGG9VBfio= -cloud.google.com/go/datacatalog v1.18.1/go.mod h1:TzAWaz+ON1tkNr4MOcak8EBHX7wIRX/gZKM+yTVsv+A= +cloud.google.com/go/datacatalog v1.19.0 h1:rbYNmHwvAOOwnW2FPXYkaK3Mf1MmGqRzK0mMiIEyLdo= +cloud.google.com/go/datacatalog v1.19.0/go.mod h1:5FR6ZIF8RZrtml0VUao22FxhdjkoG+a0866rEnObryM= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= @@ -312,8 +312,8 @@ cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGE cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= @@ -344,8 +344,8 @@ cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeN cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= -cloud.google.com/go/longrunning v0.5.2 h1:u+oFqfEwwU7F9dIELigxbe0XVnBAo9wqMuQLA50CZ5k= -cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs= +cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= @@ -699,8 +699,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/XSAM/otelsql v0.23.0 h1:NsJQS9YhI1+RDsFqE9mW5XIQmPmdF/qa8qQOLZN8XEA= -github.com/XSAM/otelsql v0.23.0/go.mod h1:oX4LXMsb+9lAZhvHjUS61oQP/hbcJRadWHnBKNL+LuM= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= @@ -1106,8 +1104,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -1168,8 +1166,9 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= @@ -1404,8 +1403,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -1665,8 +1664,8 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mazznoer/csscolorparser v0.1.3 h1:vug4zh6loQxAUxfU1DZEu70gTPufDPspamZlHAkKcxE= github.com/mazznoer/csscolorparser v0.1.3/go.mod h1:Aj22+L/rYN/Y6bj3bYqO3N6g1dtdHtGfQ32xZ5PJQic= @@ -1840,15 +1839,15 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1857,8 +1856,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1871,8 +1870,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.10.0 h1:UkG7GPYkO4UZyLnyXjaWYcgOSONqwdBqFUT95ugmt6I= -github.com/prometheus/procfs v0.10.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= @@ -1888,6 +1887,8 @@ github.com/rilldata/calcite-avatica-go/v5 v5.0.0-20231129075938-4645f00ea1f7 h1: github.com/rilldata/calcite-avatica-go/v5 v5.0.0-20231129075938-4645f00ea1f7/go.mod h1:+YWcYNcfM/D8qp3ykLq8Z0a9LjVZy3RW1d3+8Xbn4Q0= github.com/rilldata/go-duckdb v0.0.0-20231208152512-5e40f2025fc4 h1:gmcQnIR19x/RfPXqKys1f+jVP1G0sfReruQUW2Pip2g= github.com/rilldata/go-duckdb v0.0.0-20231208152512-5e40f2025fc4/go.mod h1:wm91jO2GNKa6iO9NTcjXIRsW+/ykPoJbQcHSXhdAl28= +github.com/rilldata/otelsql v0.0.0-20231213102114-31132d178f10 h1:4RQrqby7C+nuMGIeaNRIYuvqE3uBKVQz9GL814mlQNA= +github.com/rilldata/otelsql v0.0.0-20231213102114-31132d178f10/go.mod h1:wb5kU0k295/9LVCg9zsAtBfN1pAQCZJaWVgYfHhyc9w= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -1899,8 +1900,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -2081,55 +2082,51 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= -go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 h1:EbmAUG9hEAMXyfWEasIt2kmh/WmXUznUksChApTgBGc= -go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0/go.mod h1:rD9feqRYP24P14t5kmhNMqsqm1jvKmpx2H2rKVw52V8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1 h1:m9ReioVPIffxjJlGNRd0d5poy+9oTro3D+YbiEzUDOc= +go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1/go.mod h1:CANkrsXNzqOKXfOomu2zhOmc1/J5UZK9SGjrat6ZCG0= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 h1:rm+Fizi7lTM2UefJ1TO347fSRcwmIsUAaZmYmIGBRAo= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0/go.mod h1:sWFbI3jJ+6JdjOVepA5blpv/TJ20Hw+26561iMbWcwU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= -go.opentelemetry.io/otel/exporters/prometheus v0.39.0 h1:whAaiHxOatgtKd+w0dOi//1KUxj3KoPINZdtDaDj3IA= -go.opentelemetry.io/otel/exporters/prometheus v0.39.0/go.mod h1:4jo5Q4CROlCpSPsXLhymi+LYrDXd2ObU5wbKayfZs7Y= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= -go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= +go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= +go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -2139,8 +2136,8 @@ go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -2193,8 +2190,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2337,8 +2334,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2530,8 +2527,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2545,8 +2542,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2749,8 +2746,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -2887,12 +2885,12 @@ google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f7 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 h1:s1w3X6gQxwrLEpxnLd/qXTVLgQE2yXwaOaoa6IlY/+o= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0/go.mod h1:CAny0tYF+0/9rmDB9fahA9YLzX3+AEVl1qXbv5hhj6c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -2937,8 +2935,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/runtime/client/client.go b/runtime/client/client.go index 668abd140c8..67f09ac6c5f 100644 --- a/runtime/client/client.go +++ b/runtime/client/client.go @@ -27,8 +27,7 @@ func New(runtimeHost, bearerToken string) (*Client, error) { } opts := []grpc.DialOption{ - grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()), - grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()), + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), } if uri.Scheme == "http" { diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 3b0d3e44a70..f2a3737613e 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -2,7 +2,6 @@ package duckdb import ( "context" - databasesql "database/sql" "database/sql/driver" "errors" "fmt" @@ -318,12 +317,6 @@ func (c *connection) Config() map[string]any { // Close implements drivers.Connection. func (c *connection) Close() error { - // Gracefully detach all databases (otherwise DuckDB may leak memory) - _ = c.WithConnection(context.Background(), 9000, true, false, func(ctx, ensuredCtx context.Context, conn *databasesql.Conn) error { - c.detachAllDBs(ctx, conn) - return nil - }) - c.cancel() _ = c.registration.Unregister() return c.db.Close() @@ -404,14 +397,7 @@ func (c *connection) AsFileStore() (drivers.FileStore, bool) { func (c *connection) reopenDB() error { // If c.db is already open, close it first if c.db != nil { - // Try to detach all attached DBs (otherwise DuckDB leaks memory) - conn, err := c.db.Conn(context.Background()) - if err == nil { // If it's so broken we can't get a conn, we'll just risk a leak - c.detachAllDBs(context.Background(), conn) - conn.Close() - } - - err = c.db.Close() + err := c.db.Close() if err != nil { return err } @@ -835,38 +821,6 @@ func (c *connection) periodicallyCheckConnDurations(d time.Duration) { } } -// detachAllDBs detaches all attached dbs if external_table_storage config is true -func (c *connection) detachAllDBs(ctx context.Context, conn *databasesql.Conn) { - if !c.config.ExtTableStorage { - return - } - - entries, err := os.ReadDir(c.config.ExtStoragePath) - if err != nil { - c.logger.Error("unable to read ExtStoragePath", zap.String("path", c.config.ExtStoragePath), zap.Error(err)) - return - } - - for _, entry := range entries { - if !entry.IsDir() { - continue - } - version, exist, err := c.tableVersion(entry.Name()) - if err != nil { - continue - } - if !exist { - continue - } - - db := dbName(entry.Name(), version) - _, err = conn.ExecContext(ctx, fmt.Sprintf("DETACH %s", safeSQLName(db))) - if err != nil { - c.logger.Error("detach failed", zap.String("db", db), zap.Error(err)) - } - } -} - func (c *connection) logLimits(conn *sqlx.Conn) { row := conn.QueryRowContext(context.Background(), "SELECT value FROM duckdb_settings() WHERE name='max_memory'") var memory string diff --git a/runtime/drivers/duckdb/duckdb_test.go b/runtime/drivers/duckdb/duckdb_test.go index 5808847e5e0..7e963ea93ed 100644 --- a/runtime/drivers/duckdb/duckdb_test.go +++ b/runtime/drivers/duckdb/duckdb_test.go @@ -31,7 +31,6 @@ func TestOpenDrop(t *testing.T) { err = handle.Close() require.NoError(t, err) require.FileExists(t, path) - require.FileExists(t, walpath) err = Driver{}.Drop(map[string]any{"dsn": dsn}, zap.NewNop()) require.NoError(t, err) diff --git a/runtime/pkg/observability/middleware.go b/runtime/pkg/observability/middleware.go index f801461d8dc..d8781daee91 100644 --- a/runtime/pkg/observability/middleware.go +++ b/runtime/pkg/observability/middleware.go @@ -10,7 +10,6 @@ import ( grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/rilldata/rill/runtime/pkg/activity" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -28,16 +27,6 @@ func Middleware(serviceName string, logger *zap.Logger, next http.Handler) http. return TracingMiddleware(LoggingMiddleware(logger, next), serviceName) } -// TracingUnaryServerInterceptor is a gRPC unary interceptor that adds tracing to the request. -func TracingUnaryServerInterceptor() grpc.UnaryServerInterceptor { - return otelgrpc.UnaryServerInterceptor() -} - -// TracingStreamServerInterceptor is the streaming equivalent of TracingUnaryServerInterceptor -func TracingStreamServerInterceptor() grpc.StreamServerInterceptor { - return otelgrpc.StreamServerInterceptor() -} - // TracingMiddleware is HTTP middleware that adds tracing to the request. func TracingMiddleware(next http.Handler, serviceName string) http.Handler { return otelhttp.NewHandler(next, serviceName) diff --git a/runtime/pkg/observability/observability.go b/runtime/pkg/observability/observability.go index ddae51f9ca0..baa049ae5ac 100644 --- a/runtime/pkg/observability/observability.go +++ b/runtime/pkg/observability/observability.go @@ -15,7 +15,7 @@ import ( "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" "go.uber.org/zap" ) diff --git a/runtime/server/auth/interceptors.go b/runtime/server/auth/interceptors.go index 12a04a63ce8..65bb79a5b9d 100644 --- a/runtime/server/auth/interceptors.go +++ b/runtime/server/auth/interceptors.go @@ -10,7 +10,7 @@ import ( grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/runtime/server/server.go b/runtime/server/server.go index 00716b23c1f..d51c0fc8a6a 100644 --- a/runtime/server/server.go +++ b/runtime/server/server.go @@ -22,6 +22,7 @@ import ( "github.com/rilldata/rill/runtime/pkg/securetoken" "github.com/rilldata/rill/runtime/server/auth" "github.com/rs/cors" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" "go.uber.org/zap" "google.golang.org/grpc" @@ -125,7 +126,6 @@ func (s *Server) ServeGRPC(ctx context.Context) error { server := grpc.NewServer( grpc.ChainStreamInterceptor( middleware.TimeoutStreamServerInterceptor(timeoutSelector), - observability.TracingStreamServerInterceptor(), observability.LoggingStreamServerInterceptor(s.logger), grpc_validator.StreamServerInterceptor(), auth.StreamServerInterceptor(s.aud), @@ -135,7 +135,6 @@ func (s *Server) ServeGRPC(ctx context.Context) error { ), grpc.ChainUnaryInterceptor( middleware.TimeoutUnaryServerInterceptor(timeoutSelector), - observability.TracingUnaryServerInterceptor(), observability.LoggingUnaryServerInterceptor(s.logger), grpc_validator.UnaryServerInterceptor(), auth.UnaryServerInterceptor(s.aud), @@ -143,6 +142,7 @@ func (s *Server) ServeGRPC(ctx context.Context) error { errorMappingUnaryServerInterceptor(), grpc_auth.UnaryServerInterceptor(s.checkRateLimit), ), + grpc.StatsHandler(otelgrpc.NewServerHandler()), ) runtimev1.RegisterRuntimeServiceServer(server, s) From 262459aa42a9d2c692804afbe9916a62dc99f321 Mon Sep 17 00:00:00 2001 From: Dhiraj Barnwal Date: Thu, 14 Dec 2023 05:57:03 +0530 Subject: [PATCH 02/19] Feat: Add dimension label and percent of total to charts (#3672) * Show labels instantly for measure charts * Add percTotal to dimension charts * Show labels only for currently hovered chart * Clean up * Prettier fix * Fix spacing, add types * Show perc in label only when context column is percent * Remove unused code --- .../data-graphic/marks/DelayedLabel.svelte | 4 +- .../marks/MultiMetricMouseoverLabel.svelte | 22 ++++++-- .../state-managers/selectors/measures.ts | 15 ++++++ .../DimensionValueMouseover.svelte | 53 +++++++++++++------ .../time-series/MeasureChart.svelte | 8 +-- .../MetricsTimeSeriesCharts.svelte | 15 +++++- 6 files changed, 92 insertions(+), 25 deletions(-) diff --git a/web-common/src/components/data-graphic/marks/DelayedLabel.svelte b/web-common/src/components/data-graphic/marks/DelayedLabel.svelte index f37e5e1e8dc..b9f78b01a6d 100644 --- a/web-common/src/components/data-graphic/marks/DelayedLabel.svelte +++ b/web-common/src/components/data-graphic/marks/DelayedLabel.svelte @@ -36,4 +36,6 @@ }); - + diff --git a/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte b/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte index 1bd661b0aaf..842998e26d6 100644 --- a/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte +++ b/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte @@ -15,6 +15,7 @@ It is probably not the most up to date code; but it works very well in practice. interface Point { x: number; y: number; + value: string; label: string; key: string; valueColorClass?: string; @@ -57,7 +58,8 @@ It is probably not the most up to date code; but it works very well in practice. let containerWidths = []; // let labelWidth = 0; - $: fanOutLabels = !isDimension || false; + let fanOutLabels = true; + // update locations. $: nonOverlappingLocations = preventVerticalOverlap( point.map((p) => ({ @@ -135,6 +137,7 @@ It is probably not the most up to date code; but it works very well in practice. } let labelWidth = 0; + /** the full text width */ let transitionalTimeoutForCalculatingLabelWidth; $: if (container && locations && $xScale && $yScale) { @@ -206,12 +209,15 @@ It is probably not the most up to date code; but it works very well in practice. {visibility} > {#if !location?.yOverride} - {formatValue(location.y)} + {location.value + ? location.value + : formatValue(location.y)} {/if} {#if !location?.yOverride} - {formatValue(location.y)} + {location.value + ? location.value + : formatValue(location.y)} {/if} {/if} @@ -293,5 +302,12 @@ It is probably not the most up to date code; but it works very well in practice. paint-order: stroke; stroke: white; stroke-width: 3px; + white-space: pre-wrap; + /* Make all characters and numbers of equal width for easy scanibility */ + font-feature-settings: "case" 0, "cpsp" 0, "dlig" 0, "frac" 0, "dnom" 0, + "numr" 0, "salt" 0, "subs" 0, "sups" 0, "tnum", "zero" 0, "ss01", "ss02" 0, + "ss03" 0, "ss04" 0, "cv01" 0, "cv02" 0, "cv03" 0, "cv04" 0, "cv05" 0, + "cv06" 0, "cv07" 0, "cv08" 0, "cv09" 0, "cv10" 0, "cv11" 0, "calt", "ccmp", + "kern"; } diff --git a/web-common/src/features/dashboards/state-managers/selectors/measures.ts b/web-common/src/features/dashboards/state-managers/selectors/measures.ts index 864cc76b166..37d158e0704 100644 --- a/web-common/src/features/dashboards/state-managers/selectors/measures.ts +++ b/web-common/src/features/dashboards/state-managers/selectors/measures.ts @@ -18,9 +18,24 @@ export const visibleMeasures = ({ return measures === undefined ? [] : measures; }; +export const isMeasureValidPercentOfTotal = ({ + metricsSpecQueryResult, +}: DashboardDataSources): ((measureName: string) => boolean) => { + return (measureName: string) => { + const measures = metricsSpecQueryResult.data?.measures; + const selectedMeasure = measures?.find((m) => m.name === measureName); + return selectedMeasure?.validPercentOfTotal ?? false; + }; +}; + export const measureSelectors = { /** * Gets all visible measures in the dashboard. */ visibleMeasures, + + /** + * Checks if the provided measure is a valid percent of total + */ + isMeasureValidPercentOfTotal, }; diff --git a/web-common/src/features/dashboards/time-series/DimensionValueMouseover.svelte b/web-common/src/features/dashboards/time-series/DimensionValueMouseover.svelte index 52f277bcb01..13eccdbb61f 100644 --- a/web-common/src/features/dashboards/time-series/DimensionValueMouseover.svelte +++ b/web-common/src/features/dashboards/time-series/DimensionValueMouseover.svelte @@ -1,16 +1,32 @@ -
-
-
-
-
-
+ + + From 0d864682b461c860b9460870d4e26c6f58f2e9aa Mon Sep 17 00:00:00 2001 From: Brian Holmes <120223836+briangregoryholmes@users.noreply.github.com> Date: Thu, 14 Dec 2023 05:32:07 -0500 Subject: [PATCH 07/19] removed WithTween wrapper around clipPath to prevent fickering when charts are updated (#3692) --- .../data-graphic/marks/ChunkedLine.svelte | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/web-common/src/components/data-graphic/marks/ChunkedLine.svelte b/web-common/src/components/data-graphic/marks/ChunkedLine.svelte index 139f52e1cfd..6b0a186291c 100644 --- a/web-common/src/components/data-graphic/marks/ChunkedLine.svelte +++ b/web-common/src/components/data-graphic/marks/ChunkedLine.svelte @@ -170,28 +170,7 @@ Over time, we'll make this the default Line implementation, but it's not quite t {@const x = $xScale(segment[0][xAccessor])} {@const width = $xScale(segment.at(-1)[xAccessor]) - $xScale(segment[0][xAccessor])} - - - + {/each} From 7537c72eae695f8706f9bee81630e20a636b51a1 Mon Sep 17 00:00:00 2001 From: Alexander Thor Date: Thu, 14 Dec 2023 13:06:44 +0100 Subject: [PATCH 08/19] doc: hide content class (#3699) --- docs/README.md | 5 +++++ docs/src/css/custom.css | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/docs/README.md b/docs/README.md index 49f65beae2e..80666dfae8a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -40,3 +40,8 @@ The CLI reference docs in `docs/reference/cli` are auto-generated based on the C ```bash make docs.generate ``` + +## Hiding content + +Sometimes you want to merge documentation but hide it pending a future release, there are a few different short-hands available. +For individual pages add `sidebar_class_name: hidden` to the top of the page and to hide an entire category add `className: hidden` to the `_category_.yml` file. Note that the pages are only hidden but still generated so you can still link to them. \ No newline at end of file diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index aa59398cea0..9bb138aee47 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -67,4 +67,8 @@ border-color: var(--ifm-toc-border-color); border-radius: var(--ifm-global-radius); padding: 0px 20px 20px 20px; +} + +.hidden { + display: none !important; } \ No newline at end of file From 8bb50628f6fd4bedc9162ee5dfaba621ebf5a043 Mon Sep 17 00:00:00 2001 From: Janet <30886488+jkhwu@users.noreply.github.com> Date: Thu, 14 Dec 2023 08:49:04 -0800 Subject: [PATCH 09/19] Update themes.md (#3703) Added an example code block to copy/paste and additional instructions on how to apply the theme to a dashboard. --- docs/docs/reference/project-files/themes.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/docs/reference/project-files/themes.md b/docs/docs/reference/project-files/themes.md index c719f90424d..a04571c775d 100644 --- a/docs/docs/reference/project-files/themes.md +++ b/docs/docs/reference/project-files/themes.md @@ -6,6 +6,8 @@ sidebar_position: 41 In your Rill project directory, create a `.yaml` file in any directory containing `kind: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. +To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` + ## Properties _**`kind`**_ - is mandatory and must be `theme`. _(required)_ @@ -13,3 +15,12 @@ _**`kind`**_ - is mandatory and must be `theme`. _(required)_ _**`colors`**_ - used to override the dashboard colors. - _**`primary`**_ - overrides the primary blue color in the dashboard. Can have any hex (without the # character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - _**`secondary`**_ - overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the # character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + +## Example +You can copy this directly into your .yaml file: +```yaml +kind: theme +colors: + primary: crimson + secondary: lime +``` From 135177d4e7fa6ecac79bac7c02ab54dbe0343ee2 Mon Sep 17 00:00:00 2001 From: Brian Holmes <120223836+briangregoryholmes@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:25:48 -0500 Subject: [PATCH 10/19] Remove artificial time series chart delay (#3696) * update timeseries-data-store to return data as long as the primary or primaryTotal data is available (rather than both) * assign totals query a higher priority --- .../dashboards/time-series/timeseries-data-store.ts | 12 +----------- .../runtime-client/http-request-queue/priorities.ts | 1 + 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/web-common/src/features/dashboards/time-series/timeseries-data-store.ts b/web-common/src/features/dashboards/time-series/timeseries-data-store.ts index 69386df2be8..d65d45275ff 100644 --- a/web-common/src/features/dashboards/time-series/timeseries-data-store.ts +++ b/web-common/src/features/dashboards/time-series/timeseries-data-store.ts @@ -159,16 +159,6 @@ export function createTimeSeriesDataStore(ctx: StateManagers) { ]) => { let timeSeriesData = primary?.data?.data; - if (!primary?.data || !primaryTotal?.data || !unfilteredTotal?.data) { - return { - isFetching: - metricsView.isFetching || - primary?.isFetching || - primaryTotal?.isFetching || - unfilteredTotal?.isFetching, - }; - } - if (!primary.isFetching && interval) { timeSeriesData = prepareTimeSeries( primary?.data?.data, @@ -178,7 +168,7 @@ export function createTimeSeriesDataStore(ctx: StateManagers) { ); } return { - isFetching: primaryTotal?.isFetching || metricsView?.isFetching, + isFetching: !primary?.data && !primaryTotal?.data, isError: false, // FIXME Handle errors timeSeriesData, total: primaryTotal?.data?.data?.[0], diff --git a/web-common/src/runtime-client/http-request-queue/priorities.ts b/web-common/src/runtime-client/http-request-queue/priorities.ts index 9f9b76730ee..d2f1d11c8b7 100644 --- a/web-common/src/runtime-client/http-request-queue/priorities.ts +++ b/web-common/src/runtime-client/http-request-queue/priorities.ts @@ -16,6 +16,7 @@ export const QueryPriorities = { topk: 10, "rug-histogram": 10, "descriptive-statistics": 10, + totals: 30, }; export function getPriority(type: string): number { From f8899cd153679c24320d5d7883cb7471606bd8e6 Mon Sep 17 00:00:00 2001 From: bcolloran Date: Thu, 14 Dec 2023 11:03:19 -0800 Subject: [PATCH 11/19] svletecheck --ignore cleanup: fix most of "time-controls" folder (#3651) * fix most of "time-controls" folder * change default return value to `false` --- .github/workflows/web-test.yml | 2 +- .../time-controls/ComparisonSelector.svelte | 4 +++- .../time-controls/CustomTimeRangeInput.svelte | 4 ++-- .../time-controls/NoTimeDimensionCTA.svelte | 8 ++++++- .../time-controls/TimeControls.svelte | 6 ++--- .../time-controls/TimeGrainSelector.svelte | 3 ++- .../resource-status-utils.ts | 22 ++++++++++++++----- 7 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/web-test.yml b/.github/workflows/web-test.yml index f94e867533c..34fe6fbf214 100644 --- a/.github/workflows/web-test.yml +++ b/.github/workflows/web-test.yml @@ -75,7 +75,7 @@ jobs: run: |- npx prettier --check "web-common/**/*" npx eslint web-common --quiet - npx svelte-check --threshold error --workspace web-common --no-tsconfig --ignore "src/components/data-graphic,src/features/dashboards/(time-series|time-controls)" + npx svelte-check --threshold error --workspace web-common --no-tsconfig --ignore "src/components/data-graphic,src/features/dashboards/time-series,src/features/dashboards/time-controls/TimeRangeSelector.svelte,src/features/dashboards/time-controls/TimeControls.svelte" - name: Prettier checks and lint for web local if: steps.filter.outputs.local == 'true' diff --git a/web-common/src/features/dashboards/time-controls/ComparisonSelector.svelte b/web-common/src/features/dashboards/time-controls/ComparisonSelector.svelte index 7d66edff846..b7e563a771c 100644 --- a/web-common/src/features/dashboards/time-controls/ComparisonSelector.svelte +++ b/web-common/src/features/dashboards/time-controls/ComparisonSelector.svelte @@ -189,7 +189,9 @@ This component needs to do the following: { - intermediateSelection = option.name; + if (option.name) { + intermediateSelection = option.name; + } }} on:select={() => { enableComparison("dimension", option.name); diff --git a/web-common/src/features/dashboards/time-controls/CustomTimeRangeInput.svelte b/web-common/src/features/dashboards/time-controls/CustomTimeRangeInput.svelte index 5a163c5b447..ae97db88be6 100644 --- a/web-common/src/features/dashboards/time-controls/CustomTimeRangeInput.svelte +++ b/web-common/src/features/dashboards/time-controls/CustomTimeRangeInput.svelte @@ -64,7 +64,7 @@ start: Date, end: Date, minTimeGrain: V1TimeGrain - ): string { + ): string | undefined { const allowedTimeGrains = getAllowedTimeGrains(start, end); const allowedMaxGrain = allowedTimeGrains[allowedTimeGrains.length - 1]; @@ -80,7 +80,7 @@ } // HAM, you left off here. - let error = undefined; + let error: string | undefined = undefined; $: if (start && end) { error = validateTimeRange( parseLocaleStringDate(start), diff --git a/web-common/src/features/dashboards/time-controls/NoTimeDimensionCTA.svelte b/web-common/src/features/dashboards/time-controls/NoTimeDimensionCTA.svelte index a640417ce86..2780ee01585 100644 --- a/web-common/src/features/dashboards/time-controls/NoTimeDimensionCTA.svelte +++ b/web-common/src/features/dashboards/time-controls/NoTimeDimensionCTA.svelte @@ -17,7 +17,13 @@ $runtime.instanceId, modelName ); - $: timestampColumns = $timestampColumnsQuery?.data; + + // CAST SAFETY: must be string[], since we filter out undefined values + $: timestampColumns = + ($timestampColumnsQuery?.data?.filter( + (x) => x !== undefined + ) as string[]) ?? []; + $: isReadOnlyDashboard = $featureFlags.readOnly === true; $: redirectToScreen = timestampColumns?.length > 0 ? "metrics" : "model"; diff --git a/web-common/src/features/dashboards/time-controls/TimeControls.svelte b/web-common/src/features/dashboards/time-controls/TimeControls.svelte index f7f3bdd8bf2..4702e8a915a 100644 --- a/web-common/src/features/dashboards/time-controls/TimeControls.svelte +++ b/web-common/src/features/dashboards/time-controls/TimeControls.svelte @@ -40,8 +40,8 @@ const queryClient = useQueryClient(); $: dashboardStore = useDashboardStore(metricViewName); - let baseTimeRange: TimeRange; - let minTimeGrain: V1TimeGrain; + let baseTimeRange: TimeRange | undefined; + let minTimeGrain: V1TimeGrain | undefined; let availableTimeZones: string[] = []; $: metaQuery = useMetaQuery($runtime.instanceId, metricViewName); @@ -61,7 +61,7 @@ !!$metaQuery?.data?.table && !!$metaQuery?.data?.timeDimension ) { - availableTimeZones = $metaQuery?.data?.availableTimeZones; + availableTimeZones = $metaQuery?.data?.availableTimeZones ?? []; /** * Remove the timezone selector if no timezone key is present diff --git a/web-common/src/features/dashboards/time-controls/TimeGrainSelector.svelte b/web-common/src/features/dashboards/time-controls/TimeGrainSelector.svelte index 79ebc7ac4df..c4767028580 100644 --- a/web-common/src/features/dashboards/time-controls/TimeGrainSelector.svelte +++ b/web-common/src/features/dashboards/time-controls/TimeGrainSelector.svelte @@ -18,7 +18,8 @@ const timeControlsStore = useTimeControlStore(getStateManagers()); $: activeTimeGrain = $timeControlsStore.selectedTimeRange?.interval; - $: activeTimeGrainLabel = TIME_GRAIN[activeTimeGrain]?.label; + $: activeTimeGrainLabel = + activeTimeGrain && TIME_GRAIN[activeTimeGrain]?.label; $: timeGrains = timeGrainOptions ? timeGrainOptions diff --git a/web-common/src/features/entity-management/resource-status-utils.ts b/web-common/src/features/entity-management/resource-status-utils.ts index b1d67244b59..50a3c3bb23f 100644 --- a/web-common/src/features/entity-management/resource-status-utils.ts +++ b/web-common/src/features/entity-management/resource-status-utils.ts @@ -44,9 +44,9 @@ export function initialResourceStatusStore( ([resourceName, projectParserResp]) => { if ( !projectParserResp.data || - projectParserResp.data.projectParser.state.parseErrors.filter( + (projectParserResp?.data?.projectParser?.state?.parseErrors?.filter( (s) => s.filePath === filePath - ).length > 0 + ).length ?? 0) > 0 ) { return ResourceStatus.Errored; } @@ -106,12 +106,17 @@ export function resourceStatusStore( ) return { status: ResourceStatus.Busy }; + const changed = + !lastUpdatedOn || + (res.data?.meta?.stateUpdatedOn !== undefined + ? res.data?.meta?.stateUpdatedOn > lastUpdatedOn + : false); + return { status: !res.data?.meta?.reconcileError ? ResourceStatus.Idle : ResourceStatus.Errored, - changed: - !lastUpdatedOn || res.data?.meta?.stateUpdatedOn > lastUpdatedOn, + changed, }; } ); @@ -144,13 +149,18 @@ export function waitForResourceUpdate( if (status.status === ResourceStatus.Busy) return; if (timer) clearTimeout(timer); + const do_end = + status.status === ResourceStatus.Idle && + status.changed !== undefined && + status.changed; + if (idled) { - end(status.status === ResourceStatus.Idle && status.changed); + end(do_end); return; } else { idled = true; timer = setTimeout(() => { - end(status.status === ResourceStatus.Idle && status.changed); + end(do_end); }, 500); } }); From 567993e2769f6ff6414ae788589eb6afc1fbefd3 Mon Sep 17 00:00:00 2001 From: Brian Holmes <120223836+briangregoryholmes@users.noreply.github.com> Date: Thu, 14 Dec 2023 18:59:10 -0500 Subject: [PATCH 12/19] suggest models in the code editor (#3705) * adds autocomplate for models in the code editor * remove unused type import * change schema initialization, rename variable in useAllModelColumns --- web-common/src/features/models/selectors.ts | 26 +++++++++++++++++++ .../features/models/workspace/Editor.svelte | 14 ++++++++-- web-common/src/features/sources/selectors.ts | 4 +-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/models/selectors.ts b/web-common/src/features/models/selectors.ts index 358083ae9e6..a8c27a8f7f5 100644 --- a/web-common/src/features/models/selectors.ts +++ b/web-common/src/features/models/selectors.ts @@ -14,6 +14,12 @@ import { } from "@rilldata/web-common/runtime-client"; import type { QueryClient } from "@tanstack/query-core"; import { TIMESTAMPS } from "../../lib/duckdb-data-types"; +import type { Readable } from "svelte/motion"; +import { derived } from "svelte/store"; +import { + createTableColumnsWithName, + type TableColumnsWithName, +} from "../sources/selectors"; export function useModels(instanceId: string) { return useFilteredResources(instanceId, ResourceKind.Model); @@ -31,6 +37,26 @@ export function useModel(instanceId: string, name: string) { return useResource(instanceId, name, ResourceKind.Model); } +export function useAllModelColumns( + queryClient: QueryClient, + instanceId: string +): Readable> { + return derived([useModels(instanceId)], ([allModels], set) => { + if (!allModels.data?.length) { + set([]); + return; + } + + derived( + allModels.data.map((r) => + createTableColumnsWithName(queryClient, instanceId, r.meta.name.name) + ), + (modelColumnResponses) => + modelColumnResponses.filter((res) => !!res.data).map((res) => res.data) + ).subscribe(set); + }); +} + export async function getModelNames( queryClient: QueryClient, instanceId: string diff --git a/web-common/src/features/models/workspace/Editor.svelte b/web-common/src/features/models/workspace/Editor.svelte index 08383d006fe..2b9affc21f5 100644 --- a/web-common/src/features/models/workspace/Editor.svelte +++ b/web-common/src/features/models/workspace/Editor.svelte @@ -55,6 +55,7 @@ import { editorTheme } from "../../../components/editor/theme"; import { runtime } from "../../../runtime-client/runtime-store"; import { useAllSourceColumns } from "../../sources/selectors"; + import { useAllModelColumns } from "../selectors"; export let content: string; export let editorHeight = 0; @@ -89,11 +90,11 @@ "select from where group by all having order limit sample unnest with window qualify values filter exclude replace like ilike glob as case when then else end in cast left join on not desc asc sum union", }); + const schema: { [table: string]: string[] } = {}; + // Autocomplete: source tables - let schema: { [table: string]: string[] }; $: allSourceColumns = useAllSourceColumns(queryClient, $runtime?.instanceId); $: if ($allSourceColumns?.length) { - schema = {}; for (const sourceTable of $allSourceColumns) { const sourceIdentifier = sourceTable?.tableName; schema[sourceIdentifier] = @@ -101,6 +102,15 @@ } } + //Auto complete: model tables + $: allModelColumns = useAllModelColumns(queryClient, $runtime?.instanceId); + $: if ($allModelColumns?.length) { + for (const modelTable of $allModelColumns) { + const modelIdentifier = modelTable?.tableName; + schema[modelIdentifier] = modelTable.profileColumns?.map((c) => c.name); + } + } + function getTableNameFromFromClause( sql: string, schema: { [table: string]: string[] } diff --git a/web-common/src/features/sources/selectors.ts b/web-common/src/features/sources/selectors.ts index d2188bc628a..57029976a6f 100644 --- a/web-common/src/features/sources/selectors.ts +++ b/web-common/src/features/sources/selectors.ts @@ -88,7 +88,7 @@ export function useIsLocalFileConnector( ); } -type TableColumnsWithName = { +export type TableColumnsWithName = { tableName: string; profileColumns: Array; }; @@ -116,7 +116,7 @@ export function useAllSourceColumns( /** * Fetches columns and adds the table name. By using the selector the results will be cached. */ -function createTableColumnsWithName( +export function createTableColumnsWithName( queryClient: QueryClient, instanceId: string, tableName: string From b16aca4a684f954aa0369f2ca6e1ed3fd83db6e4 Mon Sep 17 00:00:00 2001 From: Brian Holmes <120223836+briangregoryholmes@users.noreply.github.com> Date: Fri, 15 Dec 2023 01:13:41 -0500 Subject: [PATCH 13/19] properly extend tailwind colors with custom theme overwrite (#3704) --- web-common/tailwind.config.cjs | 45 +++++++++++++--------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/web-common/tailwind.config.cjs b/web-common/tailwind.config.cjs index e743d67c8a6..c2200d85d0c 100644 --- a/web-common/tailwind.config.cjs +++ b/web-common/tailwind.config.cjs @@ -1,16 +1,4 @@ -const colors = require("tailwindcss/colors"); -const deprecatedColors = { - lightBlue: true, - warmGray: true, - trueGray: true, - coolGray: true, - blueGray: true -} -const newColors = {}; -for (const c in colors) { - if (c in deprecatedColors) continue; - newColors[c] = colors[c] -} +/** @type {import('tailwindcss').Config} */ module.exports = { // need to add this for storybook @@ -22,21 +10,22 @@ module.exports = { /** Once we have applied dark styling to all UI elements, remove this line */ darkMode: "class", theme: { - colors: { - ...newColors, - blue: { - 50: "var(--color-primary-50)", - 100: "var(--color-primary-100)", - 200: "var(--color-primary-200)", - 300: "var(--color-primary-300)", - 400: "var(--color-primary-400)", - 500: "var(--color-primary-500)", - 600: "var(--color-primary-600)", - 700: "var(--color-primary-700)", - 800: "var(--color-primary-800)", - 900: "var(--color-primary-900)", - 950: "var(--color-primary-950)", - }, + extend: { + colors: { + blue: { + 50: "var(--color-primary-50)", + 100: "var(--color-primary-100)", + 200: "var(--color-primary-200)", + 300: "var(--color-primary-300)", + 400: "var(--color-primary-400)", + 500: "var(--color-primary-500)", + 600: "var(--color-primary-600)", + 700: "var(--color-primary-700)", + 800: "var(--color-primary-800)", + 900: "var(--color-primary-900)", + 950: "var(--color-primary-950)", + }, + } }, }, plugins: [], From fa2e166cf70a1fba1e7b0c352e7301626b9b7e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= Date: Fri, 15 Dec 2023 12:29:03 +0100 Subject: [PATCH 14/19] Runtime: Refactor conn cache to contain and detect hanging opens/closes (#3666) * Runtime: Refactor conn cache to contain and detect hanging opens/closes * Extract connection cache to pkg + use a singleflight * Add tests * Make tests pass * increase test sleeps for clsoe * Fix various race conditions * Integrate singleflight with conncache's mutex * Increase timeouts * Better comments * Prevent deadlock when closing while opening * Address review comments * Remove redundant var --- runtime/connection_cache.go | 344 ++++-------------- runtime/connection_cache_test.go | 417 ---------------------- runtime/connections.go | 4 +- runtime/drivers/duckdb/duckdb.go | 2 +- runtime/pkg/conncache/conncache.go | 450 ++++++++++++++++++++++++ runtime/pkg/conncache/conncache_test.go | 268 ++++++++++++++ runtime/registry.go | 2 +- runtime/registry_test.go | 9 +- runtime/runtime.go | 7 +- 9 files changed, 806 insertions(+), 697 deletions(-) delete mode 100644 runtime/connection_cache_test.go create mode 100644 runtime/pkg/conncache/conncache.go create mode 100644 runtime/pkg/conncache/conncache_test.go diff --git a/runtime/connection_cache.go b/runtime/connection_cache.go index 882150d4ae3..e926b6db2e8 100644 --- a/runtime/connection_cache.go +++ b/runtime/connection_cache.go @@ -6,246 +6,100 @@ import ( "fmt" "slices" "strings" - "sync" "time" - "github.com/hashicorp/golang-lru/simplelru" "github.com/rilldata/rill/runtime/drivers" - "github.com/rilldata/rill/runtime/pkg/activity" + "github.com/rilldata/rill/runtime/pkg/conncache" "github.com/rilldata/rill/runtime/pkg/observability" + "go.opentelemetry.io/otel/metric" "go.uber.org/zap" "golang.org/x/exp/maps" ) -var errConnectionCacheClosed = errors.New("connectionCache: closed") - -const migrateTimeout = 2 * time.Minute - -// connectionCache is a thread-safe cache for open connections. -// Connections should preferably be opened only via the connection cache. -// -// TODO: It opens connections async, but it will close them sync when evicted. If a handle's close hangs, this can block the cache. -// We should move the closing to the background. However, it must then handle the case of trying to re-open a connection that's currently closing in the background. -type connectionCache struct { - size int - runtime *Runtime - logger *zap.Logger - activity activity.Client - closed bool - migrateCtx context.Context // ctx used for connection migrations - migrateCtxCancel context.CancelFunc // cancel all running migrations - lock sync.Mutex - acquired map[string]*connWithRef // items with non-zero references (in use) which should not be evicted - lru *simplelru.LRU // items with no references (opened, but not in use) ready for eviction -} +var ( + connCacheOpens = observability.Must(meter.Int64Counter("connnection_cache.opens")) + connCacheCloses = observability.Must(meter.Int64Counter("connnection_cache.closes")) + connCacheSizeTotal = observability.Must(meter.Int64UpDownCounter("connnection_cache.size_total")) + connCacheSizeLRU = observability.Must(meter.Int64UpDownCounter("connnection_cache.size_lru")) + connCacheOpenLatencyMS = observability.Must(meter.Int64Histogram("connnection_cache.open_latency", metric.WithUnit("ms"))) + connCacheCloseLatencyMS = observability.Must(meter.Int64Histogram("connnection_cache.close_latency", metric.WithUnit("ms"))) +) -type connWithRef struct { +type cachedConnectionConfig struct { instanceID string - handle drivers.Handle - err error - refs int - ready chan struct{} + driver string + shared bool + config map[string]any } -func newConnectionCache(size int, logger *zap.Logger, rt *Runtime, ac activity.Client) *connectionCache { - // LRU cache that closes evicted connections - lru, err := simplelru.NewLRU(size, func(key interface{}, value interface{}) { - // Skip if the conn has refs, since the callback also gets called when transferring to acquired cache - conn := value.(*connWithRef) - if conn.refs != 0 { - return - } - if conn.handle != nil { - if err := conn.handle.Close(); err != nil { - logger.Error("failed closing cached connection", zap.String("key", key.(string)), zap.Error(err)) - } - } +// newConnectionCache returns a concurrency-safe cache for open connections. +// Connections should preferably be opened only via the connection cache. +// It's implementation handles issues such as concurrent open/close/eviction of a connection. +// It also monitors for hanging connections. +func (r *Runtime) newConnectionCache() conncache.Cache { + return conncache.New(conncache.Options{ + MaxIdleConnections: r.opts.ConnectionCacheSize, + OpenTimeout: 10 * time.Minute, + CloseTimeout: 10 * time.Minute, + CheckHangingInterval: time.Minute, + OpenFunc: func(ctx context.Context, cfg any) (conncache.Connection, error) { + x := cfg.(cachedConnectionConfig) + return r.openAndMigrate(ctx, x) + }, + KeyFunc: func(cfg any) string { + x := cfg.(cachedConnectionConfig) + return generateKey(x) + }, + HangingFunc: func(cfg any, open bool) { + x := cfg.(cachedConnectionConfig) + r.logger.Error("connection cache: connection has been working for too long", zap.String("instance_id", x.instanceID), zap.String("driver", x.driver), zap.Bool("open", open)) + }, + Metrics: conncache.Metrics{ + Opens: connCacheOpens, + Closes: connCacheCloses, + SizeTotal: connCacheSizeTotal, + SizeLRU: connCacheSizeLRU, + OpenLatencyMS: connCacheOpenLatencyMS, + CloseLatencyMS: connCacheCloseLatencyMS, + }, }) - if err != nil { - panic(err) - } - - ctx, cancel := context.WithCancel(context.Background()) - return &connectionCache{ - size: size, - runtime: rt, - logger: logger, - activity: ac, - migrateCtx: ctx, - migrateCtxCancel: cancel, - acquired: make(map[string]*connWithRef), - lru: lru, - } -} - -func (c *connectionCache) Close() error { - c.lock.Lock() - defer c.lock.Unlock() - - if c.closed { - return errConnectionCacheClosed - } - c.closed = true - - // Cancel currently running migrations - c.migrateCtxCancel() - - var firstErr error - for _, key := range c.lru.Keys() { - val, ok := c.lru.Get(key) - if !ok { - continue - } - conn := val.(*connWithRef) - if conn.handle == nil { - continue - } - err := conn.handle.Close() - if err != nil { - c.logger.Error("failed closing cached connection", zap.Error(err)) - if firstErr == nil { - firstErr = err - } - } - } - - for _, value := range c.acquired { - if value.handle == nil { - continue - } - err := value.handle.Close() - if err != nil { - c.logger.Error("failed closing cached connection", zap.Error(err)) - if firstErr == nil { - firstErr = err - } - } - } - - return firstErr } -func (c *connectionCache) get(ctx context.Context, instanceID, driver string, config map[string]any, shared bool) (drivers.Handle, func(), error) { - var key string - if shared { - // not using instanceID to ensure all instances share the same handle - key = driver + generateKey(config) - } else { - key = instanceID + driver + generateKey(config) - } - - c.lock.Lock() - if c.closed { - c.lock.Unlock() - return nil, nil, errConnectionCacheClosed - } - - // Get conn from caches - conn, ok := c.acquired[key] - if ok { - conn.refs++ - } else { - var val any - val, ok = c.lru.Get(key) - if ok { - // Conn was found in LRU - move to acquired cache - conn = val.(*connWithRef) - conn.refs++ // NOTE: Must increment before call to c.lru.remove to avoid closing the conn - c.lru.Remove(key) - c.acquired[key] = conn - } - } - - // Cached conn not found, open a new one - if !ok { - conn = &connWithRef{ - instanceID: instanceID, - refs: 1, // Since refs is assumed to already have been incremented when checking conn.ready - ready: make(chan struct{}), - } - c.acquired[key] = conn - - if len(c.acquired)+c.lru.Len() > c.size { - c.logger.Warn("number of connections acquired and in LRU exceed total configured size", zap.Int("acquired", len(c.acquired)), zap.Int("lru", c.lru.Len())) - } - - // Open and migrate the connection in a separate goroutine (outside lock). - // Incrementing ref and releasing the conn for this operation separately to cover the case where all waiting goroutines are cancelled before the migration completes. - conn.refs++ - go func() { - handle, err := c.openAndMigrate(c.migrateCtx, instanceID, driver, shared, config) - c.lock.Lock() - conn.handle = handle - conn.err = err - c.releaseConn(key, conn) - wasClosed := c.closed - c.lock.Unlock() - close(conn.ready) - - // The cache might have been closed while the connection was being opened. - // Since we acquired the lock, the close will have already been completed, so we need to close the connection here. - if wasClosed && handle != nil { - _ = handle.Close() - } - }() - } - - // We can now release the lock and wait for the connection to be ready (it might already be) - c.lock.Unlock() - - // Wait for connection to be ready or context to be cancelled - var err error - select { - case <-conn.ready: - case <-ctx.Done(): - err = ctx.Err() // Will always be non-nil, ensuring releaseConn is called - } - - // Lock again for accessing conn - c.lock.Lock() - defer c.lock.Unlock() - - if err == nil { - err = conn.err +// getConnection returns a cached connection for the given driver configuration. +func (r *Runtime) getConnection(ctx context.Context, instanceID, driver string, config map[string]any, shared bool) (drivers.Handle, func(), error) { + cfg := cachedConnectionConfig{ + instanceID: instanceID, + driver: driver, + shared: shared, + config: config, } + handle, release, err := r.connCache.Acquire(ctx, cfg) if err != nil { - c.releaseConn(key, conn) return nil, nil, err } - release := func() { - c.lock.Lock() - c.releaseConn(key, conn) - c.lock.Unlock() - } - - return conn.handle, release, nil + return handle.(drivers.Handle), release, nil } -func (c *connectionCache) releaseConn(key string, conn *connWithRef) { - conn.refs-- - if conn.refs == 0 { - // No longer referenced. Move from acquired to LRU. - if !c.closed { - delete(c.acquired, key) - c.lru.Add(key, conn) - } - } +// evictInstanceConnections evicts all connections for the given instance. +func (r *Runtime) evictInstanceConnections(instanceID string) { + r.connCache.EvictWhere(func(cfg any) bool { + x := cfg.(cachedConnectionConfig) + return x.instanceID == instanceID + }) } -func (c *connectionCache) openAndMigrate(ctx context.Context, instanceID, driver string, shared bool, config map[string]any) (drivers.Handle, error) { - logger := c.logger - if instanceID != "default" { - logger = c.logger.With(zap.String("instance_id", instanceID), zap.String("driver", driver)) +// openAndMigrate opens a connection and migrates it. +func (r *Runtime) openAndMigrate(ctx context.Context, cfg cachedConnectionConfig) (drivers.Handle, error) { + logger := r.logger + if cfg.instanceID != "default" { + logger = r.logger.With(zap.String("instance_id", cfg.instanceID), zap.String("driver", cfg.driver)) } - ctx, cancel := context.WithTimeout(ctx, migrateTimeout) - defer cancel() - - activityClient := c.activity - if !shared { - inst, err := c.runtime.Instance(ctx, instanceID) + activityClient := r.activity + if !cfg.shared { + inst, err := r.Instance(ctx, cfg.instanceID) if err != nil { return nil, err } @@ -256,9 +110,9 @@ func (c *connectionCache) openAndMigrate(ctx context.Context, instanceID, driver } } - handle, err := drivers.Open(driver, config, shared, activityClient, logger) + handle, err := drivers.Open(cfg.driver, cfg.config, cfg.shared, activityClient, logger) if err == nil && ctx.Err() != nil { - err = fmt.Errorf("timed out while opening driver %q", driver) + err = fmt.Errorf("timed out while opening driver %q", cfg.driver) } if err != nil { return nil, err @@ -268,71 +122,23 @@ func (c *connectionCache) openAndMigrate(ctx context.Context, instanceID, driver if err != nil { handle.Close() if errors.Is(err, ctx.Err()) { - err = fmt.Errorf("timed out while migrating driver %q: %w", driver, err) + err = fmt.Errorf("timed out while migrating driver %q: %w", cfg.driver, err) } return nil, err } return handle, nil } -// evictAll closes all connections for an instance. -func (c *connectionCache) evictAll(ctx context.Context, instanceID string) { - c.lock.Lock() - defer c.lock.Unlock() - - if c.closed { - return - } - - for key, conn := range c.acquired { - if conn.instanceID != instanceID { - continue - } - - if conn.handle != nil { - err := conn.handle.Close() - if err != nil { - c.logger.Error("connection cache: failed to close cached connection", zap.Error(err), zap.String("instance", instanceID), observability.ZapCtx(ctx)) - } - conn.handle = nil - conn.err = fmt.Errorf("connection evicted") // Defensive, should never be accessed - } - - delete(c.acquired, key) - } - - for _, key := range c.lru.Keys() { - connT, ok := c.lru.Get(key) - if !ok { - panic("connection cache: key not found in LRU") - } - conn := connT.(*connWithRef) - - if conn.instanceID != instanceID { - continue - } - - if conn.handle != nil { - err := conn.handle.Close() - if err != nil { - c.logger.Error("connection cache: failed to close cached connection", zap.Error(err), zap.String("instance", instanceID), observability.ZapCtx(ctx)) - } - conn.handle = nil - conn.err = fmt.Errorf("connection evicted") // Defensive, should never be accessed - } - - c.lru.Remove(key) - } -} - -func generateKey(m map[string]any) string { +func generateKey(cfg cachedConnectionConfig) string { sb := strings.Builder{} - keys := maps.Keys(m) + sb.WriteString(cfg.instanceID) // Empty if cfg.shared + sb.WriteString(cfg.driver) + keys := maps.Keys(cfg.config) slices.Sort(keys) for _, key := range keys { sb.WriteString(key) sb.WriteString(":") - sb.WriteString(fmt.Sprint(m[key])) + sb.WriteString(fmt.Sprint(cfg.config[key])) sb.WriteString(" ") } return sb.String() diff --git a/runtime/connection_cache_test.go b/runtime/connection_cache_test.go deleted file mode 100644 index 299021e1749..00000000000 --- a/runtime/connection_cache_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package runtime - -import ( - "context" - "fmt" - "sync" - "sync/atomic" - "testing" - "time" - - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" - "github.com/rilldata/rill/runtime/drivers" - _ "github.com/rilldata/rill/runtime/drivers/sqlite" - "github.com/rilldata/rill/runtime/pkg/activity" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestConnectionCache(t *testing.T) { - ctx := context.Background() - id := "default" - - rt := newTestRuntimeWithInst(t) - c := newConnectionCache(10, zap.NewNop(), rt, activity.NewNoopClient()) - conn1, release, err := c.get(ctx, id, "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - release() - require.NotNil(t, conn1) - - conn2, release, err := c.get(ctx, id, "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - release() - require.NotNil(t, conn2) - - inst := &drivers.Instance{ - ID: "default1", - OLAPConnector: "duckdb", - RepoConnector: "repo", - CatalogConnector: "catalog", - Connectors: []*runtimev1.Connector{ - { - Type: "file", - Name: "repo", - Config: map[string]string{"dsn": ""}, - }, - { - Type: "duckdb", - Name: "duckdb", - Config: map[string]string{"dsn": ""}, - }, - { - Type: "sqlite", - Name: "catalog", - Config: map[string]string{"dsn": "file:rill?mode=memory&cache=shared"}, - }, - }, - } - require.NoError(t, rt.CreateInstance(context.Background(), inst)) - - conn3, release, err := c.get(ctx, "default1", "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - release() - require.NotNil(t, conn3) - - require.True(t, conn1 == conn2) - require.False(t, conn2 == conn3) -} - -func TestConnectionCacheWithAllShared(t *testing.T) { - ctx := context.Background() - id := "default" - - c := newConnectionCache(1, zap.NewNop(), newTestRuntimeWithInst(t), activity.NewNoopClient()) - conn1, release, err := c.get(ctx, id, "sqlite", map[string]any{"dsn": ":memory:"}, true) - require.NoError(t, err) - require.NotNil(t, conn1) - defer release() - - conn2, release, err := c.get(ctx, id, "sqlite", map[string]any{"dsn": ":memory:"}, true) - require.NoError(t, err) - require.NotNil(t, conn2) - defer release() - - conn3, release, err := c.get(ctx, "default", "sqlite", map[string]any{"dsn": ":memory:"}, true) - require.NoError(t, err) - require.NotNil(t, conn3) - defer release() - - require.True(t, conn1 == conn2) - require.True(t, conn2 == conn3) - require.Equal(t, 1, len(c.acquired)) - require.Equal(t, 0, c.lru.Len()) -} - -func TestConnectionCacheWithAllOpen(t *testing.T) { - ctx := context.Background() - - rt := newTestRuntimeWithInst(t) - c := newConnectionCache(1, zap.NewNop(), rt, activity.NewNoopClient()) - conn1, r1, err := c.get(ctx, "default", "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - require.NotNil(t, conn1) - - createInstance(t, rt, "default1") - conn2, r2, err := c.get(ctx, "default1", "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - require.NotNil(t, conn2) - - createInstance(t, rt, "default2") - conn3, r3, err := c.get(ctx, "default2", "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - require.NotNil(t, conn3) - - require.Equal(t, 3, len(c.acquired)) - require.Equal(t, 0, c.lru.Len()) - // release all connections - r1() - r2() - r3() - require.Equal(t, 0, len(c.acquired)) - require.Equal(t, 1, c.lru.Len()) - _, val, _ := c.lru.GetOldest() - require.True(t, conn3 == val.(*connWithRef).handle) -} - -func TestConnectionCacheParallel(t *testing.T) { - ctx := context.Background() - - rt := newTestRuntimeWithInst(t) - c := newConnectionCache(5, zap.NewNop(), rt, activity.NewNoopClient()) - defer c.Close() - - var wg sync.WaitGroup - wg.Add(30) - // open 10 connections and do not release - go func() { - for i := 0; i < 10; i++ { - j := i - go func() { - defer wg.Done() - id := fmt.Sprintf("default%v", 100+j) - createInstance(t, rt, id) - conn, _, err := c.get(ctx, id, "sqlite", map[string]any{"dsn": ":memory:"}, false) - require.NoError(t, err) - require.NotNil(t, conn) - time.Sleep(100 * time.Millisecond) - }() - } - }() - - // open 20 connections and release - for i := 0; i < 20; i++ { - j := i - go func() { - defer wg.Done() - id := fmt.Sprintf("default%v", 200+j) - createInstance(t, rt, id) - conn, r, err := c.get(ctx, id, "sqlite", map[string]any{"dsn": ":memory:"}, false) - defer r() - require.NoError(t, err) - require.NotNil(t, conn) - time.Sleep(100 * time.Millisecond) - }() - } - wg.Wait() - - // 10 connections were not released so should be present in in-use cache - require.Equal(t, 10, len(c.acquired)) - // 20 connections were released so 15 should be evicted - require.Equal(t, 5, c.lru.Len()) -} - -func TestConnectionCacheMultipleConfigs(t *testing.T) { - ctx := context.Background() - - c := newConnectionCache(10, zap.NewNop(), newTestRuntimeWithInst(t), activity.NewNoopClient()) - defer c.Close() - conn1, r1, err := c.get(ctx, "default", "sqlite", map[string]any{"dsn": ":memory:", "host": "localhost:8080", "allow_host_access": "true"}, true) - require.NoError(t, err) - require.NotNil(t, conn1) - - conn2, r2, err := c.get(ctx, "default", "sqlite", map[string]any{"dsn": ":memory:", "host": "localhost:8080", "allow_host_access": "true"}, true) - require.NoError(t, err) - require.NotNil(t, conn2) - - conn3, r3, err := c.get(ctx, "default", "sqlite", map[string]any{"dsn": ":memory:", "host": "localhost:8080", "allow_host_access": "true"}, true) - require.NoError(t, err) - require.NotNil(t, conn3) - - require.Equal(t, 1, len(c.acquired)) - require.Equal(t, 0, c.lru.Len()) - // release all connections - r1() - r2() - r3() - require.Equal(t, 0, len(c.acquired)) - require.Equal(t, 1, c.lru.Len()) -} - -func TestConnectionCacheParallelCalls(t *testing.T) { - ctx := context.Background() - - m := &mockDriver{} - drivers.Register("mock_driver", m) - defer func() { - delete(drivers.Drivers, "mock_driver") - }() - - rt := newTestRuntimeWithInst(t) - defer rt.Close() - - c := newConnectionCache(10, zap.NewNop(), rt, activity.NewNoopClient()) - defer c.Close() - - var wg sync.WaitGroup - wg.Add(10) - // open 10 connections and verify no error - for i := 0; i < 10; i++ { - go func() { - defer wg.Done() - conn, _, err := c.get(ctx, "default", "mock_driver", map[string]any{"sleep": int64(100)}, false) - require.NoError(t, err) - require.NotNil(t, conn) - }() - } - wg.Wait() - - require.Equal(t, int32(1), m.opened.Load()) - require.Equal(t, 1, len(c.acquired)) -} - -func TestConnectionCacheBlockingCalls(t *testing.T) { - ctx := context.Background() - - m := &mockDriver{} - drivers.Register("mock_driver", m) - defer func() { - delete(drivers.Drivers, "mock_driver") - }() - - rt := newTestRuntimeWithInst(t) - defer rt.Close() - - c := newConnectionCache(10, zap.NewNop(), rt, activity.NewNoopClient()) - defer c.Close() - - var wg sync.WaitGroup - wg.Add(12) - // open 1 slow connection - go func() { - defer wg.Done() - conn, _, err := c.get(ctx, "default", "mock_driver", map[string]any{"sleep": int64(1000)}, false) - require.NoError(t, err) - require.NotNil(t, conn) - }() - - // open 10 fast different connections(takes 10-20 ms to open) and verify not blocked - for i := 0; i < 10; i++ { - j := i - go func() { - defer wg.Done() - conn, _, err := c.get(ctx, "default", "mock_driver", map[string]any{"sleep": int64(j + 10)}, false) - require.NoError(t, err) - require.NotNil(t, conn) - }() - } - - // verify that after 200 ms 11 connections have been opened - go func() { - time.Sleep(200 * time.Millisecond) - wg.Done() - }() - wg.Wait() - - require.Equal(t, int32(11), m.opened.Load()) -} - -type mockDriver struct { - opened atomic.Int32 -} - -// Drop implements drivers.Driver. -func (*mockDriver) Drop(config map[string]any, logger *zap.Logger) error { - panic("unimplemented") -} - -// HasAnonymousSourceAccess implements drivers.Driver. -func (*mockDriver) HasAnonymousSourceAccess(ctx context.Context, src map[string]any, logger *zap.Logger) (bool, error) { - panic("unimplemented") -} - -func (*mockDriver) TertiarySourceConnectors(ctx context.Context, src map[string]any, logger *zap.Logger) ([]string, error) { - return nil, nil -} - -// Open implements drivers.Driver. -func (m *mockDriver) Open(config map[string]any, shared bool, client activity.Client, logger *zap.Logger) (drivers.Handle, error) { - m.opened.Add(1) - sleep := config["sleep"].(int64) - time.Sleep(time.Duration(sleep) * time.Millisecond) - return &mockHandle{}, nil -} - -// Spec implements drivers.Driver. -func (*mockDriver) Spec() drivers.Spec { - panic("unimplemented") -} - -var _ drivers.Driver = &mockDriver{} - -type mockHandle struct { -} - -// AsCatalogStore implements drivers.Handle. -func (*mockHandle) AsCatalogStore(instanceID string) (drivers.CatalogStore, bool) { - panic("unimplemented") -} - -// AsFileStore implements drivers.Handle. -func (*mockHandle) AsFileStore() (drivers.FileStore, bool) { - panic("unimplemented") -} - -// AsOLAP implements drivers.Handle. -func (*mockHandle) AsOLAP(instanceID string) (drivers.OLAPStore, bool) { - panic("unimplemented") -} - -// AsObjectStore implements drivers.Handle. -func (*mockHandle) AsObjectStore() (drivers.ObjectStore, bool) { - panic("unimplemented") -} - -// AsRegistry implements drivers.Handle. -func (*mockHandle) AsRegistry() (drivers.RegistryStore, bool) { - panic("unimplemented") -} - -// AsRepoStore implements drivers.Handle. -func (*mockHandle) AsRepoStore(instanceID string) (drivers.RepoStore, bool) { - panic("unimplemented") -} - -// AsAdmin implements drivers.Handle. -func (*mockHandle) AsAdmin(instanceID string) (drivers.AdminService, bool) { - panic("unimplemented") -} - -// AsSQLStore implements drivers.Handle. -func (*mockHandle) AsSQLStore() (drivers.SQLStore, bool) { - panic("unimplemented") -} - -// AsTransporter implements drivers.Handle. -func (*mockHandle) AsTransporter(from drivers.Handle, to drivers.Handle) (drivers.Transporter, bool) { - panic("unimplemented") -} - -// Close implements drivers.Handle. -func (*mockHandle) Close() error { - return nil -} - -// Config implements drivers.Handle. -func (*mockHandle) Config() map[string]any { - panic("unimplemented") -} - -// Driver implements drivers.Handle. -func (*mockHandle) Driver() string { - panic("unimplemented") -} - -// Migrate implements drivers.Handle. -func (*mockHandle) Migrate(ctx context.Context) error { - return nil -} - -// MigrationStatus implements drivers.Handle. -func (*mockHandle) MigrationStatus(ctx context.Context) (current int, desired int, err error) { - panic("unimplemented") -} - -var _ drivers.Handle = &mockHandle{} - -func newTestRuntimeWithInst(t *testing.T) *Runtime { - rt := newTestRuntime(t) - createInstance(t, rt, "default") - return rt -} - -func createInstance(t *testing.T, rt *Runtime, instanceId string) { - inst := &drivers.Instance{ - ID: instanceId, - OLAPConnector: "duckdb", - RepoConnector: "repo", - CatalogConnector: "catalog", - Connectors: []*runtimev1.Connector{ - { - Type: "file", - Name: "repo", - Config: map[string]string{"dsn": ""}, - }, - { - Type: "duckdb", - Name: "duckdb", - Config: map[string]string{"dsn": ""}, - }, - { - Type: "sqlite", - Name: "catalog", - Config: map[string]string{"dsn": "file:rill?mode=memory&cache=shared"}, - }, - }, - } - require.NoError(t, rt.CreateInstance(context.Background(), inst)) -} diff --git a/runtime/connections.go b/runtime/connections.go index ba939063c07..b72c3618551 100644 --- a/runtime/connections.go +++ b/runtime/connections.go @@ -19,7 +19,7 @@ func (r *Runtime) AcquireSystemHandle(ctx context.Context, connector string) (dr cfg[strings.ToLower(k)] = v } cfg["allow_host_access"] = r.opts.AllowHostAccess - return r.connCache.get(ctx, "", c.Type, cfg, true) + return r.getConnection(ctx, "", c.Type, cfg, true) } } return nil, nil, fmt.Errorf("connector %s doesn't exist", connector) @@ -36,7 +36,7 @@ func (r *Runtime) AcquireHandle(ctx context.Context, instanceID, connector strin // So we take this moment to make sure the ctx gets checked for cancellation at least every once in a while. return nil, nil, ctx.Err() } - return r.connCache.get(ctx, instanceID, driver, cfg, false) + return r.getConnection(ctx, instanceID, driver, cfg, false) } func (r *Runtime) Repo(ctx context.Context, instanceID string) (drivers.RepoStore, func(), error) { diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index f2a3737613e..78f57909285 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -813,7 +813,7 @@ func (c *connection) periodicallyCheckConnDurations(d time.Duration) { c.connTimesMu.Lock() for connID, connTime := range c.connTimes { if time.Since(connTime) > maxAcquiredConnDuration { - c.logger.Error("duckdb: a connection has been held for more longer than the maximum allowed duration", zap.Int("conn_id", connID), zap.Duration("duration", time.Since(connTime))) + c.logger.Error("duckdb: a connection has been held for longer than the maximum allowed duration", zap.Int("conn_id", connID), zap.Duration("duration", time.Since(connTime))) } } c.connTimesMu.Unlock() diff --git a/runtime/pkg/conncache/conncache.go b/runtime/pkg/conncache/conncache.go new file mode 100644 index 00000000000..b746f46a163 --- /dev/null +++ b/runtime/pkg/conncache/conncache.go @@ -0,0 +1,450 @@ +package conncache + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/hashicorp/golang-lru/simplelru" + "go.opentelemetry.io/otel/metric" +) + +// Cache is a concurrency-safe cache of stateful connection objects. +// It differs from a connection pool in that it's designed for caching heterogenous connections. +// The cache will at most open one connection per key, even under concurrent access. +// The cache automatically evicts connections that are not in use ("acquired") using a least-recently-used policy. +type Cache interface { + // Acquire retrieves or opens a connection for the given config. The returned ReleaseFunc must be called when the connection is no longer needed. + // While a connection is acquired, it will not be closed unless EvictWhere or Close is called. + // If Acquire is called while the underlying connection is being evicted, it will wait for the close to complete and then open a new connection. + // If opening the connection fails, Acquire may return the error on subsequent calls without trying to open again until the entry is evicted. + Acquire(ctx context.Context, cfg any) (Connection, ReleaseFunc, error) + + // EvictWhere closes the connections that match the predicate. + // It immediately starts closing the connections, even those that are currently acquired. + // It returns quickly and does not wait for connections to finish closing in the background. + EvictWhere(predicate func(cfg any) bool) + + // Close closes all open connections and prevents new connections from being acquired. + // It returns when all cached connections have been closed or when the provided ctx is cancelled. + Close(ctx context.Context) error +} + +// Connection is a connection that may be cached. +type Connection interface { + Close() error +} + +// ReleaseFunc is a function that must be called when an acquired connection is no longer needed. +type ReleaseFunc func() + +// Options configures a new connection cache. +type Options struct { + // MaxIdleConnections is the maximum number of non-acquired connections that will be kept open. + MaxIdleConnections int + // OpenTimeout is the maximum amount of time to wait for a connection to open. + OpenTimeout time.Duration + // CloseTimeout is the maximum amount of time to wait for a connection to close. + CloseTimeout time.Duration + // CheckHangingInterval is the interval at which to check for hanging open/close calls. + CheckHangingInterval time.Duration + // OpenFunc opens a connection. + OpenFunc func(ctx context.Context, cfg any) (Connection, error) + // KeyFunc computes a comparable key for a connection config. + KeyFunc func(cfg any) string + // HangingFunc is called when an open or close exceeds its timeout and does not respond to context cancellation. + HangingFunc func(cfg any, open bool) + // Metrics are optional instruments for observability. + Metrics Metrics +} + +// Metrics are optional instruments for observability. If an instrument is nil, it will not be collected. +type Metrics struct { + Opens metric.Int64Counter + Closes metric.Int64Counter + SizeTotal metric.Int64UpDownCounter + SizeLRU metric.Int64UpDownCounter + OpenLatencyMS metric.Int64Histogram + CloseLatencyMS metric.Int64Histogram +} + +var _ Cache = (*cacheImpl)(nil) + +// cacheImpl implements Cache. Implementation notes: +// - It uses an LRU to pool unused connections and eventually close them. +// - It leverages a singleflight pattern to ensure at most one open/close action runs against a connection at a time. +// - It directly implements a singleflight (instead of using a library) because it needs to use the same mutex for the singleflight and the map/LRU to avoid race conditions. +// - An entry will only have entryStatusOpening or entryStatusClosing if a singleflight call is currently running for it. +// - Any code that keeps a reference to an entry after the mutex is released must call retainEntry/releaseEntry. +// - If the ctx for an open call is cancelled, the entry will continue opening in the background (and will be put in the LRU). +// - If attempting to open a closing entry, or close an opening entry, we wait for the singleflight to complete and then retry once. To avoid infinite loops, we don't retry more than once. +type cacheImpl struct { + opts Options + closed bool + mu sync.Mutex + entries map[string]*entry + lru *simplelru.LRU + singleflight map[string]chan struct{} + ctx context.Context + cancel context.CancelFunc +} + +type entry struct { + cfg any + refs int + status entryStatus + since time.Time + closeAfterOpening bool + handle Connection + err error +} + +type entryStatus int + +const ( + entryStatusUnspecified entryStatus = iota + entryStatusOpening + entryStatusOpen // Also used for cases where open errored (i.e. entry.err != nil) + entryStatusClosing + entryStatusClosed +) + +func New(opts Options) Cache { + ctx, cancel := context.WithCancel(context.Background()) + c := &cacheImpl{ + opts: opts, + entries: make(map[string]*entry), + singleflight: make(map[string]chan struct{}), + ctx: ctx, + cancel: cancel, + } + + var err error + c.lru, err = simplelru.NewLRU(opts.MaxIdleConnections, c.lruEvictionHandler) + if err != nil { + panic(err) + } + + if opts.CheckHangingInterval != 0 { + go c.periodicallyCheckHangingConnections() + } + + return c +} + +func (c *cacheImpl) Acquire(ctx context.Context, cfg any) (Connection, ReleaseFunc, error) { + k := c.opts.KeyFunc(cfg) + + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return nil, nil, errors.New("conncache: closed") + } + + e, ok := c.entries[k] + if !ok { + e = &entry{cfg: cfg, since: time.Now()} + c.entries[k] = e + if c.opts.Metrics.SizeTotal != nil { + c.opts.Metrics.SizeTotal.Add(c.ctx, 1) + } + } + + c.retainEntry(k, e) + + if e.status == entryStatusOpen { + defer c.mu.Unlock() + if e.err != nil { + c.releaseEntry(k, e) + return nil, nil, e.err + } + return e.handle, c.releaseFunc(k, e), nil + } + + ch, ok := c.singleflight[k] + + if ok && e.status == entryStatusClosing { + c.mu.Unlock() + select { + case <-ch: + case <-ctx.Done(): + c.mu.Lock() + c.releaseEntry(k, e) + c.mu.Unlock() + return nil, nil, ctx.Err() + } + c.mu.Lock() + + // Since we released the lock, need to check c.closed and e.status again. + if c.closed { + c.releaseEntry(k, e) + c.mu.Unlock() + return nil, nil, errors.New("conncache: closed") + } + + if e.status == entryStatusOpen { + defer c.mu.Unlock() + if e.err != nil { + c.releaseEntry(k, e) + return nil, nil, e.err + } + return e.handle, c.releaseFunc(k, e), nil + } + + ch, ok = c.singleflight[k] + } + + if !ok { + c.retainEntry(k, e) // Retain again to count the goroutine's reference independently (in case ctx is cancelled while the Open continues in the background) + + ch = make(chan struct{}) + c.singleflight[k] = ch + + e.status = entryStatusOpening + e.since = time.Now() + e.handle = nil + e.err = nil + + go func() { + start := time.Now() + var handle Connection + var err error + if c.opts.OpenTimeout == 0 { + handle, err = c.opts.OpenFunc(c.ctx, cfg) + } else { + ctx, cancel := context.WithTimeout(c.ctx, c.opts.OpenTimeout) + handle, err = c.opts.OpenFunc(ctx, cfg) + cancel() + } + + if c.opts.Metrics.Opens != nil { + c.opts.Metrics.Opens.Add(c.ctx, 1) + } + if c.opts.Metrics.OpenLatencyMS != nil { + c.opts.Metrics.OpenLatencyMS.Record(c.ctx, time.Since(start).Milliseconds()) + } + + c.mu.Lock() + defer c.mu.Unlock() + + e.status = entryStatusOpen + e.since = time.Now() + e.handle = handle + e.err = err + + delete(c.singleflight, k) + close(ch) + + if e.closeAfterOpening { + e.closeAfterOpening = false + c.beginClose(k, e) + } + + c.releaseEntry(k, e) + }() + } + + c.mu.Unlock() + + select { + case <-ch: + case <-ctx.Done(): + c.mu.Lock() + c.releaseEntry(k, e) + c.mu.Unlock() + return nil, nil, ctx.Err() + } + + c.mu.Lock() + defer c.mu.Unlock() + + if e.status != entryStatusOpen { + c.releaseEntry(k, e) + return nil, nil, errors.New("conncache: connection was immediately closed after being opened") + } + + if e.err != nil { + c.releaseEntry(k, e) + return nil, nil, e.err + } + + return e.handle, c.releaseFunc(k, e), nil +} + +func (c *cacheImpl) EvictWhere(predicate func(cfg any) bool) { + c.mu.Lock() + defer c.mu.Unlock() + for k, e := range c.entries { + if predicate(e.cfg) { + c.beginClose(k, e) + } + } +} + +func (c *cacheImpl) Close(ctx context.Context) error { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return errors.New("conncache: already closed") + } + c.closed = true + + c.cancel() + + for k, e := range c.entries { + c.beginClose(k, e) + } + + c.mu.Unlock() + + for { + c.mu.Lock() + var anyCh chan struct{} + for _, ch := range c.singleflight { + anyCh = ch + break + } + c.mu.Unlock() + + if anyCh == nil { + // all entries are closed, we can return + return nil + } + + select { + case <-anyCh: + // continue + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// beginClose must be called while c.mu is held. +func (c *cacheImpl) beginClose(k string, e *entry) { + if e.status == entryStatusClosing || e.status == entryStatusClosed { + return + } + + if e.status == entryStatusOpening { + e.closeAfterOpening = true + return + } + + c.retainEntry(k, e) + + ch, ok := c.singleflight[k] + if ok { + // Should never happen, but checking since it would be pretty bad. + panic(errors.New("conncache: singleflight exists for entry that is neither opening nor closing")) + } + ch = make(chan struct{}) + c.singleflight[k] = ch + + e.status = entryStatusClosing + e.since = time.Now() + + go func() { + start := time.Now() + var err error + if e.handle != nil { + err = e.handle.Close() + } + if err == nil { + err = errors.New("conncache: connection closed") + } + + if c.opts.Metrics.Closes != nil { + c.opts.Metrics.Closes.Add(c.ctx, 1) + } + if c.opts.Metrics.CloseLatencyMS != nil { + c.opts.Metrics.CloseLatencyMS.Record(c.ctx, time.Since(start).Milliseconds()) + } + + c.mu.Lock() + defer c.mu.Unlock() + + e.status = entryStatusClosed + e.since = time.Now() + e.handle = nil + e.err = err + + delete(c.singleflight, k) + close(ch) + + c.releaseEntry(k, e) + }() +} + +func (c *cacheImpl) lruEvictionHandler(key, value any) { + k := key.(string) + e := value.(*entry) + + // The callback also gets called when removing from LRU during acquisition. + // We use conn.refs != 0 to signal that its being acquired and should not be closed. + if e.refs == 0 { + c.beginClose(k, e) + } +} + +func (c *cacheImpl) retainEntry(key string, e *entry) { + e.refs++ + if e.refs == 1 { + // NOTE: lru.Remove is safe even if it's not in the LRU (should only happen if the entry is acquired for the first time) + _ = c.lru.Remove(key) + if c.opts.Metrics.SizeLRU != nil { + c.opts.Metrics.SizeLRU.Add(c.ctx, -1) + } + } +} + +func (c *cacheImpl) releaseEntry(key string, e *entry) { + e.refs-- + if e.refs == 0 { + // If open, keep entry and put in LRU. Else remove entirely. + if e.status != entryStatusClosing && e.status != entryStatusClosed { + c.lru.Add(key, e) + if c.opts.Metrics.SizeLRU != nil { + c.opts.Metrics.SizeLRU.Add(c.ctx, 1) + } + } else { + delete(c.entries, key) + if c.opts.Metrics.SizeTotal != nil { + c.opts.Metrics.SizeTotal.Add(c.ctx, -1) + } + } + } +} + +func (c *cacheImpl) releaseFunc(key string, e *entry) ReleaseFunc { + return func() { + c.mu.Lock() + c.releaseEntry(key, e) + c.mu.Unlock() + } +} + +func (c *cacheImpl) periodicallyCheckHangingConnections() { + ticker := time.NewTicker(c.opts.CheckHangingInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + c.mu.Lock() + for k := range c.singleflight { + e := c.entries[k] + if c.opts.OpenTimeout != 0 && e.status == entryStatusOpening && time.Since(e.since) > c.opts.OpenTimeout { + c.opts.HangingFunc(e.cfg, true) + } + if c.opts.CloseTimeout != 0 && e.status == entryStatusClosing && time.Since(e.since) > c.opts.CloseTimeout { + c.opts.HangingFunc(e.cfg, false) + } + } + c.mu.Unlock() + case <-c.ctx.Done(): + return + } + } +} diff --git a/runtime/pkg/conncache/conncache_test.go b/runtime/pkg/conncache/conncache_test.go new file mode 100644 index 00000000000..8c818830907 --- /dev/null +++ b/runtime/pkg/conncache/conncache_test.go @@ -0,0 +1,268 @@ +package conncache + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type mockConn struct { + cfg string + closeDelay time.Duration + closeCalled atomic.Bool +} + +func (c *mockConn) Close() error { + c.closeCalled.Store(true) + time.Sleep(c.closeDelay) + return nil +} + +func TestBasic(t *testing.T) { + opens := atomic.Int64{} + + c := New(Options{ + MaxIdleConnections: 2, + OpenFunc: func(ctx context.Context, cfg any) (Connection, error) { + opens.Add(1) + return &mockConn{cfg: cfg.(string)}, nil + }, + KeyFunc: func(cfg any) string { + return cfg.(string) + }, + }) + + // Get "foo" + m1, r1, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Equal(t, int64(1), opens.Load()) + + // Get "foo" again + m2, r2, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Equal(t, int64(1), opens.Load()) + + // Check that they're the same + require.Equal(t, m1, m2) + + // Release the "foo"s and get "foo" again, check it's the same + r1() + r2() + m3, r3, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Equal(t, int64(1), opens.Load()) + require.Equal(t, m1, m3) + r3() + + // Open and release two more conns, check "foo" is closed (since LRU size is 2) + for i := 0; i < 2; i++ { + _, r, err := c.Acquire(context.Background(), fmt.Sprintf("bar%d", i)) + require.NoError(t, err) + require.Equal(t, int64(1+i+1), opens.Load()) + r() + } + time.Sleep(time.Second) + require.Equal(t, true, m1.(*mockConn).closeCalled.Load()) + + // Close cache + require.NoError(t, c.Close(context.Background())) +} + +func TestConcurrentOpen(t *testing.T) { + opens := atomic.Int64{} + + c := New(Options{ + MaxIdleConnections: 2, + OpenFunc: func(ctx context.Context, cfg any) (Connection, error) { + opens.Add(1) + time.Sleep(time.Second) + return &mockConn{cfg: cfg.(string)}, nil + }, + KeyFunc: func(cfg any) string { + return cfg.(string) + }, + }) + + var m1, m2 Connection + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + m, _, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + m1 = m + }() + go func() { + defer wg.Done() + m, _, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + m2 = m + }() + + wg.Wait() + require.NotNil(t, m1) + require.Equal(t, m1, m2) + require.Equal(t, int64(1), opens.Load()) + + // Close cache + require.NoError(t, c.Close(context.Background())) +} + +func TestOpenDuringClose(t *testing.T) { + opens := atomic.Int64{} + + c := New(Options{ + MaxIdleConnections: 2, + OpenFunc: func(ctx context.Context, cfg any) (Connection, error) { + opens.Add(1) + return &mockConn{ + cfg: cfg.(string), + closeDelay: time.Second, // Closes hang for 1s + }, nil + }, + KeyFunc: func(cfg any) string { + return cfg.(string) + }, + }) + + // Create conn + m1, r1, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Equal(t, int64(1), opens.Load()) + r1() + + // Evict it so it starts closing + c.EvictWhere(func(cfg any) bool { return true }) + // closeCalled is set before mockConn.Close hangs, but it will take 1s to actually close + time.Sleep(100 * time.Millisecond) + require.True(t, m1.(*mockConn).closeCalled.Load()) + + // Open again, check it takes ~1s to do so + start := time.Now() + m2, r2, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Greater(t, time.Since(start), 500*time.Millisecond) + require.Equal(t, int64(2), opens.Load()) + require.NotEqual(t, m1, m2) + r2() + + // Close cache + require.NoError(t, c.Close(context.Background())) +} + +func TestCloseDuringOpen(t *testing.T) { + opens := atomic.Int64{} + m := &mockConn{cfg: "foo"} + + c := New(Options{ + MaxIdleConnections: 2, + OpenFunc: func(ctx context.Context, cfg any) (Connection, error) { + time.Sleep(time.Second) + opens.Add(1) + return m, nil + }, + KeyFunc: func(cfg any) string { + return cfg.(string) + }, + }) + + // Start opening + go func() { + _, _, err := c.Acquire(context.Background(), "foo") + require.ErrorContains(t, err, "immediately closed") + require.Equal(t, int64(1), opens.Load()) + }() + + // Evict it so it starts closing + time.Sleep(100 * time.Millisecond) // Give it time to start opening + c.EvictWhere(func(cfg any) bool { return true }) + + // It will let the open finish before closing it, so will take ~1s + time.Sleep(2 * time.Second) + require.True(t, m.closeCalled.Load()) + + // Close cache + require.NoError(t, c.Close(context.Background())) +} + +func TestCloseInUse(t *testing.T) { + opens := atomic.Int64{} + + c := New(Options{ + MaxIdleConnections: 2, + OpenFunc: func(ctx context.Context, cfg any) (Connection, error) { + opens.Add(1) + return &mockConn{cfg: cfg.(string)}, nil + }, + KeyFunc: func(cfg any) string { + return cfg.(string) + }, + }) + + // Open conn "foo" + m1, r1, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Equal(t, int64(1), opens.Load()) + + // Evict it, check it's closed even though still in use (r1 not called) + c.EvictWhere(func(cfg any) bool { return true }) + time.Sleep(time.Second) + require.Equal(t, true, m1.(*mockConn).closeCalled.Load()) + + // Open "foo" again, check it opens a new one + m2, r2, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.Equal(t, int64(2), opens.Load()) + require.NotEqual(t, m1, m2) + + // Check that releasing m1 doesn't fail (though it's been closed) + r1() + r2() +} + +func TestHanging(t *testing.T) { + hangingOpens := atomic.Int64{} + hangingCloses := atomic.Int64{} + + c := New(Options{ + MaxIdleConnections: 2, + OpenTimeout: 100 * time.Millisecond, + CloseTimeout: 100 * time.Millisecond, + CheckHangingInterval: 100 * time.Millisecond, + OpenFunc: func(ctx context.Context, cfg any) (Connection, error) { + time.Sleep(time.Second) + return &mockConn{ + cfg: cfg.(string), + closeDelay: time.Second, // Make closes hang for 1s + }, nil + }, + KeyFunc: func(cfg any) string { + return cfg.(string) + }, + HangingFunc: func(cfg any, open bool) { + if open { + hangingOpens.Add(1) + } else { + hangingCloses.Add(1) + } + }, + }) + + // Open conn "foo" + m1, r1, err := c.Acquire(context.Background(), "foo") + require.NoError(t, err) + require.GreaterOrEqual(t, hangingOpens.Load(), int64(1)) + r1() + + // Evict it, check it's closed even though still in use (r1 not called) + c.EvictWhere(func(cfg any) bool { return true }) + time.Sleep(time.Second) + require.Equal(t, true, m1.(*mockConn).closeCalled.Load()) + require.GreaterOrEqual(t, hangingCloses.Load(), int64(1)) +} diff --git a/runtime/registry.go b/runtime/registry.go index ba521929843..093a48e4c57 100644 --- a/runtime/registry.go +++ b/runtime/registry.go @@ -375,7 +375,7 @@ func (r *registryCache) restartController(iwc *instanceWithController) { // So we want to evict all open connections for that instance, but it's unsafe to do so while the controller is running. // So this is the only place where we can do it safely. if r.baseCtx.Err() == nil { - r.rt.connCache.evictAll(r.baseCtx, iwc.instance.ID) + r.rt.evictInstanceConnections(iwc.instance.ID) } r.mu.Lock() diff --git a/runtime/registry_test.go b/runtime/registry_test.go index 5f305499fad..746d424fd4a 100644 --- a/runtime/registry_test.go +++ b/runtime/registry_test.go @@ -331,7 +331,7 @@ func TestRuntime_EditInstance(t *testing.T) { } // Wait for controller restart - time.Sleep(500 * time.Millisecond) + time.Sleep(2 * time.Second) _, err = rt.Controller(ctx, inst.ID) require.NoError(t, err) @@ -424,8 +424,9 @@ func TestRuntime_DeleteInstance(t *testing.T) { require.Error(t, err) // verify older olap connection is closed and cache updated - require.False(t, rt.connCache.lru.Contains(inst.ID+"duckdb"+fmt.Sprintf("dsn:%s ", dbFile))) - require.False(t, rt.connCache.lru.Contains(inst.ID+"file"+fmt.Sprintf("dsn:%s ", repodsn))) + // require.False(t, rt.connCache.lru.Contains(inst.ID+"duckdb"+fmt.Sprintf("dsn:%s ", dbFile))) + // require.False(t, rt.connCache.lru.Contains(inst.ID+"file"+fmt.Sprintf("dsn:%s ", repodsn))) + time.Sleep(2 * time.Second) err = olap.Exec(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM rill.migration_version"}) require.True(t, err != nil) @@ -474,7 +475,7 @@ func TestRuntime_DeleteInstance_DropCorrupted(t *testing.T) { require.NoError(t, err) // Close OLAP connection - rt.connCache.evictAll(ctx, inst.ID) + rt.evictInstanceConnections(inst.ID) // Corrupt database file err = os.WriteFile(dbpath, []byte("corrupted"), 0644) diff --git a/runtime/runtime.go b/runtime/runtime.go index 46601ac66c1..07e28bc1d5f 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -9,6 +9,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/activity" + "github.com/rilldata/rill/runtime/pkg/conncache" "github.com/rilldata/rill/runtime/pkg/email" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -36,7 +37,7 @@ type Runtime struct { activity activity.Client metastore drivers.Handle registryCache *registryCache - connCache *connectionCache + connCache conncache.Cache queryCache *queryCache securityEngine *securityEngine } @@ -55,7 +56,7 @@ func New(ctx context.Context, opts *Options, logger *zap.Logger, ac activity.Cli securityEngine: newSecurityEngine(opts.SecurityEngineCacheSize, logger), } - rt.connCache = newConnectionCache(opts.ConnectionCacheSize, logger, rt, ac) + rt.connCache = rt.newConnectionCache() store, _, err := rt.AcquireSystemHandle(ctx, opts.MetastoreConnector) if err != nil { @@ -88,7 +89,7 @@ func (r *Runtime) Close() error { defer cancel() err1 := r.registryCache.close(ctx) err2 := r.queryCache.close() - err3 := r.connCache.Close() // Also closes metastore // TODO: Propagate ctx cancellation + err3 := r.connCache.Close(ctx) // Also closes metastore // TODO: Propagate ctx cancellation return errors.Join(err1, err2, err3) } From 50b4e2b24a6c9ee860e0c96ce3ad07b6c3fd4adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= Date: Fri, 15 Dec 2023 12:47:16 +0100 Subject: [PATCH 15/19] Runtime: change reconcile and parse errors to warns (#3691) * Runtime: Refactor conn cache to contain and detect hanging opens/closes * Extract connection cache to pkg + use a singleflight * Add tests * Runtime: change reconcile and parse errors to warns --- runtime/controller.go | 4 ++-- runtime/reconcilers/project_parser.go | 6 +++--- runtime/registry.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/controller.go b/runtime/controller.go index 3274dccf1e6..88d629d54ec 100644 --- a/runtime/controller.go +++ b/runtime/controller.go @@ -1123,7 +1123,7 @@ func (c *Controller) markPending(n *runtimev1.ResourceName) (skip bool, err erro } if !r.Meta.Hidden { logArgs := []any{slog.String("name", n.Name), slog.String("kind", unqualifiedKind(n.Kind)), slog.Any("error", errCyclicDependency)} - c.Logger.Error("Skipping resource", logArgs...) + c.Logger.Warn("Skipping resource", logArgs...) } return true, nil } @@ -1350,7 +1350,7 @@ func (c *Controller) processCompletedInvocation(inv *invocation) error { errorLevel = true } if errorLevel { - c.Logger.Error("Reconcile failed", logArgs...) + c.Logger.Warn("Reconcile failed", logArgs...) } else if !inv.isHidden { c.Logger.Info("Reconciled resource", logArgs...) } diff --git a/runtime/reconcilers/project_parser.go b/runtime/reconcilers/project_parser.go index cf7cadde143..10cfedf3930 100644 --- a/runtime/reconcilers/project_parser.go +++ b/runtime/reconcilers/project_parser.go @@ -244,14 +244,14 @@ func (r *ProjectParserReconciler) reconcileParser(ctx context.Context, inst *dri if skipRillYAMLErr && e.FilePath == "/rill.yaml" { continue } - r.C.Logger.Error("Parser error", slog.String("path", e.FilePath), slog.String("err", e.Message)) + r.C.Logger.Warn("Parser error", slog.String("path", e.FilePath), slog.String("err", e.Message)) } } else if diff.Skipped { - r.C.Logger.Error("Not parsing changed paths due to broken rill.yaml") + r.C.Logger.Warn("Not parsing changed paths due to broken rill.yaml") } else { for _, e := range parser.Errors { if slices.Contains(changedPaths, e.FilePath) { - r.C.Logger.Error("Parser error", slog.String("path", e.FilePath), slog.String("err", e.Message)) + r.C.Logger.Warn("Parser error", slog.String("path", e.FilePath), slog.String("err", e.Message)) } } } diff --git a/runtime/registry.go b/runtime/registry.go index 093a48e4c57..731a2dc5c7d 100644 --- a/runtime/registry.go +++ b/runtime/registry.go @@ -336,7 +336,7 @@ func (r *registryCache) restartController(iwc *instanceWithController) { r.logger.Info("syncing repo", zap.String("instance_id", iwc.instance.ID)) err := r.ensureRepoSync(iwc.ctx, iwc.instance.ID) if err != nil { - r.logger.Error("failed to sync repo", zap.String("instance_id", iwc.instance.ID), zap.Error(err)) + r.logger.Warn("failed to sync repo", zap.String("instance_id", iwc.instance.ID), zap.Error(err)) // Even if repo sync failed, we'll start the controller } else { r.logger.Info("repo synced", zap.String("instance_id", iwc.instance.ID)) From 2398551cac4032cf551145aaf7d574d412f73cf4 Mon Sep 17 00:00:00 2001 From: Dhiraj Barnwal Date: Fri, 15 Dec 2023 19:39:49 +0530 Subject: [PATCH 16/19] Fix negative SVG values in Graphic Context (#3673) --- .../time-series/TimeSeriesChartContainer.svelte | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/dashboards/time-series/TimeSeriesChartContainer.svelte b/web-common/src/features/dashboards/time-series/TimeSeriesChartContainer.svelte index 13d688e95cf..e28666dc3f9 100644 --- a/web-common/src/features/dashboards/time-series/TimeSeriesChartContainer.svelte +++ b/web-common/src/features/dashboards/time-series/TimeSeriesChartContainer.svelte @@ -26,12 +26,14 @@ A container GraphicContext for the time series in a metrics dashboard. right={50} fontSize={11} top={4} - width={(enableFullWidth - ? workspaceWidth - paddingForFullWidth - : workspaceWidth >= MEASURE_CONFIG.breakpoint - ? MEASURE_CONFIG.container.width.full - : MEASURE_CONFIG.container.width.breakpoint) - - MEASURE_CONFIG.bigNumber.widthWithChart} + width={Math.max( + enableFullWidth + ? workspaceWidth - paddingForFullWidth + : workspaceWidth >= MEASURE_CONFIG.breakpoint + ? MEASURE_CONFIG.container.width.full + : MEASURE_CONFIG.container.width.breakpoint, + 400 + ) - MEASURE_CONFIG.bigNumber.widthWithChart} xMax={end} xMaxTweenProps={{ duration: 400 }} xMin={start} From 7f2fef80233cffa148f27624c21df2fe649310d0 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Fri, 15 Dec 2023 20:05:52 +0530 Subject: [PATCH 17/19] Runtime: unified `duckdb` connector for `motherduck` and external `db` files (#3700) * external duckdb and motherduck * adding test and lint fix * lint fix * formatting fix * formatting fix * lint fix * review comments - fix defer res.Close() * fix defer res.Close() in other places --- runtime/drivers/duckdb/duckdb.go | 20 +++--- .../duckdb/transporter_duckDB_to_duckDB.go | 61 ++++++++++++++++- .../transporter_duckDB_to_duckDB_test.go | 65 +++++++++++++++++++ .../transporter_motherduck_to_duckDB.go | 12 ++-- .../sources/modal/AddSourceModal.svelte | 6 +- .../src/features/sources/modal/yupSchemas.ts | 3 +- .../src/features/sources/sourceUtils.ts | 8 +++ 7 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 78f57909285..046b61519d9 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -49,6 +49,14 @@ var spec = drivers.Spec{ Description: "DuckDB SQL query.", Placeholder: "select * from read_csv('data/file.csv', header=true);", }, + { + Key: "db", + Type: drivers.StringPropertyType, + Required: true, + DisplayName: "DB", + Description: "Path to external DuckDB database. Use md: for motherduckb.", + Placeholder: "/path/to/main.db or md:main.db(for motherduck)", + }, }, ConfigProperties: []drivers.PropertySchema{ { @@ -58,18 +66,6 @@ var spec = drivers.Spec{ } var motherduckSpec = drivers.Spec{ - DisplayName: "MotherDuck", - Description: "Import data from MotherDuck.", - SourceProperties: []drivers.PropertySchema{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "SQL", - Description: "Query to extract data from MotherDuck.", - Placeholder: "select * from my_db.my_table;", - }, - }, ConfigProperties: []drivers.PropertySchema{ { Key: "token", diff --git a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go index 50d5b2db51c..1e8e1331cd0 100644 --- a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go @@ -2,9 +2,12 @@ package duckdb import ( "context" + "database/sql" "errors" "fmt" "net/url" + "path/filepath" + "strings" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/duckdbsql" @@ -37,6 +40,10 @@ func (t *duckDBToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps map[s return err } + if srcCfg.Database != "" { // query to be run against an external DB + return t.transferFromExternalDB(ctx, srcCfg, sinkCfg, opts) + } + // We can't just pass the SQL statement to DuckDB outright. // We need to do some rewriting for certain table references (currently object stores and local files). @@ -90,16 +97,66 @@ func (t *duckDBToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps map[s // If the path is a local file reference, rewrite to a safe and repo-relative path. if uri.Scheme == "" && uri.Host == "" { - sql, err := rewriteLocalPaths(ast, opts.RepoRoot, opts.AllowHostAccess) + rewrittenSQL, err := rewriteLocalPaths(ast, opts.RepoRoot, opts.AllowHostAccess) if err != nil { return fmt.Errorf("invalid local path: %w", err) } - srcCfg.SQL = sql + srcCfg.SQL = rewrittenSQL } return t.to.CreateTableAsSelect(ctx, sinkCfg.Table, false, srcCfg.SQL) } +func (t *duckDBToDuckDB) transferFromExternalDB(ctx context.Context, srcProps *dbSourceProperties, sinkProps *sinkProperties, opts *drivers.TransferOptions) error { + return t.to.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { + res, err := t.to.Execute(ctx, &drivers.Statement{Query: "SELECT current_database(),current_schema();"}) + if err != nil { + return err + } + + var localDB, localSchema string + for res.Next() { + if err := res.Scan(&localDB, &localSchema); err != nil { + _ = res.Close() + return err + } + } + _ = res.Close() + + // duckdb considers everything before first . as db name + // alternative solution can be to query `show databases()` before and after to identify db name + dbName, _, _ := strings.Cut(filepath.Base(srcProps.Database), ".") + if dbName == "main" { + return fmt.Errorf("`main` is a reserved db name") + } + + if err = t.to.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH %s AS %s", safeSQLString(srcProps.Database), safeSQLName(dbName))}); err != nil { + return fmt.Errorf("failed to attach db %q: %w", srcProps.Database, err) + } + + defer func() { + if err = t.to.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("DETACH %s;", safeSQLName(dbName))}); err != nil { + t.logger.Error("failed to detach db", zap.Error(err)) + } + }() + + if err := t.to.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("USE %s;", safeName(dbName))}); err != nil { + return err + } + + defer func() { // revert back to localdb + if err = t.to.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("USE %s.%s;", safeName(localDB), safeName(localSchema))}); err != nil { + t.logger.Error("failed to switch to local database", zap.Error(err)) + } + }() + + userQuery := strings.TrimSpace(srcProps.SQL) + userQuery, _ = strings.CutSuffix(userQuery, ";") // trim trailing semi colon + query := fmt.Sprintf("CREATE OR REPLACE TABLE %s.%s.%s AS (%s\n);", safeName(localDB), safeName(localSchema), safeName(sinkProps.Table), userQuery) + return t.to.Exec(ctx, &drivers.Statement{Query: query}) + }) +} + // rewriteLocalPaths rewrites a DuckDB SQL statement such that relative paths become absolute paths relative to the basePath, // and if allowHostAccess is false, returns an error if any of the paths resolve to a path outside of the basePath. func rewriteLocalPaths(ast *duckdbsql.AST, basePath string, allowHostAccess bool) (string, error) { diff --git a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go new file mode 100644 index 00000000000..a750ec637c9 --- /dev/null +++ b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go @@ -0,0 +1,65 @@ +package duckdb + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/rilldata/rill/runtime/drivers" + activity "github.com/rilldata/rill/runtime/pkg/activity" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestDuckDBToDuckDBTransfer(t *testing.T) { + tempDir := t.TempDir() + conn, err := Driver{}.Open(map[string]any{"dsn": fmt.Sprintf("%s.db?access_mode=read_write", filepath.Join(tempDir, "tranfser"))}, false, activity.NewNoopClient(), zap.NewNop()) + require.NoError(t, err) + + olap, ok := conn.AsOLAP("") + require.True(t, ok) + + err = olap.Exec(context.Background(), &drivers.Statement{ + Query: "CREATE TABLE foo(bar VARCHAR, baz INTEGER)", + }) + require.NoError(t, err) + + err = olap.Exec(context.Background(), &drivers.Statement{ + Query: "INSERT INTO foo VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)", + }) + require.NoError(t, err) + require.NoError(t, conn.Close()) + + to, err := Driver{}.Open(map[string]any{"dsn": ""}, false, activity.NewNoopClient(), zap.NewNop()) + require.NoError(t, err) + + olap, _ = to.AsOLAP("") + + tr := NewDuckDBToDuckDB(olap, zap.NewNop()) + + // transfer once + err = tr.Transfer(context.Background(), map[string]any{"sql": "SELECT * FROM foo", "db": filepath.Join(tempDir, "tranfser.db")}, map[string]any{"table": "test"}, &drivers.TransferOptions{Progress: drivers.NoOpProgress{}}) + require.NoError(t, err) + + rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM test"}) + require.NoError(t, err) + + var count int + rows.Next() + require.NoError(t, rows.Scan(&count)) + require.Equal(t, 4, count) + require.NoError(t, rows.Close()) + + // transfer again + err = tr.Transfer(context.Background(), map[string]any{"sql": "SELECT * FROM foo", "db": filepath.Join(tempDir, "tranfser.db")}, map[string]any{"table": "test"}, &drivers.TransferOptions{Progress: drivers.NoOpProgress{}}) + require.NoError(t, err) + + rows, err = olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM test"}) + require.NoError(t, err) + + rows.Next() + require.NoError(t, rows.Scan(&count)) + require.Equal(t, 4, count) + require.NoError(t, rows.Close()) +} diff --git a/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go b/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go index b97198fe127..20d32d4cbc7 100644 --- a/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go @@ -61,16 +61,18 @@ func (t *motherduckToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps m if err != nil { return err } - defer res.Close() - res.Next() var localDB, localSchema string - if err := res.Scan(&localDB, &localSchema); err != nil { - return err + for res.Next() { + if err := res.Scan(&localDB, &localSchema); err != nil { + _ = res.Close() + return err + } } + _ = res.Close() // get token - token := config["token"] + token, _ := config["token"].(string) if token == "" && config["allow_host_access"].(bool) { token = os.Getenv("motherduck_token") } diff --git a/web-common/src/features/sources/modal/AddSourceModal.svelte b/web-common/src/features/sources/modal/AddSourceModal.svelte index 5c599d6d805..bc6f96c6d9f 100644 --- a/web-common/src/features/sources/modal/AddSourceModal.svelte +++ b/web-common/src/features/sources/modal/AddSourceModal.svelte @@ -12,7 +12,7 @@ import Https from "../../../components/icons/connectors/HTTPS.svelte"; import LocalFile from "../../../components/icons/connectors/LocalFile.svelte"; import MicrosoftAzureBlobStorage from "../../../components/icons/connectors/MicrosoftAzureBlobStorage.svelte"; - import MotherDuck from "../../../components/icons/connectors/MotherDuck.svelte"; + import DuckDB from "../../../components/icons/connectors/DuckDB.svelte"; import Postgres from "../../../components/icons/connectors/Postgres.svelte"; import Snowflake from "../../../components/icons/connectors/Snowflake.svelte"; import SQLite from "../../../components/icons/connectors/SQLite.svelte"; @@ -40,7 +40,7 @@ // duckdb "bigquery", "athena", - "motherduck", + "duckdb", "postgres", "sqlite", "snowflake", @@ -55,7 +55,7 @@ // duckdb: DuckDB, bigquery: GoogleBigQuery, athena: AmazonAthena, - motherduck: MotherDuck, + duckdb: DuckDB, postgres: Postgres, sqlite: SQLite, snowflake: Snowflake, diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 108f23d7b97..6412c5c3fb5 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -46,9 +46,10 @@ export function getYupSchema(connector: V1ConnectorSpec) { ) .required("Source name is required"), }); - case "motherduck": + case "duckdb": return yup.object().shape({ sql: yup.string().required("sql is required"), + db: yup.string().required("db is required"), sourceName: yup .string() .matches( diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 7fec9dd0350..d96091fe266 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -39,6 +39,14 @@ export function compileCreateSourceYAML( delete values.db; delete values.table; break; + case "duckdb": { + const db = values.db as string; + if (db.startsWith("md:")) { + connectorName = "motherduck"; + values.db = db.replace("md:", ""); + } + break; + } } const compiledKeyValues = Object.entries(values) From cec766eb6b2b1d6fa7115b8992e88ad449aaaba8 Mon Sep 17 00:00:00 2001 From: Brian Holmes <120223836+briangregoryholmes@users.noreply.github.com> Date: Mon, 18 Dec 2023 01:20:42 -0500 Subject: [PATCH 18/19] support for Add Filter button (#3671) * rework of footer to match new design and improve readability * adding filter button functionality, plus design tweaks to relevant menus * new icon * chip should not be removed when deselecting the only selected value * removing limit from filter list query * changed searchedValues to allValues, simplified display logic * updated tests to meet updated requirements * prettier unused vars build fix * remove keep-alive event * update let directive for named slot, add timeout to dashboard.spec * remove unused function * change to active logic when mounting chip component * resolve name collision * update page wait from timeout to selector * added specific method for adding a dimension name without a value * only unselected dimension names are shown in the add filter dropdown * add back limit * toggleDimensionNameSelection is now actually a toggling function * dispatch remove event rather than handle remove directly, plus bind to active property * use state manager action and prop cleanup * add back limit --- .../chip/removable-list-chip/Footer.svelte | 21 +++- .../RemovableListChip.svelte | 45 +++++--- .../RemovableListMenu.spec.ts | 53 ++++----- .../RemovableListMenu.svelte | 78 ++++++------- .../src/components/icons/ChevronRight.svelte | 19 ++++ .../src/components/search/Search.svelte | 11 +- .../SearchableFilterDropdown.svelte | 5 +- .../src/features/dashboards/actions/index.ts | 6 - .../dashboards/filters/FilterButton.svelte | 84 ++++++++++++++ .../dashboards/filters/Filters.svelte | 107 ++++++++++-------- .../features/dashboards/selectors/index.ts | 2 +- .../actions/dimension-filters.ts | 29 +++++ web-local/test/ui/dashboards.spec.ts | 2 + 13 files changed, 309 insertions(+), 153 deletions(-) create mode 100644 web-common/src/components/icons/ChevronRight.svelte create mode 100644 web-common/src/features/dashboards/filters/FilterButton.svelte diff --git a/web-common/src/components/chip/removable-list-chip/Footer.svelte b/web-common/src/components/chip/removable-list-chip/Footer.svelte index 1ba6c126b93..0eebe05a980 100644 --- a/web-common/src/components/chip/removable-list-chip/Footer.svelte +++ b/web-common/src/components/chip/removable-list-chip/Footer.svelte @@ -1,5 +1,18 @@ -
+
-
+ + + diff --git a/web-common/src/components/chip/removable-list-chip/RemovableListChip.svelte b/web-common/src/components/chip/removable-list-chip/RemovableListChip.svelte index c34207ef3d0..dfbbb9499a5 100644 --- a/web-common/src/components/chip/removable-list-chip/RemovableListChip.svelte +++ b/web-common/src/components/chip/removable-list-chip/RemovableListChip.svelte @@ -11,9 +11,9 @@ the name and then move the cursor to the right to cancel it. existing elements in the lib as well as changing the type (include, exclude) and enabling list search. The implementation of these parts are details left to the consumer of the component; this component should remain pure-ish (only internal state) if possible. --> - + @@ -55,7 +68,10 @@ are details left to the consumer of the component; this component should remain > { + toggleFloatingElement(); + dispatch("click"); + }} on:remove={() => dispatch("remove")} {active} {...colors} @@ -91,15 +107,14 @@ are details left to the consumer of the component; this component should remain
diff --git a/web-common/src/components/chip/removable-list-chip/RemovableListMenu.spec.ts b/web-common/src/components/chip/removable-list-chip/RemovableListMenu.spec.ts index 8f3f50f1310..35ee88a02e4 100644 --- a/web-common/src/components/chip/removable-list-chip/RemovableListMenu.spec.ts +++ b/web-common/src/components/chip/removable-list-chip/RemovableListMenu.spec.ts @@ -1,28 +1,31 @@ import RemovableListMenu from "./RemovableListMenu.svelte"; import { describe, it, expect, vi } from "vitest"; import { render, waitFor, fireEvent, screen } from "@testing-library/svelte"; -import { writable } from "svelte/store"; describe("RemovableListMenu", () => { - it("renders selected values by default", async () => { + it("does not render selected values if not in all values", async () => { const { unmount } = render(RemovableListMenu, { - excludeStore: writable(false), - selectedValues: ["foo", "bar"], - searchedValues: null, + excludeMode: false, + selectedValues: ["x"], + allValues: ["foo", "bar"], }); const foo = screen.getByText("foo"); const bar = screen.getByText("bar"); expect(foo).toBeDefined(); expect(bar).toBeDefined(); + + const x = screen.queryByText("x"); + expect(x).toBeNull(); + unmount(); }); - it("renders selected values if search text is empty", async () => { + it("renders all values if search text is empty", async () => { const { unmount } = render(RemovableListMenu, { - excludeStore: writable(false), - selectedValues: ["foo", "bar"], - searchedValues: ["x"], + excludeMode: false, + selectedValues: [], + allValues: ["foo", "bar"], }); const foo = screen.getByText("foo"); @@ -34,9 +37,9 @@ describe("RemovableListMenu", () => { it("renders search values if search text is populated", async () => { const { unmount } = render(RemovableListMenu, { - excludeStore: writable(false), + excludeMode: false, selectedValues: ["foo", "bar"], - searchedValues: ["x"], + allValues: ["x"], }); const searchInput = screen.getByRole("textbox", { name: "Search list" }); @@ -51,35 +54,33 @@ describe("RemovableListMenu", () => { }); it("should render switch based on exclude store", async () => { - const excludeStore = writable(false); - const { unmount } = render(RemovableListMenu, { - excludeStore, + const { unmount, component } = render(RemovableListMenu, { + excludeMode: false, selectedValues: ["foo", "bar"], - searchedValues: ["x"], + allValues: ["x"], }); - const switchInput = screen.getByRole("switch"); - expect(switchInput.checked).toBe(false); + const switchInput = screen.getByText("Exclude"); + expect(switchInput).toBeDefined(); - excludeStore.set(true); - await waitFor(() => { - expect(switchInput.checked).toBe(true); - }); + await component.$set({ excludeMode: true }); + + const includeButton = screen.getByText("Include"); + expect(includeButton).toBeDefined(); unmount(); }); it("should dispatch toggle, apply, and search events", async () => { - const excludeStore = writable(false); const { unmount, component } = render(RemovableListMenu, { - excludeStore, - selectedValues: ["foo", "bar"], - searchedValues: ["x"], + excludeMode: false, + selectedValues: [], + allValues: ["foo", "bar"], }); const toggleSpy = vi.fn(); component.$on("toggle", toggleSpy); - const switchInput = screen.getByRole("switch"); + const switchInput = screen.getByText("Exclude"); await fireEvent.click(switchInput); expect(toggleSpy).toHaveBeenCalledOnce(); diff --git a/web-common/src/components/chip/removable-list-chip/RemovableListMenu.svelte b/web-common/src/components/chip/removable-list-chip/RemovableListMenu.svelte index 006e5a2a951..245e49f389c 100644 --- a/web-common/src/components/chip/removable-list-chip/RemovableListMenu.svelte +++ b/web-common/src/components/chip/removable-list-chip/RemovableListMenu.svelte @@ -1,17 +1,16 @@ - -
+
- {#if valuesToDisplay.length} - {#each valuesToDisplay as value} + {#if allValues?.length} + {#each allValues.sort() as value} - {#if selectedValues.includes(value) && !$excludeStore} + {#if selectedValues.includes(value) && !excludeMode} - {:else if selectedValues.includes(value) && $excludeStore} + {:else if selectedValues.includes(value) && excludeMode} {:else} @@ -95,7 +80,7 @@ {#if value?.length > 240} {value.slice(0, 240)}... @@ -110,17 +95,20 @@ {/if}
- - onToggleHandler()} checked={$excludeStore}> + + +
diff --git a/web-common/src/components/icons/ChevronRight.svelte b/web-common/src/components/icons/ChevronRight.svelte new file mode 100644 index 00000000000..2ea1015102b --- /dev/null +++ b/web-common/src/components/icons/ChevronRight.svelte @@ -0,0 +1,19 @@ + + + + + diff --git a/web-common/src/components/search/Search.svelte b/web-common/src/components/search/Search.svelte index 1dc6de3b200..76637c22832 100644 --- a/web-common/src/components/search/Search.svelte +++ b/web-common/src/components/search/Search.svelte @@ -38,9 +38,8 @@ bind:this={ref} type="text" autocomplete="off" - class="bg-white border border-gray-200 {showBorderOnFocus - ? 'focus:border-blue-400' - : ''} outline-none rounded-sm block w-full pl-8 p-1" + class:focus={showBorderOnFocus} + class="bg-slate-50 border border-gray-200 outline-none rounded-sm block w-full pl-8 p-1" {placeholder} bind:value on:input @@ -48,3 +47,9 @@ aria-label={label} /> + + diff --git a/web-common/src/components/searchable-filter-menu/SearchableFilterDropdown.svelte b/web-common/src/components/searchable-filter-menu/SearchableFilterDropdown.svelte index fd1aabba432..9b27a5cced4 100644 --- a/web-common/src/components/searchable-filter-menu/SearchableFilterDropdown.svelte +++ b/web-common/src/components/searchable-filter-menu/SearchableFilterDropdown.svelte @@ -1,7 +1,6 @@ + + + + + + + Add filter + + + { + toggleFloatingElement(); + toggleDimensionNameSelection(e.detail.name); + }} + /> + + + diff --git a/web-common/src/features/dashboards/filters/Filters.svelte b/web-common/src/features/dashboards/filters/Filters.svelte index a159fa90fdb..505edb3e07a 100644 --- a/web-common/src/features/dashboards/filters/Filters.svelte +++ b/web-common/src/features/dashboards/filters/Filters.svelte @@ -25,13 +25,18 @@ The main feature-set component for dashboard filters import { getStateManagers } from "../state-managers/state-managers"; import { clearAllFilters, - clearFilterForDimension, toggleDimensionValue, toggleFilterMode, } from "../actions"; + import FilterButton from "./FilterButton.svelte"; const StateManagers = getStateManagers(); - const { dashboardStore } = StateManagers; + const { + dashboardStore, + actions: { + dimensionsFilter: { toggleDimensionNameSelection }, + }, + } = StateManagers; /** the height of a row of chips */ const ROW_HEIGHT = "26px"; @@ -47,23 +52,25 @@ The main feature-set component for dashboard filters } let searchText = ""; - let searchedValues: string[] | null = null; - let activeDimensionName; + let allValues: string[] | null = null; + let activeDimensionName: string; + $: activeColumn = dimensions.find((d) => d.name === activeDimensionName)?.column ?? activeDimensionName; let topListQuery: ReturnType | undefined; - $: if (activeDimensionName) + $: if (activeDimensionName) { topListQuery = getFilterSearchList(StateManagers, { dimension: activeDimensionName, searchText, addNull: "null".includes(searchText), }); + } - $: if (!$topListQuery?.isFetching && searchText != "") { + $: if (!$topListQuery?.isFetching) { const topListData = $topListQuery?.data?.data ?? []; - searchedValues = topListData.map((datum) => datum[activeColumn]) ?? []; + allValues = topListData.map((datum) => datum[activeColumn]) ?? []; } $: hasFilters = isFiltered($dashboardStore.filters); @@ -120,7 +127,7 @@ The main feature-set component for dashboard filters currentDimensionFilters.sort((a, b) => (a.name > b.name ? 1 : -1)); } - function setActiveDimension(name, value) { + function setActiveDimension(name, value = "") { activeDimensionName = name; searchText = value; } @@ -144,31 +151,42 @@ The main feature-set component for dashboard filters >
- {#if currentDimensionFilters.length > 0} - + + + {#if !currentDimensionFilters.length} +
+ No filters selected +
+ {:else} {#each currentDimensionFilters as { name, label, selectedValues, filterType } (name)} {@const isInclude = filterType === "include"}
toggleFilterMode(StateManagers, name)} - on:remove={() => - clearFilterForDimension( - StateManagers, - name, - isInclude ? true : false - )} - on:apply={(event) => - toggleDimensionValue(StateManagers, name, event.detail)} + on:remove={() => toggleDimensionNameSelection(name)} + on:apply={(event) => { + toggleDimensionValue(StateManagers, name, event.detail); + }} on:search={(event) => { setActiveDimension(name, event.detail); }} + on:click={() => { + setActiveDimension(name, ""); + }} + on:mount={() => { + setActiveDimension(name); + }} typeLabel="dimension" name={isInclude ? label : `Exclude ${label}`} excludeMode={isInclude ? false : true} colors={getColorForChip(isInclude)} label="View filter" {selectedValues} - {searchedValues} + {allValues} > Click to edit the the filters in this dimension @@ -176,35 +194,26 @@ The main feature-set component for dashboard filters
{/each} - - {#if hasFilters} -
- clearAllFilters(StateManagers)} - > - - - - Clear filters - -
- {/if} - - {:else if currentDimensionFilters.length === 0} -
- No filters selected -
- {:else} -   - {/if} + {#if hasFilters} +
+ clearAllFilters(StateManagers)} + > + + + + Clear filters + +
+ {/if} + diff --git a/web-common/src/features/dashboards/selectors/index.ts b/web-common/src/features/dashboards/selectors/index.ts index dd9f11f8bd5..f448754a770 100644 --- a/web-common/src/features/dashboards/selectors/index.ts +++ b/web-common/src/features/dashboards/selectors/index.ts @@ -73,7 +73,7 @@ export const getFilterSearchList = ( measureNames: [metricsExplorer.leaderboardMeasureName], timeStart: timeControls.timeStart, timeEnd: timeControls.timeEnd, - limit: "15", + limit: "100", offset: "0", sort: [], filter: { diff --git a/web-common/src/features/dashboards/state-managers/actions/dimension-filters.ts b/web-common/src/features/dashboards/state-managers/actions/dimension-filters.ts index f163d81720f..2c56935608b 100644 --- a/web-common/src/features/dashboards/state-managers/actions/dimension-filters.ts +++ b/web-common/src/features/dashboards/state-managers/actions/dimension-filters.ts @@ -38,6 +38,34 @@ export function toggleDimensionValueSelection( } } +export function toggleDimensionNameSelection( + { dashboard, cancelQueries }: DashboardMutables, + dimensionName: string +) { + const filters = filtersForCurrentExcludeMode({ dashboard })(dimensionName); + // if there are no filters at this point we cannot update anything. + if (filters === undefined) { + return; + } + + // if we are able to update the filters, we must cancel any queries + // that are currently running. + cancelQueries(); + + const filterIndex = filters.findIndex( + (filter) => filter.name === dimensionName + ); + + if (filterIndex === -1) { + filters.push({ + name: dimensionName, + in: [], + }); + } else { + filters.splice(filterIndex, 1); + } +} + export const dimensionFilterActions = { /** * Toggles whether the given dimension value is selected in the @@ -48,4 +76,5 @@ export const dimensionFilterActions = { * the include/exclude mode is a toggle for the entire dimension. */ toggleDimensionValueSelection, + toggleDimensionNameSelection, }; diff --git a/web-local/test/ui/dashboards.spec.ts b/web-local/test/ui/dashboards.spec.ts index 3d10daf997b..79cd8bee52c 100644 --- a/web-local/test/ui/dashboards.spec.ts +++ b/web-local/test/ui/dashboards.spec.ts @@ -245,6 +245,8 @@ test.describe("dashboard", () => { // Filter to Facebook via leaderboard await page.getByRole("button", { name: "Facebook 19.3k" }).click(); + await page.waitForSelector("text=Publisher Facebook"); + // Change filter to excluded await page.getByText("Publisher Facebook").click(); await page.getByRole("button", { name: "Exclude" }).click(); From edafbe6978d2e93f74d09f8886a6bd574b8c5e29 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 18 Dec 2023 14:08:54 +0530 Subject: [PATCH 19/19] remove order by true from toplist api (#3711) --- .../queries/metricsview_comparison_toplist.go | 101 ++++++++++-------- 1 file changed, 58 insertions(+), 43 deletions(-) diff --git a/runtime/queries/metricsview_comparison_toplist.go b/runtime/queries/metricsview_comparison_toplist.go index ddf617e3284..29d365aadb3 100644 --- a/runtime/queries/metricsview_comparison_toplist.go +++ b/runtime/queries/metricsview_comparison_toplist.go @@ -301,26 +301,32 @@ func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsView args = append(args, clauseArgs...) } - orderClause := "true" + var orderClauses []string for _, s := range q.Sort { if s.Name == q.DimensionName { - orderClause += ", 1" + clause := "1" if s.Desc { - orderClause += " DESC" + clause += " DESC" } if dialect == drivers.DialectDuckDB { - orderClause += " NULLS LAST" + clause += " NULLS LAST" } + orderClauses = append(orderClauses, clause) break } - orderClause += ", " - orderClause += safeName(s.Name) + clause := safeName(s.Name) if s.Desc { - orderClause += " DESC" + clause += " DESC" } if dialect == drivers.DialectDuckDB { - orderClause += " NULLS LAST" + clause += " NULLS LAST" } + orderClauses = append(orderClauses, clause) + } + + orderByClause := "" + if len(orderClauses) > 0 { + orderByClause = "ORDER BY " + strings.Join(orderClauses, ", ") } limitClause := "" @@ -337,12 +343,12 @@ func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsView if export { labelSelectClause := strings.Join(labelCols, ", ") sql = fmt.Sprintf( - `SELECT %[9]s FROM (SELECT %[1]s FROM %[3]s %[8]s WHERE %[4]s GROUP BY %[2]s ORDER BY %[5]s %[6]s OFFSET %[7]d)`, + `SELECT %[9]s FROM (SELECT %[1]s FROM %[3]s %[8]s WHERE %[4]s GROUP BY %[2]s %[5]s %[6]s OFFSET %[7]d)`, selectClause, // 1 groupByCol, // 2 safeName(mv.Table), // 3 baseWhereClause, // 4 - orderClause, // 5 + orderByClause, // 5 limitClause, // 6 q.Offset, // 7 unnestClause, // 8 @@ -350,12 +356,12 @@ func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsView ) } else { sql = fmt.Sprintf( - `SELECT %[1]s FROM %[3]s %[8]s WHERE %[4]s GROUP BY %[2]s ORDER BY %[5]s %[6]s OFFSET %[7]d`, + `SELECT %[1]s FROM %[3]s %[8]s WHERE %[4]s GROUP BY %[2]s %[5]s %[6]s OFFSET %[7]d`, selectClause, // 1 groupByCol, // 2 safeName(mv.Table), // 3 baseWhereClause, // 4 - orderClause, // 5 + orderByClause, // 5 limitClause, // 6 q.Offset, // 7 unnestClause, // 8 @@ -515,12 +521,12 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M return "", nil, err } - orderClause := "true" - subQueryOrderClause := "true" + var orderClauses []string + var subQueryOrderClauses []string for _, s := range q.Sort { if s.Name == q.DimensionName { - orderClause += ", 1" - subQueryOrderClause += ", 1" + clause := "1" + subQueryClause := "1" var ending string if s.Desc { ending += " DESC" @@ -528,16 +534,17 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M if dialect == drivers.DialectDuckDB { ending += " NULLS LAST" } - orderClause += ending - subQueryOrderClause += ending + clause += ending + subQueryClause += ending + orderClauses = append(orderClauses, clause) + subQueryOrderClauses = append(subQueryOrderClauses, subQueryClause) break } i, ok := measureMap[s.Name] if !ok { return "", nil, fmt.Errorf("metrics view '%s' doesn't contain '%s' sort column", q.MetricsViewName, s.Name) } - orderClause += ", " - subQueryOrderClause += ", " + var pos int switch s.Type { case runtimev1.MetricsViewComparisonSortType_METRICS_VIEW_COMPARISON_SORT_TYPE_BASE_VALUE: @@ -551,8 +558,8 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M default: return "", nil, fmt.Errorf("undefined sort type for measure %s", s.Name) } - orderClause += fmt.Sprint(pos) - subQueryOrderClause += fmt.Sprint(i + 2) // 1-based + skip the first dim column + orderClause := fmt.Sprint(pos) + subQueryOrderClause := fmt.Sprint(i + 2) // 1-based + skip the first dim column ending := "" if s.Desc { ending += " DESC" @@ -562,6 +569,15 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M } orderClause += ending subQueryOrderClause += ending + orderClauses = append(orderClauses, orderClause) + subQueryOrderClauses = append(subQueryOrderClauses, subQueryOrderClause) + } + + orderByClause := "" + subQueryOrderByClause := "" + if len(orderClauses) > 0 { + orderByClause = "ORDER BY " + strings.Join(orderClauses, ", ") + subQueryOrderByClause = "ORDER BY " + strings.Join(subQueryOrderClauses, ", ") } limitClause := "" @@ -588,13 +604,13 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M if q.Sort[0].Type == runtimev1.MetricsViewComparisonSortType_METRICS_VIEW_COMPARISON_SORT_TYPE_BASE_VALUE || deltaComparison { joinType = "LEFT OUTER" - baseLimitClause = fmt.Sprintf("ORDER BY %s", subQueryOrderClause) + baseLimitClause = subQueryOrderByClause if approximationLimit > 0 { baseLimitClause += fmt.Sprintf(" LIMIT %d", approximationLimit) } } else if q.Sort[0].Type == runtimev1.MetricsViewComparisonSortType_METRICS_VIEW_COMPARISON_SORT_TYPE_COMPARISON_VALUE { joinType = "RIGHT OUTER" - comparisonLimitClause = fmt.Sprintf("ORDER BY %s", subQueryOrderClause) + comparisonLimitClause = subQueryOrderByClause if approximationLimit > 0 { comparisonLimitClause += fmt.Sprintf(" LIMIT %d", approximationLimit) } @@ -641,8 +657,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M ) comparison ON base.%[2]s = comparison.%[2]s OR (base.%[2]s is null and comparison.%[2]s is null) - ORDER BY - %[6]s + %[6]s %[7]s OFFSET %[8]d @@ -652,7 +667,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M safeName(mv.Table), // 3 baseWhereClause, // 4 comparisonWhereClause, // 5 - orderClause, // 6 + orderByClause, // 6 limitClause, // 7 q.Offset, // 8 finalSelectClause, // 9 @@ -716,30 +731,30 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M sql = fmt.Sprintf(` WITH %[11]s AS ( - SELECT %[1]s FROM %[3]s WHERE %[4]s GROUP BY %[2]s ORDER BY %[13]s %[10]s OFFSET %[8]d + SELECT %[1]s FROM %[3]s WHERE %[4]s GROUP BY %[2]s %[13]s %[10]s OFFSET %[8]d ), %[12]s AS ( SELECT %[1]s FROM %[3]s WHERE %[5]s AND %[2]s IN (SELECT %[2]s FROM %[11]s) GROUP BY %[2]s %[10]s ) SELECT %[11]s.%[2]s AS %[14]s, %[9]s FROM %[11]s LEFT JOIN %[12]s ON base.%[2]s = comparison.%[2]s GROUP BY 1 - ORDER BY %[6]s + %[6]s %[7]s OFFSET %[8]d `, - subSelectClause, // 1 - colName, // 2 - safeName(mv.Table), // 3 - leftWhereClause, // 4 - rightWhereClause, // 5 - orderClause, // 6 - limitClause, // 7 - q.Offset, // 8 - finalSelectClause, // 9 - twiceTheLimitClause, // 10 - leftSubQueryAlias, // 11 - rightSubQueryAlias, // 12 - subQueryOrderClause, // 13 - finalDimName, // 14 + subSelectClause, // 1 + colName, // 2 + safeName(mv.Table), // 3 + leftWhereClause, // 4 + rightWhereClause, // 5 + orderByClause, // 6 + limitClause, // 7 + q.Offset, // 8 + finalSelectClause, // 9 + twiceTheLimitClause, // 10 + leftSubQueryAlias, // 11 + rightSubQueryAlias, // 12 + subQueryOrderByClause, // 13 + finalDimName, // 14 ) }