diff --git a/Makefile b/Makefile index b81c4caaf0ae4..c212efd7aec00 100644 --- a/Makefile +++ b/Makefile @@ -372,7 +372,7 @@ clean: milvus-tools: print-build-info @echo "Building tools ..." - @mkdir -p $(INSTALL_PATH)/tools && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build \ + @. $(PWD)/scripts/setenv.sh && mkdir -p $(INSTALL_PATH)/tools && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build \ -pgo=$(PGO_PATH)/default.pgo -ldflags="-X 'main.BuildTags=$(BUILD_TAGS)' -X 'main.BuildTime=$(BUILD_TIME)' -X 'main.GitCommit=$(GIT_COMMIT)' -X 'main.GoVersion=$(GO_VERSION)'" \ -o $(INSTALL_PATH)/tools $(PWD)/cmd/tools/* 1>/dev/null diff --git a/cmd/tools/config-docs-generator/main.go b/cmd/tools/config-docs-generator/main.go index 7463e8753d66a..7a7163d5c9e58 100644 --- a/cmd/tools/config-docs-generator/main.go +++ b/cmd/tools/config-docs-generator/main.go @@ -217,6 +217,9 @@ func (s Section) sectionPageContent() string { ret += fmt.Sprintf("# %s-related Configurations"+mdNextLine, s.Name) ret += s.descriptionContent() + mdNextLine for _, field := range s.Fields { + if len(field.Description) == 0 || field.Description[0] == "" { + continue + } ret += field.sectionPageContent() + mdNextLine } @@ -248,9 +251,6 @@ const fieldTableTemplate = ` func (f Field) sectionPageContent() string { ret := fmt.Sprintf("## `%s`", f.Name) + mdNextLine desp := f.descriptionContent() - if len(desp) > 0 { - desp = "\n" + desp + " " - } ret += fmt.Sprintf(fieldTableTemplate, f.Name, desp, f.DefaultValue) return ret } @@ -258,11 +258,13 @@ func (f Field) sectionPageContent() string { func (f Field) descriptionContent() string { var ret string lines := len(f.Description) - for i, descLine := range f.Description { - ret += fmt.Sprintf("
  • %s
    • ", descLine) - if i < lines-1 { - ret += "\n" + if lines > 1 { + for _, descLine := range f.Description { + ret += fmt.Sprintf("\n
  • %s
    • ", descLine) } + } else { + ret = fmt.Sprintf(" %s ", f.Description[0]) } + return ret } diff --git a/cmd/tools/config/generate_test.go b/cmd/tools/config/generate_test.go index 485476bfc69aa..43665ac766d4d 100644 --- a/cmd/tools/config/generate_test.go +++ b/cmd/tools/config/generate_test.go @@ -16,6 +16,7 @@ import ( "bytes" "fmt" "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -30,20 +31,26 @@ import ( // Please be noted that milvus.yaml is generated by code, so don't edit it directly, instead, change the code in paramtable // and run `make milvus-tools && ./bin/tools/config gen-yaml && mv milvus.yaml configs/milvus.yaml`. func TestYamlFile(t *testing.T) { + log.SetLevel(zap.InfoLevel) w := bytes.Buffer{} WriteYaml(&w) base := paramtable.NewBaseTable() f, err := os.Open(fmt.Sprintf("%s/%s", base.GetConfigDir(), "milvus.yaml")) assert.NoError(t, err, "expecting configs/milvus.yaml") + log.Info("Verifying config", zap.String("file", f.Name())) defer f.Close() fileScanner := bufio.NewScanner(f) codeScanner := bufio.NewScanner(&w) + for fileScanner.Scan() && codeScanner.Scan() { + if strings.Contains(codeScanner.Text(), "etcd:") || strings.Contains(codeScanner.Text(), "minio:") || strings.Contains(codeScanner.Text(), "pulsar:") { + // Skip check of endpoints given by .env + continue + } if fileScanner.Text() != codeScanner.Text() { assert.FailNow(t, fmt.Sprintf("configs/milvus.yaml is not consistent with paramtable, file: [%s], code: [%s]. Do not edit milvus.yaml directly.", fileScanner.Text(), codeScanner.Text())) } - log.Error("", zap.Any("file", fileScanner.Text()), zap.Any("code", codeScanner.Text())) } } diff --git a/configs/milvus.yaml b/configs/milvus.yaml index c03a5c56bbf6a..913c4d3a868bf 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -16,10 +16,24 @@ # Related configuration of etcd, used to store Milvus metadata & service discovery. etcd: + # Endpoints used to access etcd service. You can change this parameter as the endpoints of your own etcd cluster. + # Environment variable: ETCD_ENDPOINTS + # etcd preferentially acquires valid address from environment variable ETCD_ENDPOINTS when Milvus is started. endpoints: localhost:2379 - rootPath: by-dev # The root path where data is stored in etcd - metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath - kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath + # Root prefix of the key to where Milvus stores data in etcd. + # It is recommended to change this parameter before starting Milvus for the first time. + # To share an etcd instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. + # Set an easy-to-identify root path for Milvus if etcd service already exists. + # Changing this for an already running Milvus instance may result in failures to read legacy data. + rootPath: by-dev + # Sub-prefix of the key to where Milvus stores metadata-related information in etcd. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended to change this parameter before starting Milvus for the first time. + metaSubPath: meta + # Sub-prefix of the key to where Milvus stores timestamps in etcd. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended not to change this parameter if there is no specific reason. + kvSubPath: kv log: level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. # path is one of: @@ -68,20 +82,49 @@ tikv: tlsCACert: # path to your CACert file localStorage: - path: /var/lib/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/ + # Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended to change this parameter before starting Milvus for the first time. + path: /var/lib/milvus/data/ # Related configuration of MinIO/S3/GCS or any other service supports S3 API, which is responsible for data persistence for Milvus. # We refer to the storage service as MinIO/S3 in the following description for simplicity. minio: - address: localhost # Address of MinIO/S3 - port: 9000 # Port of MinIO/S3 - accessKeyID: minioadmin # accessKeyID of MinIO/S3 - secretAccessKey: minioadmin # MinIO/S3 encryption string - useSSL: false # Access to MinIO/S3 with SSL + # IP address of MinIO or S3 service. + # Environment variable: MINIO_ADDRESS + # minio.address and minio.port together generate the valid access to MinIO or S3 service. + # MinIO preferentially acquires the valid IP address from the environment variable MINIO_ADDRESS when Milvus is started. + # Default value applies when MinIO or S3 is running on the same network with Milvus. + address: localhost + port: 9000 # Port of MinIO or S3 service. + # Access key ID that MinIO or S3 issues to user for authorized access. + # Environment variable: MINIO_ACCESS_KEY_ID or minio.accessKeyID + # minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. + # This configuration must be set identical to the environment variable MINIO_ACCESS_KEY_ID, which is necessary for starting MinIO or S3. + # The default value applies to MinIO or S3 service that started with the default docker-compose.yml file. + accessKeyID: minioadmin + # Secret key used to encrypt the signature string and verify the signature string on server. It must be kept strictly confidential and accessible only to the MinIO or S3 server and users. + # Environment variable: MINIO_SECRET_ACCESS_KEY or minio.secretAccessKey + # minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. + # This configuration must be set identical to the environment variable MINIO_SECRET_ACCESS_KEY, which is necessary for starting MinIO or S3. + # The default value applies to MinIO or S3 service that started with the default docker-compose.yml file. + secretAccessKey: minioadmin + useSSL: false # Switch value to control if to access the MinIO or S3 service through SSL. ssl: tlsCACert: /path/to/public.crt # path to your CACert file - bucketName: a-bucket # Bucket name in MinIO/S3 - rootPath: files # The root path where the message is stored in MinIO/S3 + # Name of the bucket where Milvus stores data in MinIO or S3. + # Milvus 2.0.0 does not support storing data in multiple buckets. + # Bucket with this name will be created if it does not exist. If the bucket already exists and is accessible, it will be used directly. Otherwise, there will be an error. + # To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. + # The data will be stored in the local Docker if Docker is used to start the MinIO service locally. Ensure that there is sufficient storage space. + # A bucket name is globally unique in one MinIO or S3 instance. + bucketName: a-bucket + # Root prefix of the key to where Milvus stores data in MinIO or S3. + # It is recommended to change this parameter before starting Milvus for the first time. + # To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. + # Set an easy-to-identify root key prefix for Milvus if etcd service already exists. + # Changing this for an already running Milvus instance may result in failures to read legacy data. + rootPath: files # Whether to useIAM role to access S3/GCS instead of access/secret keys # For more information, refer to # aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html @@ -126,12 +169,22 @@ mq: # Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services. pulsar: - address: localhost # Address of pulsar - port: 6650 # Port of Pulsar - webport: 80 # Web port of pulsar, if you connect directly without proxy, should use 8080 - maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar. + # IP address of Pulsar service. + # Environment variable: PULSAR_ADDRESS + # pulsar.address and pulsar.port together generate the valid access to Pulsar. + # Pulsar preferentially acquires the valid IP address from the environment variable PULSAR_ADDRESS when Milvus is started. + # Default value applies when Pulsar is running on the same network with Milvus. + address: localhost + port: 6650 # Port of Pulsar service. + webport: 80 # Web port of of Pulsar service. If you connect direcly without proxy, should use 8080. + # The maximum size of each message in Pulsar. Unit: Byte. + # By default, Pulsar can transmit at most 5 MB of data in a single message. When the size of inserted data is greater than this value, proxy fragments the data into multiple messages to ensure that they can be transmitted correctly. + # If the corresponding parameter in Pulsar remains unchanged, increasing this configuration will cause Milvus to fail, and reducing it produces no advantage. + maxMessageSize: 5242880 + # Pulsar can be provisioned for specific tenants with appropriate capacity allocated to the tenant. + # To share a Pulsar instance among multiple Milvus instances, you can change this to an Pulsar tenant rather than the default one for each Milvus instance before you start them. However, if you do not want Pulsar multi-tenancy, you are advised to change msgChannel.chanNamePrefix.cluster to the different value. tenant: public - namespace: default + namespace: default # A Pulsar namespace is the administrative unit nomenclature within a tenant. requestTimeout: 60 # pulsar client global request timeout in seconds enableClientMetrics: false # Whether to register pulsar client metrics into milvus metrics path. @@ -151,21 +204,23 @@ pulsar: # readTimeout: 10 rocksmq: - # The path where the message is stored in rocksmq - # please adjust in embedded Milvus: /tmp/milvus/rdb_data + # Prefix of the key to where Milvus stores data in RocksMQ. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended to change this parameter before starting Milvus for the first time. + # Set an easy-to-identify root key prefix for Milvus if etcd service already exists. path: /var/lib/milvus/rdb_data lrucacheratio: 0.06 # rocksdb cache memory ratio - rocksmqPageSize: 67108864 # 64 MB, 64 * 1024 * 1024 bytes, The size of each page of messages in rocksmq - retentionTimeInMinutes: 4320 # 3 days, 3 * 24 * 60 minutes, The retention time of the message in rocksmq. - retentionSizeInMB: 8192 # 8 GB, 8 * 1024 MB, The retention size of the message in rocksmq. - compactionInterval: 86400 # 1 day, trigger rocksdb compaction every day to remove deleted data + rocksmqPageSize: 67108864 # The maximum size of messages in each page in RocksMQ. Messages in RocksMQ are checked and cleared (when expired) in batch based on this parameters. Unit: Byte. + retentionTimeInMinutes: 4320 # The maximum retention time of acked messages in RocksMQ. Acked messages in RocksMQ are retained for the specified period of time and then cleared. Unit: Minute. + retentionSizeInMB: 8192 # The maximum retention size of acked messages of each topic in RocksMQ. Acked messages in each topic are cleared if their size exceed this parameter. Unit: MB. + compactionInterval: 86400 # Time interval to trigger rocksdb compaction to remove deleted data. Unit: Second compressionTypes: 0,0,7,7,7 # compaction compression type, only support use 0,7. 0 means not compress, 7 will use zstd. Length of types means num of rocksdb level. # natsmq configuration. # more detail: https://docs.nats.io/running-a-nats-service/configuration natsmq: server: - port: 4222 # Port for nats server listening + port: 4222 # Listening port of the NATS server. storeDir: /var/lib/milvus/nats # Directory to use for JetStream storage of nats maxFileStore: 17179869184 # Maximum size of the 'file' storage maxPayload: 8388608 # Maximum number of bytes in a message payload @@ -184,59 +239,61 @@ natsmq: # Related configuration of rootCoord, used to handle data definition language (DDL) and data control language (DCL) requests rootCoord: - dmlChannelNum: 16 # The number of dml channels created at system startup - maxPartitionNum: 1024 # Maximum number of partitions in a collection - minSegmentSizeToEnableIndex: 1024 # It's a threshold. When the segment size is less than this value, the segment will not be indexed + dmlChannelNum: 16 # The number of DML-Channels to create at the root coord startup. + # The maximum number of partitions in each collection. + # New partitions cannot be created if this parameter is set as 0 or 1. + # Range: [0, INT64MAX] + maxPartitionNum: 1024 + # The minimum row count of a segment required for creating index. + # Segments with smaller size than this parameter will not be indexed, and will be searched with brute force. + minSegmentSizeToEnableIndex: 1024 enableActiveStandby: false maxDatabaseNum: 64 # Maximum number of database maxGeneralCapacity: 65536 # upper limit for the sum of of product of partitionNumber and shardNumber gracefulStopTimeout: 5 # seconds. force stop node without graceful stop - ip: # if not specified, use the first unicastable address - port: 53100 + ip: # TCP/IP address of rootCoord. If not specified, use the first unicastable address + port: 53100 # TCP port of rootCoord grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the rootCoord can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the rootCoord can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on rootCoord can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on rootCoord can receive, unit: byte # Related configuration of proxy, used to validate client requests and reduce the returned results. proxy: - timeTickInterval: 200 # ms, the interval that proxy synchronize the time tick + timeTickInterval: 200 # The interval at which proxy synchronizes the time tick, unit: ms. healthCheckTimeout: 3000 # ms, the interval that to do component healthy check msgStream: timeTick: - bufSize: 512 - maxNameLength: 255 # Maximum length of name for a collection or alias - # Maximum number of fields in a collection. - # As of today (2.2.0 and after) it is strongly DISCOURAGED to set maxFieldNum >= 64. - # So adjust at your risk! - maxFieldNum: 64 - maxVectorFieldNum: 4 # Maximum number of vector fields in a collection. - maxShardNum: 16 # Maximum number of shards in a collection - maxDimension: 32768 # Maximum dimension of a vector + bufSize: 512 # The maximum number of messages can be buffered in the timeTick message stream of the proxy when producing messages. + maxNameLength: 255 # The maximum length of the name or alias that can be created in Milvus, including the collection name, collection alias, partition name, and field name. + maxFieldNum: 64 # The maximum number of field can be created when creating in a collection. It is strongly DISCOURAGED to set maxFieldNum >= 64. + maxVectorFieldNum: 4 # The maximum number of vector fields that can be specified in a collection. Value range: [1, 10]. + maxShardNum: 16 # The maximum number of shards can be created when creating in a collection. + maxDimension: 32768 # The maximum number of dimensions of a vector can have when creating in a collection. # Whether to produce gin logs.\n # please adjust in embedded Milvus: false ginLogging: true ginLogSkipPaths: / # skip url path for gin log - maxTaskNum: 1024 # max task number of proxy task queue + maxTaskNum: 1024 # The maximum number of tasks in the task queue of the proxy. mustUsePartitionKey: false # switch for whether proxy must use partition key for the collection accessLog: - enable: false # if use access log - minioEnable: false # if upload sealed access log file to minio - localPath: /tmp/milvus_access - filename: # Log filename, leave empty to use stdout. - maxSize: 64 # Max size for a single file, in MB. - cacheSize: 0 # Size of log write cache, in B - cacheFlushInterval: 3 # time interval of auto flush write cache, in Seconds. (Close auto flush if interval was 0) - rotatedTime: 0 # Max time for single access log file in seconds - remotePath: access_log/ # File path in minIO - remoteMaxTime: 0 # Max time for log file in minIO, in hours + enable: false # Whether to enable the access log feature. + minioEnable: false # Whether to upload local access log files to MinIO. This parameter can be specified when proxy.accessLog.filename is not empty. + localPath: /tmp/milvus_access # The local folder path where the access log file is stored. This parameter can be specified when proxy.accessLog.filename is not empty. + filename: # The name of the access log file. If you leave this parameter empty, access logs will be printed to stdout. + maxSize: 64 # The maximum size allowed for a single access log file. If the log file size reaches this limit, a rotation process will be triggered. This process seals the current access log file, creates a new log file, and clears the contents of the original log file. Unit: MB. + rotatedTime: 0 # The maximum time interval allowed for rotating a single access log file. Upon reaching the specified time interval, a rotation process is triggered, resulting in the creation of a new access log file and sealing of the previous one. Unit: seconds + remotePath: access_log/ # The path of the object storage for uploading access log files. + remoteMaxTime: 0 # The time interval allowed for uploading access log files. If the upload time of a log file exceeds this interval, the file will be deleted. Setting the value to 0 disables this feature. formatters: base: format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost]" query: format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost] [database: $database_name] [collection: $collection_name] [partitions: $partition_name] [expr: $method_expr]" methods: "Query,Search,Delete" + cacheSize: 0 # Size of log of write cache, in byte. (Close write cache if size was 0) + cacheFlushInterval: 3 # time interval of auto flush write cache, in seconds. (Close auto flush if interval was 0) connectionCheckIntervalSeconds: 120 # the interval time(in seconds) for connection manager to scan inactive client info connectionClientInfoTTLSeconds: 86400 # inactive client info TTL duration, in seconds maxConnectionNum: 10000 # the max client info numbers that proxy should manage, avoid too many client infos @@ -250,29 +307,31 @@ proxy: port: # high-level restful api acceptTypeAllowInt64: true # high-level restful api, whether http client can deal with int64 enablePprof: true # Whether to enable pprof middleware on the metrics port - ip: # if not specified, use the first unicastable address - port: 19530 + ip: # TCP/IP address of proxy. If not specified, use the first unicastable address + port: 19530 # TCP port of proxy internalPort: 19529 grpc: - serverMaxSendSize: 268435456 - serverMaxRecvSize: 67108864 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 67108864 + serverMaxSendSize: 268435456 # The maximum size of each RPC request that the proxy can send, unit: byte + serverMaxRecvSize: 67108864 # The maximum size of each RPC request that the proxy can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on proxy can send, unit: byte + clientMaxRecvSize: 67108864 # The maximum size of each RPC request that the clients on proxy can receive, unit: byte # Related configuration of queryCoord, used to manage topology and load balancing for the query nodes, and handoff from growing segments to sealed segments. queryCoord: taskMergeCap: 1 taskExecutionCap: 256 - autoHandoff: true # Enable auto handoff - autoBalance: true # Enable auto balance + # Switch value to control if to automatically replace a growing segment with the corresponding indexed sealed segment when the growing segment reaches the sealing threshold. + # If this parameter is set false, Milvus simply searches the growing segments with brute force. + autoHandoff: true + autoBalance: true # Switch value to control if to automatically balance the memory usage among query nodes by distributing segment loading and releasing operations evenly. autoBalanceChannel: true # Enable auto balance channel balancer: ScoreBasedBalancer # auto balancer used for segments on queryNodes globalRowCountFactor: 0.1 # the weight used when balancing segments among queryNodes scoreUnbalanceTolerationFactor: 0.05 # the least value for unbalanced extent between from and to nodes when doing balance reverseUnBalanceTolerationFactor: 1.3 # the largest value for unbalanced extent between from and to nodes after doing balance - overloadedMemoryThresholdPercentage: 90 # The threshold percentage that memory overload - balanceIntervalSeconds: 60 - memoryUsageMaxDifferencePercentage: 30 + overloadedMemoryThresholdPercentage: 90 # The threshold of memory usage (in percentage) in a query node to trigger the sealed segment balancing. + balanceIntervalSeconds: 60 # The interval at which query coord balances the memory usage among query nodes. + memoryUsageMaxDifferencePercentage: 30 # The threshold of memory usage difference (in percentage) between any two query nodes to trigger the sealed segment balancing. rowCountFactor: 0.4 # the row count weight used when balancing segments among queryNodes segmentCountFactor: 0.4 # the segment count weight used when balancing segments among queryNodes globalSegmentCountFactor: 0.1 # the segment count weight used when balancing segments among queryNodes @@ -289,8 +348,6 @@ queryCoord: channelTaskTimeout: 60000 # 1 minute segmentTaskTimeout: 120000 # 2 minute distPullInterval: 500 - collectionObserverInterval: 200 - checkExecutedFlagInterval: 100 heartbeatAvailableInterval: 10000 # 10s, Only QueryNodes which fetched heartbeats within the duration are available loadTimeoutSeconds: 600 distRequestTimeout: 5000 # the request timeout for querycoord fetching data distribution from querynodes, in milliseconds @@ -308,24 +365,30 @@ queryCoord: gracefulStopTimeout: 5 # seconds. force stop node without graceful stop enableStoppingBalance: true # whether enable stopping balance channelExclusiveNodeFactor: 4 # the least node number for enable channel's exclusive mode + collectionObserverInterval: 200 # the interval of collection observer + checkExecutedFlagInterval: 100 # the interval of check executed flag to force to pull dist + updateCollectionLoadStatusInterval: 5 # 5m, max interval for updating collection loaded status cleanExcludeSegmentInterval: 60 # the time duration of clean pipeline exclude segment which used for filter invalid data, in seconds - ip: # if not specified, use the first unicastable address - port: 19531 + ip: # TCP/IP address of queryCoord. If not specified, use the first unicastable address + port: 19531 # TCP port of queryCoord grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the queryCoord can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the queryCoord can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on queryCoord can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on queryCoord can receive, unit: byte # Related configuration of queryNode, used to run hybrid search between vector and scalar data. queryNode: stats: - publishInterval: 1000 # Interval for querynode to report node information (milliseconds) + publishInterval: 1000 # The interval that query node publishes the node statistics information, including segment status, cpu usage, memory usage, health status, etc. Unit: ms. segcore: knowhereThreadPoolNumRatio: 4 # The number of threads in knowhere's thread pool. If disk is enabled, the pool size will multiply with knowhereThreadPoolNumRatio([1, 32]). - chunkRows: 128 # The number of vectors in a chunk. + chunkRows: 128 # Row count by which Segcore divides a segment into chunks. interimIndex: - enableIndex: true # Enable segment build with index to accelerate vector search when segment is in growing or binlog. + # Whether to create a temporary index for growing segments and sealed segments not yet indexed, improving search performance. + # Milvus will eventually seals and indexes all segments, but enabling this optimizes search performance for immediate queries following data insertion. + # This defaults to true, indicating that Milvus creates temporary index for growing segments and the sealed segments that are not indexed upon searches. + enableIndex: true nlist: 128 # temp index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist memExpansionRate: 1.15 # extra memory needed by building interim index @@ -347,9 +410,9 @@ queryNode: warmup: disable mmap: mmapEnabled: false # Enable mmap for loading data - growingMmapEnabled: false # Enable mmap for growing segment - fixedFileSizeForMmapAlloc: 1 #MB, fixed file size for mmap chunk manager to store chunk data - maxDiskUsagePercentageForMmapAlloc: 50 # max percentage of disk usage in memory mapping + growingMmapEnabled: false # Enable mmap for using in growing raw data + fixedFileSizeForMmapAlloc: 1 # tmp file size for mmap chunk manager + maxDiskUsagePercentageForMmapAlloc: 50 # disk percentage used in mmap chunk manager lazyload: enabled: false # Enable lazyload for loading data waitTimeout: 30000 # max wait timeout duration in milliseconds before start to do lazyload search and retrieve @@ -386,19 +449,20 @@ queryNode: maxPendingTaskPerUser: 1024 # Max pending task per user in scheduler dataSync: flowGraph: - maxQueueLength: 16 # Maximum length of task queue in flowgraph + maxQueueLength: 16 # The maximum size of task queue cache in flow graph in query node. maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph - enableSegmentPrune: false # use partition prune function on shard delegator + enableSegmentPrune: false # use partition stats to prune data in search/query on shard delegator + bloomFilterApplyParallelFactor: 4 # parallel factor when to apply pk to bloom filter, default to 4*CPU_CORE_NUM queryStreamBatchSize: 4194304 # return batch size of stream query - ip: # if not specified, use the first unicastable address - port: 21123 workerPooling: size: 10 # the size for worker querynode client pool + ip: # TCP/IP address of queryNode. If not specified, use the first unicastable address + port: 21123 # TCP port of queryNode grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the queryNode can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the queryNode can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on queryNode can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on queryNode can receive, unit: byte indexCoord: bindIndexNodeMode: @@ -414,13 +478,13 @@ indexNode: buildParallel: 1 enableDisk: true # enable index node build disk vector index maxDiskUsagePercentage: 95 - ip: # if not specified, use the first unicastable address - port: 21121 + ip: # TCP/IP address of indexNode. If not specified, use the first unicastable address + port: 21121 # TCP port of indexNode grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the indexNode can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the indexNode can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on indexNode can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on indexNode can receive, unit: byte dataCoord: channel: @@ -432,10 +496,10 @@ dataCoord: checkInterval: 1 # The interval in seconds with which the channel manager advances channel states notifyChannelOperationTimeout: 5 # Timeout notifing channel operations (in seconds). segment: - maxSize: 1024 # Maximum size of a segment in MB + maxSize: 1024 # The maximum size of a segment, unit: MB. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed. diskSegmentMaxSize: 2048 # Maximun size of a segment in MB for collection which has Disk index - sealProportion: 0.12 - assignmentExpiration: 2000 # The time of the assignment expiration in ms + sealProportion: 0.12 # The minimum proportion to datacoord.segment.maxSize to seal a segment. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed. + assignmentExpiration: 2000 # Expiration time of the segment assignment, unit: ms allocLatestExpireAttempt: 200 # The time attempting to alloc latest lastExpire from rootCoord after restart maxLife: 86400 # The max lifetime of segment in seconds, 24*60*60 # If a segment didn't accept dml records in maxIdleTime and the size of segment is greater than @@ -454,57 +518,51 @@ dataCoord: # MUST BE GREATER THAN OR EQUAL TO !!! # During compaction, the size of segment # of rows is able to exceed segment max # of rows by (expansionRate-1) * 100%. expansionRate: 1.25 - segmentFlushInterval: 2 # the minimal interval duration(unit: Seconds) between flusing operation on same segment sealPolicy: channel: - # The size threshold in MB, if the total size of growing segments of each shard + # The size threshold in MB, if the total size of growing segments of each shard # exceeds this threshold, the largest growing segment will be sealed. growingSegmentsMemSize: 4096 autoUpgradeSegmentIndex: false # whether auto upgrade segment index to index engine's version - enableCompaction: true # Enable data segment compaction + segmentFlushInterval: 2 # the minimal interval duration(unit: Seconds) between flusing operation on same segment + # Switch value to control if to enable segment compaction. + # Compaction merges small-size segments into a large segment, and clears the entities deleted beyond the rentention duration of Time Travel. + enableCompaction: true compaction: + # Switch value to control if to enable automatic segment compaction during which data coord locates and merges compactable segments in the background. + # This configuration takes effect only when dataCoord.enableCompaction is set as true. enableAutoCompaction: true indexBasedCompaction: true rpcTimeout: 10 maxParallelTaskNum: 10 workerMaxParallelTaskNum: 2 - gcInterval: 1800 # The time interval in seconds for compaction gc - dropTolerance: 86400 # Compaction task will be cleaned after finish longer than this time(in seconds) clustering: enable: true # Enable clustering compaction - autoEnable: false # Enable auto background clustering compaction + autoEnable: false # Enable auto clustering compaction triggerInterval: 600 # clustering compaction trigger interval in seconds minInterval: 3600 # The minimum interval between clustering compaction executions of one collection, to avoid redundant compaction maxInterval: 259200 # If a collection haven't been clustering compacted for longer than maxInterval, force compact newDataSizeThreshold: 512m # If new data size is large than newDataSizeThreshold, execute clustering compaction - timeout: 7200 # timeout in seconds for clustering compaction, the task will stop if timeout - dropTolerance: 86400 - - # clustering compaction will try best to distribute data into segments with size range in [preferSegmentSize, maxSegmentSize]. - # data will be clustered by preferSegmentSize, if a cluster is larger than maxSegmentSize, will spilt it into multi segment - # buffer between (preferSegmentSize, maxSegmentSize) is left for new data in the same cluster(range), to avoid globally redistribute too often preferSegmentSizeRatio: 0.8 maxSegmentSizeRatio: 1 - - # vector clustering related maxTrainSizeRatio: 0.8 # max data size ratio in Kmeans train, if larger than it, will down sampling to meet this limit maxCentroidsNum: 10240 # maximum centroids number in Kmeans train minCentroidsNum: 16 # minimum centroids number in Kmeans train minClusterSizeRatio: 0.01 # minimum cluster size / avg size in Kmeans train - maxClusterSizeRatio: 10 #maximum cluster size / avg size in Kmeans train + maxClusterSizeRatio: 10 # maximum cluster size / avg size in Kmeans train maxClusterSize: 5g # maximum cluster size in Kmeans train - levelzero: forceTrigger: minSize: 8388608 # The minmum size in bytes to force trigger a LevelZero Compaction, default as 8MB maxSize: 67108864 # The maxmum size in bytes to force trigger a LevelZero Compaction, default as 64MB deltalogMinNum: 10 # The minimum number of deltalog files to force trigger a LevelZero Compaction deltalogMaxNum: 30 # The maxmum number of deltalog files to force trigger a LevelZero Compaction, default as 30 - enableGarbageCollection: true + syncSegmentsInterval: 300 # The time interval for regularly syncing segments + enableGarbageCollection: true # Switch value to control if to enable garbage collection to clear the discarded data in MinIO or S3 service. gc: - interval: 3600 # meta-based gc scanning interval in seconds - missingTolerance: 86400 # orphan file gc tolerance duration in seconds (orphan file which last modified time before the tolerance interval ago will be deleted) - dropTolerance: 10800 # meta-based gc tolerace duration in seconds (file which meta is marked as dropped before the tolerace interval ago will be deleted) + interval: 3600 # The interval at which data coord performs garbage collection, unit: second. + missingTolerance: 86400 # The retention duration of the unrecorded binary log (binlog) files. Setting a reasonably large value for this parameter avoids erroneously deleting the newly created binlog files that lack metadata. Unit: second. + dropTolerance: 10800 # The retention duration of the binlog files of the deleted segments before they are cleared, unit: second. removeConcurrent: 32 # number of concurrent goroutines to remove dropped s3 objects scanInterval: 168 # orphan file (file on oss but has not been registered on meta) on object storage garbage collection scanning interval in hours enableActiveStandby: false @@ -521,18 +579,17 @@ dataCoord: maxImportFileNumPerReq: 1024 # The maximum number of files allowed per single import request. waitForIndex: true # Indicates whether the import operation waits for the completion of index building. gracefulStopTimeout: 5 # seconds. force stop node without graceful stop - ip: # if not specified, use the first unicastable address - port: 13333 + slot: + clusteringCompactionUsage: 16 # slot usage of clustering compaction job. + mixCompactionUsage: 8 # slot usage of mix compaction job. + l0DeleteCompactionUsage: 8 # slot usage of l0 compaction job. + ip: # TCP/IP address of dataCoord. If not specified, use the first unicastable address + port: 13333 # TCP port of dataCoord grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 - syncSegmentsInterval: 300 - slot: - clusteringCompactionUsage: 16 - mixCompactionUsage: 8 - l0DeleteCompactionUsage: 8 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the dataCoord can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the dataCoord can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on dataCoord can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on dataCoord can receive, unit: byte dataNode: dataSync: @@ -545,7 +602,10 @@ dataNode: skipNum: 4 # Consume one for every n records skipped coldTime: 60 # Turn on skip mode after there are only timetick msg for x seconds segment: - insertBufSize: 16777216 # Max buffer size to flush for a single segment. + # The maximum size of each binlog file in a segment buffered in memory. Binlog files whose size exceeds this value are then flushed to MinIO or S3 service. + # Unit: Byte + # Setting this parameter too small causes the system to store a small amount of data too frequently. Setting it too large increases the system's demand for memory. + insertBufSize: 16777216 deleteBufBytes: 16777216 # Max buffer size in bytes to flush del for a single channel, default as 16MB syncPeriod: 600 # The period to sync segments if buffer is not empty. memory: @@ -561,6 +621,8 @@ dataNode: # if this parameter <= 0, will set it as the maximum number of CPUs that can be executing # suggest to set it bigger on large collection numbers to avoid blocking workPoolSize: -1 + # specify the size of global work pool for channel checkpoint updating + # if this parameter <= 0, will set it as 10 updateChannelCheckpointMaxParallel: 10 updateChannelCheckpointInterval: 60 # the interval duration(in seconds) for datanode to update channel checkpoint of each channel updateChannelCheckpointRPCTimeout: 20 # timeout in seconds for UpdateChannelCheckpoint RPC call @@ -574,29 +636,34 @@ dataNode: levelZeroBatchMemoryRatio: 0.05 # The minimal memory ratio of free memory for level zero compaction executing in batch mode levelZeroMaxBatchSize: -1 # Max batch size refers to the max number of L1/L2 segments in a batch when executing L0 compaction. Default to -1, any value that is less than 1 means no limit. Valid range: >= 1. gracefulStopTimeout: 1800 # seconds. force stop node without graceful stop - ip: # if not specified, use the first unicastable address - port: 21124 - grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 slot: - slotCap: 16 # The maximum number of tasks(e.g. compaction, importing) allowed to run concurrently on a datanode. - + slotCap: 16 # The maximum number of tasks(e.g. compaction, importing) allowed to run concurrently on a datanode clusteringCompaction: memoryBufferRatio: 0.1 # The ratio of memory buffer of clustering compaction. Data larger than threshold will be flushed to storage. - workPoolSize: 8 # worker pool size for one clustering compaction job + workPoolSize: 8 # worker pool size for one clustering compaction job. + ip: # TCP/IP address of dataNode. If not specified, use the first unicastable address + port: 21124 # TCP port of dataNode + grpc: + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the dataNode can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the dataNode can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on dataNode can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on dataNode can receive, unit: byte # Configures the system log output. log: - level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + # Milvus log level. Option: debug, info, warn, error, panic, and fatal. + # It is recommended to use debug level under test and development environments, and info level in production environment. + level: info file: - rootPath: # root dir path to put logs, default "" means no log file will print. please adjust in embedded Milvus: /tmp/milvus/logs - maxSize: 300 # MB - maxAge: 10 # Maximum time for log retention in day. - maxBackups: 20 - format: text # text or json + # Root path to the log files. + # The default value is set empty, indicating to output log files to standard output (stdout) and standard error (stderr). + # If this parameter is set to a valid local path, Milvus writes and stores log files in this path. + # Set this parameter as the path that you have permission to write. + rootPath: + maxSize: 300 # The maximum size of a log file, unit: MB. + maxAge: 10 # The maximum retention time before a log file is automatically cleared, unit: day. The minimum value is 1. + maxBackups: 20 # The maximum number of log files to back up, unit: day. The minimum value is 1. + format: text # Milvus log format. Option: text and JSON stdout: true # Stdout enable or not grpc: @@ -622,10 +689,10 @@ tls: caPemPath: configs/cert/ca.pem common: - defaultPartitionName: _default # default partition name for a collection - defaultIndexName: _default_idx # default index name + defaultPartitionName: _default # Name of the default partition when a collection is created + defaultIndexName: _default_idx # Name of the index when it is created with name unspecified entityExpiration: -1 # Entity expiration in seconds, CAUTION -1 means never expire - indexSliceSize: 16 # MB + indexSliceSize: 16 # Index slice size in MB threadCoreCoefficient: highPriority: 10 # This parameter specify how many times the number of threads is the number of cores in high priority pool middlePriority: 5 # This parameter specify how many times the number of threads is the number of cores in middle priority pool @@ -651,8 +718,8 @@ common: # The superusers will ignore some system check processes, # like the old password verification when updating the credential superUsers: + defaultRootPassword: Milvus # default password for root user tlsMode: 0 - defaultRootPassword: Milvus session: ttl: 30 # ttl value when session granting a lease to register service retryTimes: 30 # retry times when session sending etcd requests @@ -665,14 +732,18 @@ common: storage: scheme: s3 enablev2: false - ttMsgEnabled: true # Whether the instance disable sending ts messages + # Whether to disable the internal time messaging mechanism for the system. + # If disabled (set to false), the system will not allow DML operations, including insertion, deletion, queries, and searches. + # This helps Milvus-CDC synchronize incremental data + ttMsgEnabled: true traceLogMode: 0 # trace request info bloomFilterSize: 100000 # bloom filter initial size maxBloomFalsePositive: 0.001 # max false positive rate for bloom filter - # clustering key/compaction related - usePartitionKeyAsClusteringKey: false - useVectorAsClusteringKey: false - enableVectorClusteringKey: false + bloomFilterType: BasicBloomFilter # bloom filter type, support BasicBloomFilter and BlockedBloomFilter + bloomFilterApplyBatchSize: 1000 # batch size when to apply pk to bloom filter + usePartitionKeyAsClusteringKey: false # if true, do clustering compaction and segment prune on partition key field + useVectorAsClusteringKey: false # if true, do clustering compaction and segment prune on vector field + enableVectorClusteringKey: false # if true, enable vector clustering key and vector clustering compaction # QuotaConfig, configurations of Milvus quota and limits. # By default, we enable: @@ -696,43 +767,62 @@ quotaAndLimits: allocWaitInterval: 1000 # retry wait duration when delete alloc forward data rate failed, in millisecond complexDeleteLimitEnable: false # whether complex delete check forward data by limiter maxCollectionNum: 65536 - maxCollectionNumPerDB: 65536 + maxCollectionNumPerDB: 65536 # Maximum number of collections per database. maxInsertSize: -1 # maximum size of a single insert request, in bytes, -1 means no limit maxResourceGroupNumOfQueryNode: 1024 # maximum number of resource groups of query nodes ddl: - enabled: false - collectionRate: -1 # qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection - partitionRate: -1 # qps, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition + enabled: false # Whether DDL request throttling is enabled. + # Maximum number of collection-related DDL requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 collection-related DDL requests per second, including collection creation requests, collection drop requests, collection load requests, and collection release requests. + # To use this setting, set quotaAndLimits.ddl.enabled to true at the same time. + collectionRate: -1 + # Maximum number of partition-related DDL requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including partition creation requests, partition drop requests, partition load requests, and partition release requests. + # To use this setting, set quotaAndLimits.ddl.enabled to true at the same time. + partitionRate: -1 db: collectionRate: -1 # qps of db level , default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection partitionRate: -1 # qps of db level, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition indexRate: - enabled: false - max: -1 # qps, default no limit, rate for CreateIndex, DropIndex + enabled: false # Whether index-related request throttling is enabled. + # Maximum number of index-related requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including index creation requests and index drop requests. + # To use this setting, set quotaAndLimits.indexRate.enabled to true at the same time. + max: -1 db: max: -1 # qps of db level, default no limit, rate for CreateIndex, DropIndex flushRate: - enabled: true - max: -1 # qps, default no limit, rate for flush + enabled: true # Whether flush request throttling is enabled. + # Maximum number of flush requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 flush requests per second. + # To use this setting, set quotaAndLimits.flushRate.enabled to true at the same time. + max: -1 collection: max: 0.1 # qps, default no limit, rate for flush at collection level. db: max: -1 # qps of db level, default no limit, rate for flush compactionRate: - enabled: false - max: -1 # qps, default no limit, rate for manualCompaction + enabled: false # Whether manual compaction request throttling is enabled. + # Maximum number of manual-compaction requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 manual-compaction requests per second. + # To use this setting, set quotaAndLimits.compaction.enabled to true at the same time. + max: -1 db: max: -1 # qps of db level, default no limit, rate for manualCompaction dml: - # dml limit rates, default no limit. - # The maximum rate will not be greater than max. - enabled: false + enabled: false # Whether DML request throttling is enabled. insertRate: - max: -1 # MB/s, default no limit + # Highest data insertion rate per second. + # Setting this item to 5 indicates that Milvus only allows data insertion at the rate of 5 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 db: max: -1 # MB/s, default no limit collection: - max: -1 # MB/s, default no limit + # Highest data insertion rate per collection per second. + # Setting this item to 5 indicates that Milvus only allows data insertion to any collection at the rate of 5 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 partition: max: -1 # MB/s, default no limit upsertRate: @@ -744,11 +834,17 @@ quotaAndLimits: partition: max: -1 # MB/s, default no limit deleteRate: - max: -1 # MB/s, default no limit + # Highest data deletion rate per second. + # Setting this item to 0.1 indicates that Milvus only allows data deletion at the rate of 0.1 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 db: max: -1 # MB/s, default no limit collection: - max: -1 # MB/s, default no limit + # Highest data deletion rate per second. + # Setting this item to 0.1 indicates that Milvus only allows data deletion from any collection at the rate of 0.1 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 partition: max: -1 # MB/s, default no limit bulkLoadRate: @@ -760,23 +856,33 @@ quotaAndLimits: partition: max: -1 # MB/s, default no limit, not support yet. TODO: limit partition bulkLoad rate dql: - # dql limit rates, default no limit. - # The maximum rate will not be greater than max. - enabled: false + enabled: false # Whether DQL request throttling is enabled. searchRate: - max: -1 # vps (vectors per second), default no limit + # Maximum number of vectors to search per second. + # Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second no matter whether these 100 vectors are all in one search or scattered across multiple searches. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 db: max: -1 # vps (vectors per second), default no limit collection: - max: -1 # vps (vectors per second), default no limit + # Maximum number of vectors to search per collection per second. + # Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second per collection no matter whether these 100 vectors are all in one search or scattered across multiple searches. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 partition: max: -1 # vps (vectors per second), default no limit queryRate: - max: -1 # qps, default no limit + # Maximum number of queries per second. + # Setting this item to 100 indicates that Milvus only allows 100 queries per second. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 db: max: -1 # qps, default no limit collection: - max: -1 # qps, default no limit + # Maximum number of queries per collection per second. + # Setting this item to 100 indicates that Milvus only allows 100 queries per collection per second. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 partition: max: -1 # qps, default no limit limitWriting: @@ -813,6 +919,10 @@ quotaAndLimits: diskQuotaPerDB: -1 # MB, (0, +inf), default no limit diskQuotaPerCollection: -1 # MB, (0, +inf), default no limit diskQuotaPerPartition: -1 # MB, (0, +inf), default no limit + l0SegmentsRowCountProtection: + enabled: false # switch to enable l0 segment row count quota + lowWaterLevel: 32768 # l0 segment row count quota, low water level + highWaterLevel: 65536 # l0 segment row count quota, low water level limitReading: # forceDeny false means dql requests are allowed (except for some # specific conditions, such as collection has been dropped), true means always reject all dql requests. @@ -857,6 +967,7 @@ trace: endpoint: # example: "127.0.0.1:4317" for grpc, "127.0.0.1:4318" for http method: # otlp export method, acceptable values: ["grpc", "http"], using "grpc" by default secure: true + initTimeoutSeconds: 10 # segcore initialization timeout in seconds, preventing otlp grpc hangs forever #when using GPU indexing, Milvus will utilize a memory pool to avoid frequent memory allocation and deallocation. #here, you can set the size of the memory occupied by the memory pool, with the unit being MB. diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 1354f8705c5cf..3d46353606e70 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -405,7 +405,7 @@ func (p *commonConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "_default", Forbidden: true, - Doc: "default partition name for a collection", + Doc: "Name of the default partition when a collection is created", Export: true, } p.DefaultPartitionName.Init(base.mgr) @@ -414,7 +414,7 @@ func (p *commonConfig) init(base *BaseTable) { Key: "common.defaultIndexName", Version: "2.0.0", DefaultValue: "_default_idx", - Doc: "default index name", + Doc: "Name of the index when it is created with name unspecified", Export: true, } p.DefaultIndexName.Init(base.mgr) @@ -452,7 +452,7 @@ This configuration is only used by querynode and indexnode, it selects CPU instr Key: "common.indexSliceSize", Version: "2.0.0", DefaultValue: strconv.Itoa(DefaultIndexSliceSize), - Doc: "MB", + Doc: "Index slice size in MB", Export: true, } p.IndexSliceSize.Init(base.mgr) @@ -725,8 +725,10 @@ like the old password verification when updating the credential`, Key: "common.ttMsgEnabled", Version: "2.3.2", DefaultValue: "true", - Doc: "Whether the instance disable sending ts messages", - Export: true, + Doc: `Whether to disable the internal time messaging mechanism for the system. +If disabled (set to false), the system will not allow DML operations, including insertion, deletion, queries, and searches. +This helps Milvus-CDC synchronize incremental data`, + Export: true, } p.TTMsgEnabled.Init(base.mgr) @@ -788,6 +790,7 @@ like the old password verification when updating the credential`, Version: "2.4.6", Doc: "if true, do clustering compaction and segment prune on partition key field", DefaultValue: "false", + Export: true, } p.UsePartitionKeyAsClusteringKey.Init(base.mgr) @@ -796,6 +799,7 @@ like the old password verification when updating the credential`, Version: "2.4.6", Doc: "if true, do clustering compaction and segment prune on vector field", DefaultValue: "false", + Export: true, } p.UseVectorAsClusteringKey.Init(base.mgr) @@ -804,6 +808,7 @@ like the old password verification when updating the credential`, Version: "2.4.6", Doc: "if true, enable vector clustering key and vector clustering compaction", DefaultValue: "false", + Export: true, } p.EnableVectorClusteringKey.Init(base.mgr) @@ -961,16 +966,20 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.level", DefaultValue: "info", Version: "2.0.0", - Doc: "Only supports debug, info, warn, error, panic, or fatal. Default 'info'.", - Export: true, + Doc: `Milvus log level. Option: debug, info, warn, error, panic, and fatal. +It is recommended to use debug level under test and development environments, and info level in production environment.`, + Export: true, } l.Level.Init(base.mgr) l.RootPath = ParamItem{ Key: "log.file.rootPath", Version: "2.0.0", - Doc: "root dir path to put logs, default \"\" means no log file will print. please adjust in embedded Milvus: /tmp/milvus/logs", - Export: true, + Doc: `Root path to the log files. +The default value is set empty, indicating to output log files to standard output (stdout) and standard error (stderr). +If this parameter is set to a valid local path, Milvus writes and stores log files in this path. +Set this parameter as the path that you have permission to write.`, + Export: true, } l.RootPath.Init(base.mgr) @@ -978,7 +987,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.file.maxSize", DefaultValue: "300", Version: "2.0.0", - Doc: "MB", + Doc: "The maximum size of a log file, unit: MB.", Export: true, } l.MaxSize.Init(base.mgr) @@ -987,7 +996,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.file.maxAge", DefaultValue: "10", Version: "2.0.0", - Doc: "Maximum time for log retention in day.", + Doc: "The maximum retention time before a log file is automatically cleared, unit: day. The minimum value is 1.", Export: true, } l.MaxAge.Init(base.mgr) @@ -996,6 +1005,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.file.maxBackups", DefaultValue: "20", Version: "2.0.0", + Doc: "The maximum number of log files to back up, unit: day. The minimum value is 1.", Export: true, } l.MaxBackups.Init(base.mgr) @@ -1004,7 +1014,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.format", DefaultValue: "text", Version: "2.0.0", - Doc: "text or json", + Doc: "Milvus log format. Option: text and JSON", Export: true, } l.Format.Init(base.mgr) @@ -1045,7 +1055,7 @@ func (p *rootCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "16", Forbidden: true, - Doc: "The number of dml channels created at system startup", + Doc: "The number of DML-Channels to create at the root coord startup.", Export: true, } p.DmlChannelNum.Init(base.mgr) @@ -1054,8 +1064,10 @@ func (p *rootCoordConfig) init(base *BaseTable) { Key: "rootCoord.maxPartitionNum", Version: "2.0.0", DefaultValue: "1024", - Doc: "Maximum number of partitions in a collection", - Export: true, + Doc: `The maximum number of partitions in each collection. +New partitions cannot be created if this parameter is set as 0 or 1. +Range: [0, INT64MAX]`, + Export: true, } p.MaxPartitionNum.Init(base.mgr) @@ -1063,8 +1075,9 @@ func (p *rootCoordConfig) init(base *BaseTable) { Key: "rootCoord.minSegmentSizeToEnableIndex", Version: "2.0.0", DefaultValue: "1024", - Doc: "It's a threshold. When the segment size is less than this value, the segment will not be indexed", - Export: true, + Doc: `The minimum row count of a segment required for creating index. +Segments with smaller size than this parameter will not be indexed, and will be searched with brute force.`, + Export: true, } p.MinSegmentSizeToEnableIndex.Init(base.mgr) @@ -1179,7 +1192,7 @@ func (p *proxyConfig) init(base *BaseTable) { Version: "2.2.0", DefaultValue: "200", PanicIfEmpty: true, - Doc: "ms, the interval that proxy synchronize the time tick", + Doc: "The interval at which proxy synchronizes the time tick, unit: ms.", Export: true, } p.TimeTickInterval.Init(base.mgr) @@ -1200,6 +1213,7 @@ func (p *proxyConfig) init(base *BaseTable) { Version: "2.2.0", DefaultValue: "512", PanicIfEmpty: true, + Doc: "The maximum number of messages can be buffered in the timeTick message stream of the proxy when producing messages.", Export: true, } p.MsgStreamTimeTickBufSize.Init(base.mgr) @@ -1209,7 +1223,7 @@ func (p *proxyConfig) init(base *BaseTable) { DefaultValue: "255", Version: "2.0.0", PanicIfEmpty: true, - Doc: "Maximum length of name for a collection or alias", + Doc: "The maximum length of the name or alias that can be created in Milvus, including the collection name, collection alias, partition name, and field name.", Export: true, } p.MaxNameLength.Init(base.mgr) @@ -1243,10 +1257,8 @@ func (p *proxyConfig) init(base *BaseTable) { DefaultValue: "64", Version: "2.0.0", PanicIfEmpty: true, - Doc: `Maximum number of fields in a collection. -As of today (2.2.0 and after) it is strongly DISCOURAGED to set maxFieldNum >= 64. -So adjust at your risk!`, - Export: true, + Doc: "The maximum number of field can be created when creating in a collection. It is strongly DISCOURAGED to set maxFieldNum >= 64.", + Export: true, } p.MaxFieldNum.Init(base.mgr) @@ -1255,7 +1267,7 @@ So adjust at your risk!`, Version: "2.4.0", DefaultValue: "4", PanicIfEmpty: true, - Doc: "Maximum number of vector fields in a collection.", + Doc: "The maximum number of vector fields that can be specified in a collection. Value range: [1, 10].", Export: true, } p.MaxVectorFieldNum.Init(base.mgr) @@ -1269,7 +1281,7 @@ So adjust at your risk!`, DefaultValue: "16", Version: "2.0.0", PanicIfEmpty: true, - Doc: "Maximum number of shards in a collection", + Doc: "The maximum number of shards can be created when creating in a collection.", Export: true, } p.MaxShardNum.Init(base.mgr) @@ -1279,7 +1291,7 @@ So adjust at your risk!`, DefaultValue: "32768", Version: "2.0.0", PanicIfEmpty: true, - Doc: "Maximum dimension of a vector", + Doc: "The maximum number of dimensions of a vector can have when creating in a collection.", Export: true, } p.MaxDimension.Init(base.mgr) @@ -1288,7 +1300,7 @@ So adjust at your risk!`, Key: "proxy.maxTaskNum", Version: "2.2.0", DefaultValue: "10000", - Doc: "max task number of proxy task queue", + Doc: "The maximum number of tasks in the task queue of the proxy.", Export: true, } p.MaxTaskNum.Init(base.mgr) @@ -1339,7 +1351,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.enable", Version: "2.2.0", DefaultValue: "false", - Doc: "if use access log", + Doc: "Whether to enable the access log feature.", Export: true, } p.AccessLog.Enable.Init(base.mgr) @@ -1348,7 +1360,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.minioEnable", Version: "2.2.0", DefaultValue: "false", - Doc: "if upload sealed access log file to minio", + Doc: "Whether to upload local access log files to MinIO. This parameter can be specified when proxy.accessLog.filename is not empty.", Export: true, } p.AccessLog.MinioEnable.Init(base.mgr) @@ -1357,6 +1369,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.localPath", Version: "2.2.0", DefaultValue: "/tmp/milvus_access", + Doc: "The local folder path where the access log file is stored. This parameter can be specified when proxy.accessLog.filename is not empty.", Export: true, } p.AccessLog.LocalPath.Init(base.mgr) @@ -1365,7 +1378,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.filename", Version: "2.2.0", DefaultValue: "", - Doc: "Log filename, leave empty to use stdout.", + Doc: "The name of the access log file. If you leave this parameter empty, access logs will be printed to stdout.", Export: true, } p.AccessLog.Filename.Init(base.mgr) @@ -1374,7 +1387,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.maxSize", Version: "2.2.0", DefaultValue: "64", - Doc: "Max size for a single file, in MB.", + Doc: "The maximum size allowed for a single access log file. If the log file size reaches this limit, a rotation process will be triggered. This process seals the current access log file, creates a new log file, and clears the contents of the original log file. Unit: MB.", Export: true, } p.AccessLog.MaxSize.Init(base.mgr) @@ -1383,7 +1396,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.cacheSize", Version: "2.3.2", DefaultValue: "0", - Doc: "Size of log of write cache, in B. (Close write cache if size was 0", + Doc: "Size of log of write cache, in byte. (Close write cache if size was 0)", Export: true, } p.AccessLog.CacheSize.Init(base.mgr) @@ -1392,7 +1405,8 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.cacheFlushInterval", Version: "2.4.0", DefaultValue: "3", - Doc: "time interval of auto flush write cache, in Seconds. (Close auto flush if interval was 0)", + Doc: "time interval of auto flush write cache, in seconds. (Close auto flush if interval was 0)", + Export: true, } p.AccessLog.CacheFlushInterval.Init(base.mgr) @@ -1400,7 +1414,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.maxBackups", Version: "2.2.0", DefaultValue: "8", - Doc: "Maximum number of old log files to retain.", + Doc: "The maximum number of sealed access log files that can be retained. If the number of sealed access log files exceeds this limit, the oldest one will be deleted.", } p.AccessLog.MaxBackups.Init(base.mgr) @@ -1408,7 +1422,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.rotatedTime", Version: "2.2.0", DefaultValue: "0", - Doc: "Max time for single access log file in seconds", + Doc: "The maximum time interval allowed for rotating a single access log file. Upon reaching the specified time interval, a rotation process is triggered, resulting in the creation of a new access log file and sealing of the previous one. Unit: seconds", Export: true, } p.AccessLog.RotatedTime.Init(base.mgr) @@ -1417,7 +1431,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.remotePath", Version: "2.2.0", DefaultValue: "access_log/", - Doc: "File path in minIO", + Doc: "The path of the object storage for uploading access log files.", Export: true, } p.AccessLog.RemotePath.Init(base.mgr) @@ -1426,7 +1440,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.remoteMaxTime", Version: "2.2.0", DefaultValue: "0", - Doc: "Max time for log file in minIO, in hours", + Doc: "The time interval allowed for uploading access log files. If the upload time of a log file exceeds this interval, the file will be deleted. Setting the value to 0 disables this feature.", Export: true, } p.AccessLog.RemoteMaxTime.Init(base.mgr) @@ -1435,7 +1449,7 @@ please adjust in embedded Milvus: false`, KeyPrefix: "proxy.accessLog.formatters.", Version: "2.3.4", Export: true, - Doc: "access log formatters for specified methods, if not set, use the base formatter.", + Doc: "Access log formatters for specified methods, if not set, use the base formatter.", } p.AccessLog.Formatter.Init(base.mgr) @@ -1701,8 +1715,9 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "true", PanicIfEmpty: true, - Doc: "Enable auto handoff", - Export: true, + Doc: `Switch value to control if to automatically replace a growing segment with the corresponding indexed sealed segment when the growing segment reaches the sealing threshold. +If this parameter is set false, Milvus simply searches the growing segments with brute force.`, + Export: true, } p.AutoHandoff.Init(base.mgr) @@ -1711,7 +1726,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "true", PanicIfEmpty: true, - Doc: "Enable auto balance", + Doc: "Switch value to control if to automatically balance the memory usage among query nodes by distributing segment loading and releasing operations evenly.", Export: true, } p.AutoBalance.Init(base.mgr) @@ -1831,7 +1846,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "90", PanicIfEmpty: true, - Doc: "The threshold percentage that memory overload", + Doc: "The threshold of memory usage (in percentage) in a query node to trigger the sealed segment balancing.", Export: true, } p.OverloadedMemoryThresholdPercentage.Init(base.mgr) @@ -1841,6 +1856,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "60", PanicIfEmpty: true, + Doc: "The interval at which query coord balances the memory usage among query nodes.", Export: true, } p.BalanceIntervalSeconds.Init(base.mgr) @@ -1880,6 +1896,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "30", PanicIfEmpty: true, + Doc: "The threshold of memory usage difference (in percentage) between any two query nodes to trigger the sealed segment balancing.", Export: true, } p.MemoryUsageMaxDifferencePercentage.Init(base.mgr) @@ -2174,7 +2191,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.4.4", DefaultValue: "200", Doc: "the interval of collection observer", - Export: false, + Export: true, } p.CollectionObserverInterval.Init(base.mgr) @@ -2183,7 +2200,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.4.4", DefaultValue: "100", Doc: "the interval of check executed flag to force to pull dist", - Export: false, + Export: true, } p.CheckExecutedFlagInterval.Init(base.mgr) @@ -2331,7 +2348,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { Key: "queryNode.dataSync.flowGraph.maxQueueLength", Version: "2.0.0", DefaultValue: "16", - Doc: "Maximum length of task queue in flowgraph", + Doc: "The maximum size of task queue cache in flow graph in query node.", Export: true, } p.FlowGraphMaxQueueLength.Init(base.mgr) @@ -2349,7 +2366,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { Key: "queryNode.stats.publishInterval", Version: "2.0.0", DefaultValue: "1000", - Doc: "Interval for querynode to report node information (milliseconds)", + Doc: "The interval that query node publishes the node statistics information, including segment status, cpu usage, memory usage, health status, etc. Unit: ms.", Export: true, } p.StatsPublishInterval.Init(base.mgr) @@ -2383,7 +2400,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { } return v }, - Doc: "The number of vectors in a chunk.", + Doc: "Row count by which Segcore divides a segment into chunks.", Export: true, } p.ChunkRows.Init(base.mgr) @@ -2392,8 +2409,10 @@ func (p *queryNodeConfig) init(base *BaseTable) { Key: "queryNode.segcore.interimIndex.enableIndex", Version: "2.0.0", DefaultValue: "false", - Doc: "Enable segment build with index to accelerate vector search when segment is in growing or binlog.", - Export: true, + Doc: `Whether to create a temporary index for growing segments and sealed segments not yet indexed, improving search performance. +Milvus will eventually seals and indexes all segments, but enabling this optimizes search performance for immediate queries following data insertion. +This defaults to true, indicating that Milvus creates temporary index for growing segments and the sealed segments that are not indexed upon searches.`, + Export: true, } p.EnableTempSegmentIndex.Init(base.mgr) @@ -2541,7 +2560,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { p.FixedFileSizeForMmapManager.Init(base.mgr) p.MaxMmapDiskPercentageForMmapManager = ParamItem{ - Key: "querynode.mmap.maxDiskUsagePercentageForMmapAlloc", + Key: "queryNode.mmap.maxDiskUsagePercentageForMmapAlloc", Version: "2.4.6", DefaultValue: "20", Doc: "disk percentage used in mmap chunk manager", @@ -3150,7 +3169,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { Key: "dataCoord.segment.maxSize", Version: "2.0.0", DefaultValue: "1024", - Doc: "Maximum size of a segment in MB", + Doc: "The maximum size of a segment, unit: MB. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed.", Export: true, } p.SegmentMaxSize.Init(base.mgr) @@ -3168,6 +3187,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { Key: "dataCoord.segment.sealProportion", Version: "2.0.0", DefaultValue: "0.12", + Doc: "The minimum proportion to datacoord.segment.maxSize to seal a segment. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed.", Export: true, } p.SegmentSealProportion.Init(base.mgr) @@ -3176,7 +3196,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { Key: "dataCoord.segment.assignmentExpiration", Version: "2.0.0", DefaultValue: "2000", - Doc: "The time of the assignment expiration in ms", + Doc: "Expiration time of the segment assignment, unit: ms", Export: true, } p.SegAssignmentExpiration.Init(base.mgr) @@ -3243,8 +3263,9 @@ exceeds this threshold, the largest growing segment will be sealed.`, Key: "dataCoord.enableCompaction", Version: "2.0.0", DefaultValue: "true", - Doc: "Enable data segment compaction", - Export: true, + Doc: `Switch value to control if to enable segment compaction. +Compaction merges small-size segments into a large segment, and clears the entities deleted beyond the rentention duration of Time Travel.`, + Export: true, } p.EnableCompaction.Init(base.mgr) @@ -3252,7 +3273,9 @@ exceeds this threshold, the largest growing segment will be sealed.`, Key: "dataCoord.compaction.enableAutoCompaction", Version: "2.0.0", DefaultValue: "true", - Export: true, + Doc: `Switch value to control if to enable automatic segment compaction during which data coord locates and merges compactable segments in the background. +This configuration takes effect only when dataCoord.enableCompaction is set as true.`, + Export: true, } p.EnableAutoCompaction.Init(base.mgr) @@ -3410,6 +3433,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Version: "2.4.6", Doc: "The time interval for regularly syncing segments", DefaultValue: "300", // 5 * 60 seconds + Export: true, } p.SyncSegmentsInterval.Init(base.mgr) @@ -3481,6 +3505,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Version: "2.4.7", DefaultValue: "600", Doc: "clustering compaction trigger interval in seconds", + Export: true, } p.ClusteringCompactionTriggerInterval.Init(base.mgr) @@ -3489,6 +3514,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Version: "2.4.7", Doc: "The minimum interval between clustering compaction executions of one collection, to avoid redundant compaction", DefaultValue: "3600", + Export: true, } p.ClusteringCompactionMinInterval.Init(base.mgr) @@ -3497,6 +3523,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Version: "2.4.7", Doc: "If a collection haven't been clustering compacted for longer than maxInterval, force compact", DefaultValue: "86400", + Export: true, } p.ClusteringCompactionMaxInterval.Init(base.mgr) @@ -3505,6 +3532,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Version: "2.4.7", Doc: "If new data size is large than newDataSizeThreshold, execute clustering compaction", DefaultValue: "512m", + Export: true, } p.ClusteringCompactionNewDataSizeThreshold.Init(base.mgr) @@ -3512,7 +3540,6 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.compaction.clustering.timeout", Version: "2.4.7", DefaultValue: "3600", - Doc: "timeout in seconds for clustering compaction, the task will stop if timeout", } p.ClusteringCompactionTimeoutInSeconds.Init(base.mgr) @@ -3592,7 +3619,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.enableGarbageCollection", Version: "2.0.0", DefaultValue: "true", - Doc: "", + Doc: "Switch value to control if to enable garbage collection to clear the discarded data in MinIO or S3 service.", Export: true, } p.EnableGarbageCollection.Init(base.mgr) @@ -3601,7 +3628,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.gc.interval", Version: "2.0.0", DefaultValue: "3600", - Doc: "meta-based gc scanning interval in seconds", + Doc: "The interval at which data coord performs garbage collection, unit: second.", Export: true, } p.GCInterval.Init(base.mgr) @@ -3620,7 +3647,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.gc.missingTolerance", Version: "2.0.0", DefaultValue: "86400", - Doc: "orphan file gc tolerance duration in seconds (orphan file which last modified time before the tolerance interval ago will be deleted)", + Doc: "The retention duration of the unrecorded binary log (binlog) files. Setting a reasonably large value for this parameter avoids erroneously deleting the newly created binlog files that lack metadata. Unit: second.", Export: true, } p.GCMissingTolerance.Init(base.mgr) @@ -3629,7 +3656,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.gc.dropTolerance", Version: "2.0.0", DefaultValue: "10800", - Doc: "meta-based gc tolerace duration in seconds (file which meta is marked as dropped before the tolerace interval ago will be deleted)", + Doc: "The retention duration of the binlog files of the deleted segments before they are cleared, unit: second.", Export: true, } p.GCDropTolerance.Init(base.mgr) @@ -4018,8 +4045,10 @@ func (p *dataNodeConfig) init(base *BaseTable) { FallbackKeys: []string{"DATA_NODE_IBUFSIZE"}, DefaultValue: "16777216", PanicIfEmpty: true, - Doc: "Max buffer size to flush for a single segment.", - Export: true, + Doc: `The maximum size of each binlog file in a segment buffered in memory. Binlog files whose size exceeds this value are then flushed to MinIO or S3 service. +Unit: Byte +Setting this parameter too small causes the system to store a small amount of data too frequently. Setting it too large increases the system's demand for memory.`, + Export: true, } p.FlushInsertBufferSize.Init(base.mgr) diff --git a/pkg/util/paramtable/grpc_param.go b/pkg/util/paramtable/grpc_param.go index 509af635260e0..e9240855f3e03 100644 --- a/pkg/util/paramtable/grpc_param.go +++ b/pkg/util/paramtable/grpc_param.go @@ -71,7 +71,7 @@ func (p *grpcConfig) init(domain string, base *BaseTable) { p.IPItem = ParamItem{ Key: p.Domain + ".ip", Version: "2.3.3", - Doc: "if not specified, use the first unicastable address", + Doc: "TCP/IP address of " + p.Domain + ". If not specified, use the first unicastable address", Export: true, } p.IPItem.Init(base.mgr) @@ -81,6 +81,7 @@ func (p *grpcConfig) init(domain string, base *BaseTable) { Key: p.Domain + ".port", Version: "2.0.0", DefaultValue: strconv.FormatInt(ProxyExternalPort, 10), + Doc: "TCP port of " + p.Domain, Export: true, } p.Port.Init(base.mgr) @@ -162,6 +163,7 @@ func (p *GrpcServerConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the " + domain + " can send, unit: byte", Export: true, } p.ServerMaxSendSize.Init(base.mgr) @@ -184,6 +186,7 @@ func (p *GrpcServerConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the " + domain + " can receive, unit: byte", Export: true, } p.ServerMaxRecvSize.Init(base.mgr) @@ -240,6 +243,7 @@ func (p *GrpcClientConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the clients on " + domain + " can send, unit: byte", Export: true, } p.ClientMaxSendSize.Init(base.mgr) @@ -262,6 +266,7 @@ func (p *GrpcClientConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the clients on " + domain + " can receive, unit: byte", Export: true, } p.ClientMaxRecvSize.Init(base.mgr) diff --git a/pkg/util/paramtable/quota_param.go b/pkg/util/paramtable/quota_param.go index d7a276514af67..74e6b8d4a1d9b 100644 --- a/pkg/util/paramtable/quota_param.go +++ b/pkg/util/paramtable/quota_param.go @@ -204,6 +204,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.ddl.enabled", Version: "2.2.0", DefaultValue: "false", + Doc: "Whether DDL request throttling is enabled.", Export: true, } p.DDLLimitEnabled.Init(base.mgr) @@ -222,7 +223,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection", + Doc: `Maximum number of collection-related DDL requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 collection-related DDL requests per second, including collection creation requests, collection drop requests, collection load requests, and collection release requests. +To use this setting, set quotaAndLimits.ddl.enabled to true at the same time.`, Export: true, } p.DDLCollectionRate.Init(base.mgr) @@ -260,7 +263,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition", + Doc: `Maximum number of partition-related DDL requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including partition creation requests, partition drop requests, partition load requests, and partition release requests. +To use this setting, set quotaAndLimits.ddl.enabled to true at the same time.`, Export: true, } p.DDLPartitionRate.Init(base.mgr) @@ -288,6 +293,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.indexRate.enabled", Version: "2.2.0", DefaultValue: "false", + Doc: "Whether index-related request throttling is enabled.", Export: true, } p.IndexLimitEnabled.Init(base.mgr) @@ -306,7 +312,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for CreateIndex, DropIndex", + Doc: `Maximum number of index-related requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including index creation requests and index drop requests. +To use this setting, set quotaAndLimits.indexRate.enabled to true at the same time.`, Export: true, } p.MaxIndexRate.Init(base.mgr) @@ -334,6 +342,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.flushRate.enabled", Version: "2.2.0", DefaultValue: "true", + Doc: "Whether flush request throttling is enabled.", Export: true, } p.FlushLimitEnabled.Init(base.mgr) @@ -352,7 +361,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for flush", + Doc: `Maximum number of flush requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 flush requests per second. +To use this setting, set quotaAndLimits.flushRate.enabled to true at the same time.`, Export: true, } p.MaxFlushRate.Init(base.mgr) @@ -399,6 +410,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.compactionRate.enabled", Version: "2.2.0", DefaultValue: "false", + Doc: "Whether manual compaction request throttling is enabled.", Export: true, } p.CompactionLimitEnabled.Init(base.mgr) @@ -417,7 +429,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for manualCompaction", + Doc: `Maximum number of manual-compaction requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 manual-compaction requests per second. +To use this setting, set quotaAndLimits.compaction.enabled to true at the same time.`, Export: true, } p.MaxCompactionRate.Init(base.mgr) @@ -446,9 +460,8 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.dml.enabled", Version: "2.2.0", DefaultValue: "false", - Doc: `dml limit rates, default no limit. -The maximum rate will not be greater than ` + "max" + `.`, - Export: true, + Doc: "Whether DML request throttling is enabled.", + Export: true, } p.DMLLimitEnabled.Init(base.mgr) @@ -470,7 +483,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data insertion rate per second. +Setting this item to 5 indicates that Milvus only allows data insertion at the rate of 5 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxInsertRate.Init(base.mgr) @@ -558,7 +573,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data insertion rate per collection per second. +Setting this item to 5 indicates that Milvus only allows data insertion to any collection at the rate of 5 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxInsertRatePerCollection.Init(base.mgr) @@ -822,7 +839,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data deletion rate per second. +Setting this item to 0.1 indicates that Milvus only allows data deletion at the rate of 0.1 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxDeleteRate.Init(base.mgr) @@ -910,7 +929,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data deletion rate per second. +Setting this item to 0.1 indicates that Milvus only allows data deletion from any collection at the rate of 0.1 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxDeleteRatePerCollection.Init(base.mgr) @@ -1161,9 +1182,8 @@ The maximum rate will not be greater than ` + "max" + `.`, Key: "quotaAndLimits.dql.enabled", Version: "2.2.0", DefaultValue: "false", - Doc: `dql limit rates, default no limit. -The maximum rate will not be greater than ` + "max" + `.`, - Export: true, + Doc: "Whether DQL request throttling is enabled.", + Export: true, } p.DQLLimitEnabled.Init(base.mgr) @@ -1181,7 +1201,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "vps (vectors per second), default no limit", + Doc: `Maximum number of vectors to search per second. +Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second no matter whether these 100 vectors are all in one search or scattered across multiple searches. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxSearchRate.Init(base.mgr) @@ -1261,7 +1283,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "vps (vectors per second), default no limit", + Doc: `Maximum number of vectors to search per collection per second. +Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second per collection no matter whether these 100 vectors are all in one search or scattered across multiple searches. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxSearchRatePerCollection.Init(base.mgr) @@ -1341,7 +1365,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "qps, default no limit", + Doc: `Maximum number of queries per second. +Setting this item to 100 indicates that Milvus only allows 100 queries per second. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxQueryRate.Init(base.mgr) @@ -1421,7 +1447,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "qps, default no limit", + Doc: `Maximum number of queries per collection per second. +Setting this item to 100 indicates that Milvus only allows 100 queries per collection per second. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxQueryRatePerCollection.Init(base.mgr) @@ -1500,6 +1528,7 @@ The maximum rate will not be greater than ` + "max" + `.`, Key: "quotaAndLimits.limits.maxCollectionNumPerDB", Version: "2.2.0", DefaultValue: "65536", + Doc: "Maximum number of collections per database.", Export: true, } p.MaxCollectionNumPerDB.Init(base.mgr) diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index 2a41d30b69072..04d1da31c72cb 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -121,7 +121,10 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "localhost:2379", PanicIfEmpty: true, - Export: true, + Doc: `Endpoints used to access etcd service. You can change this parameter as the endpoints of your own etcd cluster. +Environment variable: ETCD_ENDPOINTS +etcd preferentially acquires valid address from environment variable ETCD_ENDPOINTS when Milvus is started.`, + Export: true, } p.Endpoints.Init(base.mgr) @@ -159,8 +162,12 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "by-dev", PanicIfEmpty: true, - Doc: "The root path where data is stored in etcd", - Export: true, + Doc: `Root prefix of the key to where Milvus stores data in etcd. +It is recommended to change this parameter before starting Milvus for the first time. +To share an etcd instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. +Set an easy-to-identify root path for Milvus if etcd service already exists. +Changing this for an already running Milvus instance may result in failures to read legacy data.`, + Export: true, } p.RootPath.Init(base.mgr) @@ -169,8 +176,10 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "meta", PanicIfEmpty: true, - Doc: "metaRootPath = rootPath + '/' + metaSubPath", - Export: true, + Doc: `Sub-prefix of the key to where Milvus stores metadata-related information in etcd. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended to change this parameter before starting Milvus for the first time.`, + Export: true, } p.MetaSubPath.Init(base.mgr) @@ -186,8 +195,10 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "kv", PanicIfEmpty: true, - Doc: "kvRootPath = rootPath + '/' + kvSubPath", - Export: true, + Doc: `Sub-prefix of the key to where Milvus stores timestamps in etcd. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended not to change this parameter if there is no specific reason.`, + Export: true, } p.KvSubPath.Init(base.mgr) @@ -437,8 +448,10 @@ func (p *LocalStorageConfig) Init(base *BaseTable) { Key: "localStorage.path", Version: "2.0.0", DefaultValue: "/var/lib/milvus/data", - Doc: "please adjust in embedded Milvus: /tmp/milvus/data/", - Export: true, + Doc: `Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended to change this parameter before starting Milvus for the first time.`, + Export: true, } p.Path.Init(base.mgr) } @@ -603,7 +616,7 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.port", Version: "2.0.0", DefaultValue: "6650", - Doc: "Port of Pulsar", + Doc: "Port of Pulsar service.", Export: true, } p.Port.Init(base.mgr) @@ -623,7 +636,11 @@ func (p *PulsarConfig) Init(base *BaseTable) { port, _ := p.Port.get() return "pulsar://" + addr + ":" + port }, - Doc: "Address of pulsar", + Doc: `IP address of Pulsar service. +Environment variable: PULSAR_ADDRESS +pulsar.address and pulsar.port together generate the valid access to Pulsar. +Pulsar preferentially acquires the valid IP address from the environment variable PULSAR_ADDRESS when Milvus is started. +Default value applies when Pulsar is running on the same network with Milvus.`, Export: true, } p.Address.Init(base.mgr) @@ -632,7 +649,7 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.webport", Version: "2.0.0", DefaultValue: "80", - Doc: "Web port of pulsar, if you connect directly without proxy, should use 8080", + Doc: "Web port of of Pulsar service. If you connect direcly without proxy, should use 8080.", Export: true, } p.WebPort.Init(base.mgr) @@ -656,8 +673,10 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.maxMessageSize", Version: "2.0.0", DefaultValue: strconv.Itoa(SuggestPulsarMaxMessageSize), - Doc: "5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.", - Export: true, + Doc: `The maximum size of each message in Pulsar. Unit: Byte. +By default, Pulsar can transmit at most 5 MB of data in a single message. When the size of inserted data is greater than this value, proxy fragments the data into multiple messages to ensure that they can be transmitted correctly. +If the corresponding parameter in Pulsar remains unchanged, increasing this configuration will cause Milvus to fail, and reducing it produces no advantage.`, + Export: true, } p.MaxMessageSize.Init(base.mgr) @@ -665,7 +684,9 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.tenant", Version: "2.2.0", DefaultValue: "public", - Export: true, + Doc: `Pulsar can be provisioned for specific tenants with appropriate capacity allocated to the tenant. +To share a Pulsar instance among multiple Milvus instances, you can change this to an Pulsar tenant rather than the default one for each Milvus instance before you start them. However, if you do not want Pulsar multi-tenancy, you are advised to change msgChannel.chanNamePrefix.cluster to the different value.`, + Export: true, } p.Tenant.Init(base.mgr) @@ -673,6 +694,7 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.namespace", Version: "2.2.0", DefaultValue: "default", + Doc: "A Pulsar namespace is the administrative unit nomenclature within a tenant.", Export: true, } p.Namespace.Init(base.mgr) @@ -867,8 +889,10 @@ func (r *RocksmqConfig) Init(base *BaseTable) { r.Path = ParamItem{ Key: "rocksmq.path", Version: "2.0.0", - Doc: `The path where the message is stored in rocksmq -please adjust in embedded Milvus: /tmp/milvus/rdb_data`, + Doc: `Prefix of the key to where Milvus stores data in RocksMQ. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended to change this parameter before starting Milvus for the first time. +Set an easy-to-identify root key prefix for Milvus if etcd service already exists.`, Export: true, } r.Path.Init(base.mgr) @@ -886,7 +910,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.rocksmqPageSize", DefaultValue: strconv.FormatInt(64<<20, 10), Version: "2.0.0", - Doc: "64 MB, 64 * 1024 * 1024 bytes, The size of each page of messages in rocksmq", + Doc: "The maximum size of messages in each page in RocksMQ. Messages in RocksMQ are checked and cleared (when expired) in batch based on this parameters. Unit: Byte.", Export: true, } r.PageSize.Init(base.mgr) @@ -895,7 +919,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.retentionTimeInMinutes", DefaultValue: "4320", Version: "2.0.0", - Doc: "3 days, 3 * 24 * 60 minutes, The retention time of the message in rocksmq.", + Doc: "The maximum retention time of acked messages in RocksMQ. Acked messages in RocksMQ are retained for the specified period of time and then cleared. Unit: Minute.", Export: true, } r.RetentionTimeInMinutes.Init(base.mgr) @@ -904,7 +928,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.retentionSizeInMB", DefaultValue: "7200", Version: "2.0.0", - Doc: "8 GB, 8 * 1024 MB, The retention size of the message in rocksmq.", + Doc: "The maximum retention size of acked messages of each topic in RocksMQ. Acked messages in each topic are cleared if their size exceed this parameter. Unit: MB.", Export: true, } r.RetentionSizeInMB.Init(base.mgr) @@ -913,7 +937,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.compactionInterval", DefaultValue: "86400", Version: "2.0.0", - Doc: "1 day, trigger rocksdb compaction every day to remove deleted data", + Doc: "Time interval to trigger rocksdb compaction to remove deleted data. Unit: Second", Export: true, } r.CompactionInterval.Init(base.mgr) @@ -959,7 +983,7 @@ func (r *NatsmqConfig) Init(base *BaseTable) { Key: "natsmq.server.port", Version: "2.3.0", DefaultValue: "4222", - Doc: `Port for nats server listening`, + Doc: "Listening port of the NATS server.", Export: true, } r.ServerPort.Init(base.mgr) @@ -1096,7 +1120,7 @@ func (p *MinioConfig) Init(base *BaseTable) { Key: "minio.port", DefaultValue: "9000", Version: "2.0.0", - Doc: "Port of MinIO/S3", + Doc: "Port of MinIO or S3 service.", PanicIfEmpty: true, Export: true, } @@ -1116,7 +1140,11 @@ func (p *MinioConfig) Init(base *BaseTable) { port, _ := p.Port.get() return addr + ":" + port }, - Doc: "Address of MinIO/S3", + Doc: `IP address of MinIO or S3 service. +Environment variable: MINIO_ADDRESS +minio.address and minio.port together generate the valid access to MinIO or S3 service. +MinIO preferentially acquires the valid IP address from the environment variable MINIO_ADDRESS when Milvus is started. +Default value applies when MinIO or S3 is running on the same network with Milvus.`, Export: true, } p.Address.Init(base.mgr) @@ -1126,8 +1154,12 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "minioadmin", PanicIfEmpty: false, // tmp fix, need to be conditional - Doc: "accessKeyID of MinIO/S3", - Export: true, + Doc: `Access key ID that MinIO or S3 issues to user for authorized access. +Environment variable: MINIO_ACCESS_KEY_ID or minio.accessKeyID +minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. +This configuration must be set identical to the environment variable MINIO_ACCESS_KEY_ID, which is necessary for starting MinIO or S3. +The default value applies to MinIO or S3 service that started with the default docker-compose.yml file.`, + Export: true, } p.AccessKeyID.Init(base.mgr) @@ -1136,8 +1168,12 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "minioadmin", PanicIfEmpty: false, // tmp fix, need to be conditional - Doc: "MinIO/S3 encryption string", - Export: true, + Doc: `Secret key used to encrypt the signature string and verify the signature string on server. It must be kept strictly confidential and accessible only to the MinIO or S3 server and users. +Environment variable: MINIO_SECRET_ACCESS_KEY or minio.secretAccessKey +minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. +This configuration must be set identical to the environment variable MINIO_SECRET_ACCESS_KEY, which is necessary for starting MinIO or S3. +The default value applies to MinIO or S3 service that started with the default docker-compose.yml file.`, + Export: true, } p.SecretAccessKey.Init(base.mgr) @@ -1146,7 +1182,7 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "false", PanicIfEmpty: true, - Doc: "Access to MinIO/S3 with SSL", + Doc: "Switch value to control if to access the MinIO or S3 service through SSL.", Export: true, } p.UseSSL.Init(base.mgr) @@ -1164,8 +1200,13 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "a-bucket", PanicIfEmpty: true, - Doc: "Bucket name in MinIO/S3", - Export: true, + Doc: `Name of the bucket where Milvus stores data in MinIO or S3. +Milvus 2.0.0 does not support storing data in multiple buckets. +Bucket with this name will be created if it does not exist. If the bucket already exists and is accessible, it will be used directly. Otherwise, there will be an error. +To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. +The data will be stored in the local Docker if Docker is used to start the MinIO service locally. Ensure that there is sufficient storage space. +A bucket name is globally unique in one MinIO or S3 instance.`, + Export: true, } p.BucketName.Init(base.mgr) @@ -1180,8 +1221,12 @@ func (p *MinioConfig) Init(base *BaseTable) { return path.Clean(rootPath) }, PanicIfEmpty: false, - Doc: "The root path where the message is stored in MinIO/S3", - Export: true, + Doc: `Root prefix of the key to where Milvus stores data in MinIO or S3. +It is recommended to change this parameter before starting Milvus for the first time. +To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. +Set an easy-to-identify root key prefix for Milvus if etcd service already exists. +Changing this for an already running Milvus instance may result in failures to read legacy data.`, + Export: true, } p.RootPath.Init(base.mgr) diff --git a/scripts/run_go_codecov.sh b/scripts/run_go_codecov.sh index 174e730fa2ae2..661a3899997cf 100755 --- a/scripts/run_go_codecov.sh +++ b/scripts/run_go_codecov.sh @@ -35,6 +35,13 @@ fi # starting the timer beginTime=`date +%s` +pushd cmd/tools +$TEST_CMD -race -tags dynamic,test -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic ./... +if [ -f profile.out ]; then + grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ../${FILE_COVERAGE_INFO} + rm profile.out +fi +popd for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do $TEST_CMD -race -tags dynamic,test -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" if [ -f profile.out ]; then @@ -42,13 +49,6 @@ for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/g rm profile.out fi done -for d in $(go list ./cmd/tools/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do - $TEST_CMD -race -tags dynamic,test -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" - if [ -f profile.out ]; then - grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ../${FILE_COVERAGE_INFO} - rm profile.out - fi -done pushd pkg for d in $(go list ./... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do $TEST_CMD -race -tags dynamic,test -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"