From fed579220b2f599fe91e89260667fd82517c645f Mon Sep 17 00:00:00 2001 From: Ted Xu Date: Thu, 1 Aug 2024 12:38:13 +0800 Subject: [PATCH] enhance: update param documents (#35152) See: #35161 --------- Signed-off-by: Ted Xu Signed-off-by: Sumit Dubey --- cmd/tools/config-docs-generator/main.go | 16 +- configs/milvus.yaml | 391 +++++++++++++++--------- pkg/util/paramtable/component_param.go | 133 ++++---- pkg/util/paramtable/grpc_param.go | 7 +- pkg/util/paramtable/quota_param.go | 67 ++-- pkg/util/paramtable/service_param.go | 111 +++++-- 6 files changed, 470 insertions(+), 255 deletions(-) diff --git a/cmd/tools/config-docs-generator/main.go b/cmd/tools/config-docs-generator/main.go index 7463e8753d66a..7a7163d5c9e58 100644 --- a/cmd/tools/config-docs-generator/main.go +++ b/cmd/tools/config-docs-generator/main.go @@ -217,6 +217,9 @@ func (s Section) sectionPageContent() string { ret += fmt.Sprintf("# %s-related Configurations"+mdNextLine, s.Name) ret += s.descriptionContent() + mdNextLine for _, field := range s.Fields { + if len(field.Description) == 0 || field.Description[0] == "" { + continue + } ret += field.sectionPageContent() + mdNextLine } @@ -248,9 +251,6 @@ const fieldTableTemplate = ` func (f Field) sectionPageContent() string { ret := fmt.Sprintf("## `%s`", f.Name) + mdNextLine desp := f.descriptionContent() - if len(desp) > 0 { - desp = "\n" + desp + " " - } ret += fmt.Sprintf(fieldTableTemplate, f.Name, desp, f.DefaultValue) return ret } @@ -258,11 +258,13 @@ func (f Field) sectionPageContent() string { func (f Field) descriptionContent() string { var ret string lines := len(f.Description) - for i, descLine := range f.Description { - ret += fmt.Sprintf("
  • %s
    • ", descLine) - if i < lines-1 { - ret += "\n" + if lines > 1 { + for _, descLine := range f.Description { + ret += fmt.Sprintf("\n
  • %s
    • ", descLine) } + } else { + ret = fmt.Sprintf(" %s ", f.Description[0]) } + return ret } diff --git a/configs/milvus.yaml b/configs/milvus.yaml index e0c5ea35df15e..49e5b1ad04152 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -16,10 +16,24 @@ # Related configuration of etcd, used to store Milvus metadata & service discovery. etcd: + # Endpoints used to access etcd service. You can change this parameter as the endpoints of your own etcd cluster. + # Environment variable: ETCD_ENDPOINTS + # etcd preferentially acquires valid address from environment variable ETCD_ENDPOINTS when Milvus is started. endpoints: localhost:2379 - rootPath: by-dev # The root path where data is stored in etcd - metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath - kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath + # Root prefix of the key to where Milvus stores data in etcd. + # It is recommended to change this parameter before starting Milvus for the first time. + # To share an etcd instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. + # Set an easy-to-identify root path for Milvus if etcd service already exists. + # Changing this for an already running Milvus instance may result in failures to read legacy data. + rootPath: by-dev + # Sub-prefix of the key to where Milvus stores metadata-related information in etcd. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended to change this parameter before starting Milvus for the first time. + metaSubPath: meta + # Sub-prefix of the key to where Milvus stores timestamps in etcd. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended not to change this parameter if there is no specific reason. + kvSubPath: kv log: level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. # path is one of: @@ -68,20 +82,49 @@ tikv: tlsCACert: # path to your CACert file localStorage: - path: /var/lib/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/ + # Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended to change this parameter before starting Milvus for the first time. + path: /var/lib/milvus/data/ # Related configuration of MinIO/S3/GCS or any other service supports S3 API, which is responsible for data persistence for Milvus. # We refer to the storage service as MinIO/S3 in the following description for simplicity. minio: - address: localhost # Address of MinIO/S3 - port: 9000 # Port of MinIO/S3 - accessKeyID: minioadmin # accessKeyID of MinIO/S3 - secretAccessKey: minioadmin # MinIO/S3 encryption string - useSSL: false # Access to MinIO/S3 with SSL + # IP address of MinIO or S3 service. + # Environment variable: MINIO_ADDRESS + # minio.address and minio.port together generate the valid access to MinIO or S3 service. + # MinIO preferentially acquires the valid IP address from the environment variable MINIO_ADDRESS when Milvus is started. + # Default value applies when MinIO or S3 is running on the same network with Milvus. + address: localhost + port: 9000 # Port of MinIO or S3 service. + # Access key ID that MinIO or S3 issues to user for authorized access. + # Environment variable: MINIO_ACCESS_KEY_ID or minio.accessKeyID + # minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. + # This configuration must be set identical to the environment variable MINIO_ACCESS_KEY_ID, which is necessary for starting MinIO or S3. + # The default value applies to MinIO or S3 service that started with the default docker-compose.yml file. + accessKeyID: minioadmin + # Secret key used to encrypt the signature string and verify the signature string on server. It must be kept strictly confidential and accessible only to the MinIO or S3 server and users. + # Environment variable: MINIO_SECRET_ACCESS_KEY or minio.secretAccessKey + # minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. + # This configuration must be set identical to the environment variable MINIO_SECRET_ACCESS_KEY, which is necessary for starting MinIO or S3. + # The default value applies to MinIO or S3 service that started with the default docker-compose.yml file. + secretAccessKey: minioadmin + useSSL: false # Switch value to control if to access the MinIO or S3 service through SSL. ssl: tlsCACert: /path/to/public.crt # path to your CACert file - bucketName: a-bucket # Bucket name in MinIO/S3 - rootPath: files # The root path where the message is stored in MinIO/S3 + # Name of the bucket where Milvus stores data in MinIO or S3. + # Milvus 2.0.0 does not support storing data in multiple buckets. + # Bucket with this name will be created if it does not exist. If the bucket already exists and is accessible, it will be used directly. Otherwise, there will be an error. + # To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. + # The data will be stored in the local Docker if Docker is used to start the MinIO service locally. Ensure that there is sufficient storage space. + # A bucket name is globally unique in one MinIO or S3 instance. + bucketName: a-bucket + # Root prefix of the key to where Milvus stores data in MinIO or S3. + # It is recommended to change this parameter before starting Milvus for the first time. + # To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. + # Set an easy-to-identify root key prefix for Milvus if etcd service already exists. + # Changing this for an already running Milvus instance may result in failures to read legacy data. + rootPath: files # Whether to useIAM role to access S3/GCS instead of access/secret keys # For more information, refer to # aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html @@ -126,12 +169,22 @@ mq: # Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services. pulsar: - address: localhost # Address of pulsar - port: 6650 # Port of Pulsar - webport: 80 # Web port of pulsar, if you connect directly without proxy, should use 8080 - maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar. + # IP address of Pulsar service. + # Environment variable: PULSAR_ADDRESS + # pulsar.address and pulsar.port together generate the valid access to Pulsar. + # Pulsar preferentially acquires the valid IP address from the environment variable PULSAR_ADDRESS when Milvus is started. + # Default value applies when Pulsar is running on the same network with Milvus. + address: localhost + port: 6650 # Port of Pulsar service. + webport: 80 # Web port of of Pulsar service. If you connect direcly without proxy, should use 8080. + # The maximum size of each message in Pulsar. Unit: Byte. + # By default, Pulsar can transmit at most 5 MB of data in a single message. When the size of inserted data is greater than this value, proxy fragments the data into multiple messages to ensure that they can be transmitted correctly. + # If the corresponding parameter in Pulsar remains unchanged, increasing this configuration will cause Milvus to fail, and reducing it produces no advantage. + maxMessageSize: 5242880 + # Pulsar can be provisioned for specific tenants with appropriate capacity allocated to the tenant. + # To share a Pulsar instance among multiple Milvus instances, you can change this to an Pulsar tenant rather than the default one for each Milvus instance before you start them. However, if you do not want Pulsar multi-tenancy, you are advised to change msgChannel.chanNamePrefix.cluster to the different value. tenant: public - namespace: default + namespace: default # A Pulsar namespace is the administrative unit nomenclature within a tenant. requestTimeout: 60 # pulsar client global request timeout in seconds enableClientMetrics: false # Whether to register pulsar client metrics into milvus metrics path. @@ -151,21 +204,23 @@ pulsar: # readTimeout: 10 rocksmq: - # The path where the message is stored in rocksmq - # please adjust in embedded Milvus: /tmp/milvus/rdb_data + # Prefix of the key to where Milvus stores data in RocksMQ. + # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. + # It is recommended to change this parameter before starting Milvus for the first time. + # Set an easy-to-identify root key prefix for Milvus if etcd service already exists. path: /var/lib/milvus/rdb_data lrucacheratio: 0.06 # rocksdb cache memory ratio - rocksmqPageSize: 67108864 # 64 MB, 64 * 1024 * 1024 bytes, The size of each page of messages in rocksmq - retentionTimeInMinutes: 4320 # 3 days, 3 * 24 * 60 minutes, The retention time of the message in rocksmq. - retentionSizeInMB: 8192 # 8 GB, 8 * 1024 MB, The retention size of the message in rocksmq. - compactionInterval: 86400 # 1 day, trigger rocksdb compaction every day to remove deleted data + rocksmqPageSize: 67108864 # The maximum size of messages in each page in RocksMQ. Messages in RocksMQ are checked and cleared (when expired) in batch based on this parameters. Unit: Byte. + retentionTimeInMinutes: 4320 # The maximum retention time of acked messages in RocksMQ. Acked messages in RocksMQ are retained for the specified period of time and then cleared. Unit: Minute. + retentionSizeInMB: 8192 # The maximum retention size of acked messages of each topic in RocksMQ. Acked messages in each topic are cleared if their size exceed this parameter. Unit: MB. + compactionInterval: 86400 # Time interval to trigger rocksdb compaction to remove deleted data. Unit: Second compressionTypes: 0,0,7,7,7 # compaction compression type, only support use 0,7. 0 means not compress, 7 will use zstd. Length of types means num of rocksdb level. # natsmq configuration. # more detail: https://docs.nats.io/running-a-nats-service/configuration natsmq: server: - port: 4222 # Port for nats server listening + port: 4222 # Listening port of the NATS server. storeDir: /var/lib/milvus/nats # Directory to use for JetStream storage of nats maxFileStore: 17179869184 # Maximum size of the 'file' storage maxPayload: 8388608 # Maximum number of bytes in a message payload @@ -184,51 +239,53 @@ natsmq: # Related configuration of rootCoord, used to handle data definition language (DDL) and data control language (DCL) requests rootCoord: - dmlChannelNum: 16 # The number of dml channels created at system startup - maxPartitionNum: 1024 # Maximum number of partitions in a collection - minSegmentSizeToEnableIndex: 1024 # It's a threshold. When the segment size is less than this value, the segment will not be indexed + dmlChannelNum: 16 # The number of DML-Channels to create at the root coord startup. + # The maximum number of partitions in each collection. + # New partitions cannot be created if this parameter is set as 0 or 1. + # Range: [0, INT64MAX] + maxPartitionNum: 1024 + # The minimum row count of a segment required for creating index. + # Segments with smaller size than this parameter will not be indexed, and will be searched with brute force. + minSegmentSizeToEnableIndex: 1024 enableActiveStandby: false maxDatabaseNum: 64 # Maximum number of database maxGeneralCapacity: 65536 # upper limit for the sum of of product of partitionNumber and shardNumber gracefulStopTimeout: 5 # seconds. force stop node without graceful stop - ip: # if not specified, use the first unicastable address - port: 53100 + ip: # TCP/IP address of rootCoord. If not specified, use the first unicastable address + port: 53100 # TCP port of rootCoord grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the rootCoord can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the rootCoord can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on rootCoord can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on rootCoord can receive, unit: byte # Related configuration of proxy, used to validate client requests and reduce the returned results. proxy: - timeTickInterval: 200 # ms, the interval that proxy synchronize the time tick + timeTickInterval: 200 # The interval at which proxy synchronizes the time tick, unit: ms. healthCheckTimeout: 3000 # ms, the interval that to do component healthy check msgStream: timeTick: - bufSize: 512 - maxNameLength: 255 # Maximum length of name for a collection or alias - # Maximum number of fields in a collection. - # As of today (2.2.0 and after) it is strongly DISCOURAGED to set maxFieldNum >= 64. - # So adjust at your risk! - maxFieldNum: 64 - maxVectorFieldNum: 4 # Maximum number of vector fields in a collection. - maxShardNum: 16 # Maximum number of shards in a collection - maxDimension: 32768 # Maximum dimension of a vector + bufSize: 512 # The maximum number of messages can be buffered in the timeTick message stream of the proxy when producing messages. + maxNameLength: 255 # The maximum length of the name or alias that can be created in Milvus, including the collection name, collection alias, partition name, and field name. + maxFieldNum: 64 # The maximum number of field can be created when creating in a collection. It is strongly DISCOURAGED to set maxFieldNum >= 64. + maxVectorFieldNum: 4 # The maximum number of vector fields that can be specified in a collection. Value range: [1, 10]. + maxShardNum: 16 # The maximum number of shards can be created when creating in a collection. + maxDimension: 32768 # The maximum number of dimensions of a vector can have when creating in a collection. # Whether to produce gin logs.\n # please adjust in embedded Milvus: false ginLogging: true ginLogSkipPaths: / # skip url path for gin log - maxTaskNum: 1024 # max task number of proxy task queue + maxTaskNum: 1024 # The maximum number of tasks in the task queue of the proxy. mustUsePartitionKey: false # switch for whether proxy must use partition key for the collection accessLog: - enable: false # if use access log - minioEnable: false # if upload sealed access log file to minio - localPath: /tmp/milvus_access - filename: # Log filename, leave empty to use stdout. - maxSize: 64 # Max size for a single file, in MB. - rotatedTime: 0 # Max time for single access log file in seconds - remotePath: access_log/ # File path in minIO - remoteMaxTime: 0 # Max time for log file in minIO, in hours + enable: false # Whether to enable the access log feature. + minioEnable: false # Whether to upload local access log files to MinIO. This parameter can be specified when proxy.accessLog.filename is not empty. + localPath: /tmp/milvus_access # The local folder path where the access log file is stored. This parameter can be specified when proxy.accessLog.filename is not empty. + filename: # The name of the access log file. If you leave this parameter empty, access logs will be printed to stdout. + maxSize: 64 # The maximum size allowed for a single access log file. If the log file size reaches this limit, a rotation process will be triggered. This process seals the current access log file, creates a new log file, and clears the contents of the original log file. Unit: MB. + rotatedTime: 0 # The maximum time interval allowed for rotating a single access log file. Upon reaching the specified time interval, a rotation process is triggered, resulting in the creation of a new access log file and sealing of the previous one. Unit: seconds + remotePath: access_log/ # The path of the object storage for uploading access log files. + remoteMaxTime: 0 # The time interval allowed for uploading access log files. If the upload time of a log file exceeds this interval, the file will be deleted. Setting the value to 0 disables this feature. formatters: base: format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost]" @@ -248,29 +305,31 @@ proxy: port: # high-level restful api acceptTypeAllowInt64: true # high-level restful api, whether http client can deal with int64 enablePprof: true # Whether to enable pprof middleware on the metrics port - ip: # if not specified, use the first unicastable address - port: 19530 + ip: # TCP/IP address of proxy. If not specified, use the first unicastable address + port: 19530 # TCP port of proxy internalPort: 19529 grpc: - serverMaxSendSize: 268435456 - serverMaxRecvSize: 67108864 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 67108864 + serverMaxSendSize: 268435456 # The maximum size of each RPC request that the proxy can send, unit: byte + serverMaxRecvSize: 67108864 # The maximum size of each RPC request that the proxy can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on proxy can send, unit: byte + clientMaxRecvSize: 67108864 # The maximum size of each RPC request that the clients on proxy can receive, unit: byte # Related configuration of queryCoord, used to manage topology and load balancing for the query nodes, and handoff from growing segments to sealed segments. queryCoord: taskMergeCap: 1 taskExecutionCap: 256 - autoHandoff: true # Enable auto handoff - autoBalance: true # Enable auto balance + # Switch value to control if to automatically replace a growing segment with the corresponding indexed sealed segment when the growing segment reaches the sealing threshold. + # If this parameter is set false, Milvus simply searches the growing segments with brute force. + autoHandoff: true + autoBalance: true # Switch value to control if to automatically balance the memory usage among query nodes by distributing segment loading and releasing operations evenly. autoBalanceChannel: true # Enable auto balance channel balancer: ScoreBasedBalancer # auto balancer used for segments on queryNodes globalRowCountFactor: 0.1 # the weight used when balancing segments among queryNodes scoreUnbalanceTolerationFactor: 0.05 # the least value for unbalanced extent between from and to nodes when doing balance reverseUnBalanceTolerationFactor: 1.3 # the largest value for unbalanced extent between from and to nodes after doing balance - overloadedMemoryThresholdPercentage: 90 # The threshold percentage that memory overload - balanceIntervalSeconds: 60 - memoryUsageMaxDifferencePercentage: 30 + overloadedMemoryThresholdPercentage: 90 # The threshold of memory usage (in percentage) in a query node to trigger the sealed segment balancing. + balanceIntervalSeconds: 60 # The interval at which query coord balances the memory usage among query nodes. + memoryUsageMaxDifferencePercentage: 30 # The threshold of memory usage difference (in percentage) between any two query nodes to trigger the sealed segment balancing. rowCountFactor: 0.4 # the row count weight used when balancing segments among queryNodes segmentCountFactor: 0.4 # the segment count weight used when balancing segments among queryNodes globalSegmentCountFactor: 0.1 # the segment count weight used when balancing segments among queryNodes @@ -307,23 +366,26 @@ queryCoord: collectionObserverInterval: 200 # the interval of collection observer checkExecutedFlagInterval: 100 # the interval of check executed flag to force to pull dist cleanExcludeSegmentInterval: 60 # the time duration of clean pipeline exclude segment which used for filter invalid data, in seconds - ip: # if not specified, use the first unicastable address - port: 19531 + ip: # TCP/IP address of queryCoord. If not specified, use the first unicastable address + port: 19531 # TCP port of queryCoord grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the queryCoord can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the queryCoord can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on queryCoord can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on queryCoord can receive, unit: byte # Related configuration of queryNode, used to run hybrid search between vector and scalar data. queryNode: stats: - publishInterval: 1000 # Interval for querynode to report node information (milliseconds) + publishInterval: 1000 # The interval that query node publishes the node statistics information, including segment status, cpu usage, memory usage, health status, etc. Unit: ms. segcore: knowhereThreadPoolNumRatio: 4 # The number of threads in knowhere's thread pool. If disk is enabled, the pool size will multiply with knowhereThreadPoolNumRatio([1, 32]). - chunkRows: 128 # The number of vectors in a chunk. + chunkRows: 128 # Row count by which Segcore divides a segment into chunks. interimIndex: - enableIndex: true # Enable segment build with index to accelerate vector search when segment is in growing or binlog. + # Whether to create a temporary index for growing segments and sealed segments not yet indexed, improving search performance. + # Milvus will eventually seals and indexes all segments, but enabling this optimizes search performance for immediate queries following data insertion. + # This defaults to true, indicating that Milvus creates temporary index for growing segments and the sealed segments that are not indexed upon searches. + enableIndex: true nlist: 128 # temp index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist memExpansionRate: 1.15 # extra memory needed by building interim index @@ -384,18 +446,18 @@ queryNode: maxPendingTaskPerUser: 1024 # Max pending task per user in scheduler dataSync: flowGraph: - maxQueueLength: 16 # Maximum length of task queue in flowgraph + maxQueueLength: 16 # The maximum size of task queue cache in flow graph in query node. maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph enableSegmentPrune: false # use partition stats to prune data in search/query on shard delegator queryStreamBatchSize: 4194304 # return batch size of stream query bloomFilterApplyParallelFactor: 4 # parallel factor when to apply pk to bloom filter, default to 4*CPU_CORE_NUM - ip: # if not specified, use the first unicastable address - port: 21123 + ip: # TCP/IP address of queryNode. If not specified, use the first unicastable address + port: 21123 # TCP port of queryNode grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the queryNode can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the queryNode can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on queryNode can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on queryNode can receive, unit: byte indexCoord: bindIndexNodeMode: @@ -411,13 +473,13 @@ indexNode: buildParallel: 1 enableDisk: true # enable index node build disk vector index maxDiskUsagePercentage: 95 - ip: # if not specified, use the first unicastable address - port: 21121 + ip: # TCP/IP address of indexNode. If not specified, use the first unicastable address + port: 21121 # TCP port of indexNode grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the indexNode can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the indexNode can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on indexNode can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on indexNode can receive, unit: byte dataCoord: channel: @@ -428,11 +490,11 @@ dataCoord: checkInterval: 1 # The interval in seconds with which the channel manager advances channel states notifyChannelOperationTimeout: 5 # Timeout notifing channel operations (in seconds). segment: - maxSize: 1024 # Maximum size of a segment in MB + maxSize: 1024 # The maximum size of a segment, unit: MB. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed. diskSegmentMaxSize: 2048 # Maximun size of a segment in MB for collection which has Disk index - sealProportion: 0.12 + sealProportion: 0.12 # The minimum proportion to datacoord.segment.maxSize to seal a segment. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed. sealProportionJitter: 0.1 # segment seal proportion jitter ratio, default value 0.1(10%), if seal proportion is 12%, with jitter=0.1, the actuall applied ratio will be 10.8~12% - assignmentExpiration: 2000 # The time of the assignment expiration in ms + assignmentExpiration: 2000 # Expiration time of the segment assignment, unit: ms allocLatestExpireAttempt: 200 # The time attempting to alloc latest lastExpire from rootCoord after restart maxLife: 86400 # The max lifetime of segment in seconds, 24*60*60 # If a segment didn't accept dml records in maxIdleTime and the size of segment is greater than @@ -458,8 +520,12 @@ dataCoord: growingSegmentsMemSize: 4096 autoUpgradeSegmentIndex: false # whether auto upgrade segment index to index engine's version segmentFlushInterval: 2 # the minimal interval duration(unit: Seconds) between flusing operation on same segment - enableCompaction: true # Enable data segment compaction + # Switch value to control if to enable segment compaction. + # Compaction merges small-size segments into a large segment, and clears the entities deleted beyond the rentention duration of Time Travel. + enableCompaction: true compaction: + # Switch value to control if to enable automatic segment compaction during which data coord locates and merges compactable segments in the background. + # This configuration takes effect only when dataCoord.enableCompaction is set as true. enableAutoCompaction: true indexBasedCompaction: true rpcTimeout: 10 @@ -489,11 +555,11 @@ dataCoord: deltalogMinNum: 10 # The minimum number of deltalog files to force trigger a LevelZero Compaction deltalogMaxNum: 30 # The maxmum number of deltalog files to force trigger a LevelZero Compaction, default as 30 syncSegmentsInterval: 300 # The time interval for regularly syncing segments - enableGarbageCollection: true + enableGarbageCollection: true # Switch value to control if to enable garbage collection to clear the discarded data in MinIO or S3 service. gc: - interval: 3600 # meta-based gc scanning interval in seconds - missingTolerance: 86400 # orphan file gc tolerance duration in seconds (orphan file which last modified time before the tolerance interval ago will be deleted) - dropTolerance: 10800 # meta-based gc tolerace duration in seconds (file which meta is marked as dropped before the tolerace interval ago will be deleted) + interval: 3600 # The interval at which data coord performs garbage collection, unit: second. + missingTolerance: 86400 # The retention duration of the unrecorded binary log (binlog) files. Setting a reasonably large value for this parameter avoids erroneously deleting the newly created binlog files that lack metadata. Unit: second. + dropTolerance: 10800 # The retention duration of the binlog files of the deleted segments before they are cleared, unit: second. removeConcurrent: 32 # number of concurrent goroutines to remove dropped s3 objects scanInterval: 168 # orphan file (file on oss but has not been registered on meta) on object storage garbage collection scanning interval in hours enableActiveStandby: false @@ -514,13 +580,13 @@ dataCoord: clusteringCompactionUsage: 16 # slot usage of clustering compaction job. mixCompactionUsage: 8 # slot usage of mix compaction job. l0DeleteCompactionUsage: 8 # slot usage of l0 compaction job. - ip: # if not specified, use the first unicastable address - port: 13333 + ip: # TCP/IP address of dataCoord. If not specified, use the first unicastable address + port: 13333 # TCP port of dataCoord grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the dataCoord can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the dataCoord can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on dataCoord can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on dataCoord can receive, unit: byte dataNode: dataSync: @@ -533,7 +599,10 @@ dataNode: skipNum: 4 # Consume one for every n records skipped coldTime: 60 # Turn on skip mode after there are only timetick msg for x seconds segment: - insertBufSize: 16777216 # Max buffer size to flush for a single segment. + # The maximum size of each binlog file in a segment buffered in memory. Binlog files whose size exceeds this value are then flushed to MinIO or S3 service. + # Unit: Byte + # Setting this parameter too small causes the system to store a small amount of data too frequently. Setting it too large increases the system's demand for memory. + insertBufSize: 16777216 deleteBufBytes: 16777216 # Max buffer size in bytes to flush del for a single channel, default as 16MB syncPeriod: 600 # The period to sync segments if buffer is not empty. memory: @@ -568,23 +637,29 @@ dataNode: clusteringCompaction: memoryBufferRatio: 0.1 # The ratio of memory buffer of clustering compaction. Data larger than threshold will be flushed to storage. workPoolSize: 8 # worker pool size for one clustering compaction job. - ip: # if not specified, use the first unicastable address - port: 21124 + ip: # TCP/IP address of dataNode. If not specified, use the first unicastable address + port: 21124 # TCP port of dataNode grpc: - serverMaxSendSize: 536870912 - serverMaxRecvSize: 268435456 - clientMaxSendSize: 268435456 - clientMaxRecvSize: 536870912 + serverMaxSendSize: 536870912 # The maximum size of each RPC request that the dataNode can send, unit: byte + serverMaxRecvSize: 268435456 # The maximum size of each RPC request that the dataNode can receive, unit: byte + clientMaxSendSize: 268435456 # The maximum size of each RPC request that the clients on dataNode can send, unit: byte + clientMaxRecvSize: 536870912 # The maximum size of each RPC request that the clients on dataNode can receive, unit: byte # Configures the system log output. log: - level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + # Milvus log level. Option: debug, info, warn, error, panic, and fatal. + # It is recommended to use debug level under test and development environments, and info level in production environment. + level: info file: - rootPath: # root dir path to put logs, default "" means no log file will print. please adjust in embedded Milvus: /tmp/milvus/logs - maxSize: 300 # MB - maxAge: 10 # Maximum time for log retention in day. - maxBackups: 20 - format: text # text or json + # Root path to the log files. + # The default value is set empty, indicating to output log files to standard output (stdout) and standard error (stderr). + # If this parameter is set to a valid local path, Milvus writes and stores log files in this path. + # Set this parameter as the path that you have permission to write. + rootPath: + maxSize: 300 # The maximum size of a log file, unit: MB. + maxAge: 10 # The maximum retention time before a log file is automatically cleared, unit: day. The minimum value is 1. + maxBackups: 20 # The maximum number of log files to back up, unit: day. The minimum value is 1. + format: text # Milvus log format. Option: text and JSON stdout: true # Stdout enable or not grpc: @@ -611,10 +686,10 @@ tls: caPemPath: configs/cert/ca.pem common: - defaultPartitionName: _default # default partition name for a collection - defaultIndexName: _default_idx # default index name + defaultPartitionName: _default # Name of the default partition when a collection is created + defaultIndexName: _default_idx # Name of the index when it is created with name unspecified entityExpiration: -1 # Entity expiration in seconds, CAUTION -1 means never expire - indexSliceSize: 16 # MB + indexSliceSize: 16 # Index slice size in MB threadCoreCoefficient: highPriority: 10 # This parameter specify how many times the number of threads is the number of cores in high priority pool middlePriority: 5 # This parameter specify how many times the number of threads is the number of cores in middle priority pool @@ -655,7 +730,10 @@ common: storage: scheme: s3 enablev2: false - ttMsgEnabled: true # Whether the instance disable sending ts messages + # Whether to disable the internal time messaging mechanism for the system. + # If disabled (set to false), the system will not allow DML operations, including insertion, deletion, queries, and searches. + # This helps Milvus-CDC synchronize incremental data + ttMsgEnabled: true traceLogMode: 0 # trace request info bloomFilterSize: 100000 # bloom filter initial size bloomFilterType: BlockedBloomFilter # bloom filter type, support BasicBloomFilter and BlockedBloomFilter @@ -687,43 +765,62 @@ quotaAndLimits: allocWaitInterval: 1000 # retry wait duration when delete alloc forward data rate failed, in millisecond complexDeleteLimitEnable: false # whether complex delete check forward data by limiter maxCollectionNum: 65536 - maxCollectionNumPerDB: 65536 + maxCollectionNumPerDB: 65536 # Maximum number of collections per database. maxInsertSize: -1 # maximum size of a single insert request, in bytes, -1 means no limit maxResourceGroupNumOfQueryNode: 1024 # maximum number of resource groups of query nodes ddl: - enabled: false - collectionRate: -1 # qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection - partitionRate: -1 # qps, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition + enabled: false # Whether DDL request throttling is enabled. + # Maximum number of collection-related DDL requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 collection-related DDL requests per second, including collection creation requests, collection drop requests, collection load requests, and collection release requests. + # To use this setting, set quotaAndLimits.ddl.enabled to true at the same time. + collectionRate: -1 + # Maximum number of partition-related DDL requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including partition creation requests, partition drop requests, partition load requests, and partition release requests. + # To use this setting, set quotaAndLimits.ddl.enabled to true at the same time. + partitionRate: -1 db: collectionRate: -1 # qps of db level , default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection partitionRate: -1 # qps of db level, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition indexRate: - enabled: false - max: -1 # qps, default no limit, rate for CreateIndex, DropIndex + enabled: false # Whether index-related request throttling is enabled. + # Maximum number of index-related requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including index creation requests and index drop requests. + # To use this setting, set quotaAndLimits.indexRate.enabled to true at the same time. + max: -1 db: max: -1 # qps of db level, default no limit, rate for CreateIndex, DropIndex flushRate: - enabled: true - max: -1 # qps, default no limit, rate for flush + enabled: true # Whether flush request throttling is enabled. + # Maximum number of flush requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 flush requests per second. + # To use this setting, set quotaAndLimits.flushRate.enabled to true at the same time. + max: -1 collection: max: 0.1 # qps, default no limit, rate for flush at collection level. db: max: -1 # qps of db level, default no limit, rate for flush compactionRate: - enabled: false - max: -1 # qps, default no limit, rate for manualCompaction + enabled: false # Whether manual compaction request throttling is enabled. + # Maximum number of manual-compaction requests per second. + # Setting this item to 10 indicates that Milvus processes no more than 10 manual-compaction requests per second. + # To use this setting, set quotaAndLimits.compaction.enabled to true at the same time. + max: -1 db: max: -1 # qps of db level, default no limit, rate for manualCompaction dml: - # dml limit rates, default no limit. - # The maximum rate will not be greater than max. - enabled: false + enabled: false # Whether DML request throttling is enabled. insertRate: - max: -1 # MB/s, default no limit + # Highest data insertion rate per second. + # Setting this item to 5 indicates that Milvus only allows data insertion at the rate of 5 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 db: max: -1 # MB/s, default no limit collection: - max: -1 # MB/s, default no limit + # Highest data insertion rate per collection per second. + # Setting this item to 5 indicates that Milvus only allows data insertion to any collection at the rate of 5 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 partition: max: -1 # MB/s, default no limit upsertRate: @@ -735,11 +832,17 @@ quotaAndLimits: partition: max: -1 # MB/s, default no limit deleteRate: - max: -1 # MB/s, default no limit + # Highest data deletion rate per second. + # Setting this item to 0.1 indicates that Milvus only allows data deletion at the rate of 0.1 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 db: max: -1 # MB/s, default no limit collection: - max: -1 # MB/s, default no limit + # Highest data deletion rate per second. + # Setting this item to 0.1 indicates that Milvus only allows data deletion from any collection at the rate of 0.1 MB/s. + # To use this setting, set quotaAndLimits.dml.enabled to true at the same time. + max: -1 partition: max: -1 # MB/s, default no limit bulkLoadRate: @@ -751,23 +854,33 @@ quotaAndLimits: partition: max: -1 # MB/s, default no limit, not support yet. TODO: limit partition bulkLoad rate dql: - # dql limit rates, default no limit. - # The maximum rate will not be greater than max. - enabled: false + enabled: false # Whether DQL request throttling is enabled. searchRate: - max: -1 # vps (vectors per second), default no limit + # Maximum number of vectors to search per second. + # Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second no matter whether these 100 vectors are all in one search or scattered across multiple searches. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 db: max: -1 # vps (vectors per second), default no limit collection: - max: -1 # vps (vectors per second), default no limit + # Maximum number of vectors to search per collection per second. + # Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second per collection no matter whether these 100 vectors are all in one search or scattered across multiple searches. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 partition: max: -1 # vps (vectors per second), default no limit queryRate: - max: -1 # qps, default no limit + # Maximum number of queries per second. + # Setting this item to 100 indicates that Milvus only allows 100 queries per second. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 db: max: -1 # qps, default no limit collection: - max: -1 # qps, default no limit + # Maximum number of queries per collection per second. + # Setting this item to 100 indicates that Milvus only allows 100 queries per collection per second. + # To use this setting, set quotaAndLimits.dql.enabled to true at the same time. + max: -1 partition: max: -1 # qps, default no limit limitWriting: diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 640eddc842c90..28d267c4b44dc 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -422,7 +422,7 @@ func (p *commonConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "_default", Forbidden: true, - Doc: "default partition name for a collection", + Doc: "Name of the default partition when a collection is created", Export: true, } p.DefaultPartitionName.Init(base.mgr) @@ -431,7 +431,7 @@ func (p *commonConfig) init(base *BaseTable) { Key: "common.defaultIndexName", Version: "2.0.0", DefaultValue: "_default_idx", - Doc: "default index name", + Doc: "Name of the index when it is created with name unspecified", Export: true, } p.DefaultIndexName.Init(base.mgr) @@ -469,7 +469,7 @@ This configuration is only used by querynode and indexnode, it selects CPU instr Key: "common.indexSliceSize", Version: "2.0.0", DefaultValue: strconv.Itoa(DefaultIndexSliceSize), - Doc: "MB", + Doc: "Index slice size in MB", Export: true, } p.IndexSliceSize.Init(base.mgr) @@ -750,8 +750,10 @@ like the old password verification when updating the credential`, Key: "common.ttMsgEnabled", Version: "2.3.2", DefaultValue: "true", - Doc: "Whether the instance disable sending ts messages", - Export: true, + Doc: `Whether to disable the internal time messaging mechanism for the system. +If disabled (set to false), the system will not allow DML operations, including insertion, deletion, queries, and searches. +This helps Milvus-CDC synchronize incremental data`, + Export: true, } p.TTMsgEnabled.Init(base.mgr) @@ -989,16 +991,20 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.level", DefaultValue: "info", Version: "2.0.0", - Doc: "Only supports debug, info, warn, error, panic, or fatal. Default 'info'.", - Export: true, + Doc: `Milvus log level. Option: debug, info, warn, error, panic, and fatal. +It is recommended to use debug level under test and development environments, and info level in production environment.`, + Export: true, } l.Level.Init(base.mgr) l.RootPath = ParamItem{ Key: "log.file.rootPath", Version: "2.0.0", - Doc: "root dir path to put logs, default \"\" means no log file will print. please adjust in embedded Milvus: /tmp/milvus/logs", - Export: true, + Doc: `Root path to the log files. +The default value is set empty, indicating to output log files to standard output (stdout) and standard error (stderr). +If this parameter is set to a valid local path, Milvus writes and stores log files in this path. +Set this parameter as the path that you have permission to write.`, + Export: true, } l.RootPath.Init(base.mgr) @@ -1006,7 +1012,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.file.maxSize", DefaultValue: "300", Version: "2.0.0", - Doc: "MB", + Doc: "The maximum size of a log file, unit: MB.", Export: true, } l.MaxSize.Init(base.mgr) @@ -1015,7 +1021,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.file.maxAge", DefaultValue: "10", Version: "2.0.0", - Doc: "Maximum time for log retention in day.", + Doc: "The maximum retention time before a log file is automatically cleared, unit: day. The minimum value is 1.", Export: true, } l.MaxAge.Init(base.mgr) @@ -1024,6 +1030,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.file.maxBackups", DefaultValue: "20", Version: "2.0.0", + Doc: "The maximum number of log files to back up, unit: day. The minimum value is 1.", Export: true, } l.MaxBackups.Init(base.mgr) @@ -1032,7 +1039,7 @@ func (l *logConfig) init(base *BaseTable) { Key: "log.format", DefaultValue: "text", Version: "2.0.0", - Doc: "text or json", + Doc: "Milvus log format. Option: text and JSON", Export: true, } l.Format.Init(base.mgr) @@ -1073,7 +1080,7 @@ func (p *rootCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "16", Forbidden: true, - Doc: "The number of dml channels created at system startup", + Doc: "The number of DML-Channels to create at the root coord startup.", Export: true, } p.DmlChannelNum.Init(base.mgr) @@ -1082,8 +1089,10 @@ func (p *rootCoordConfig) init(base *BaseTable) { Key: "rootCoord.maxPartitionNum", Version: "2.0.0", DefaultValue: "1024", - Doc: "Maximum number of partitions in a collection", - Export: true, + Doc: `The maximum number of partitions in each collection. +New partitions cannot be created if this parameter is set as 0 or 1. +Range: [0, INT64MAX]`, + Export: true, } p.MaxPartitionNum.Init(base.mgr) @@ -1091,8 +1100,9 @@ func (p *rootCoordConfig) init(base *BaseTable) { Key: "rootCoord.minSegmentSizeToEnableIndex", Version: "2.0.0", DefaultValue: "1024", - Doc: "It's a threshold. When the segment size is less than this value, the segment will not be indexed", - Export: true, + Doc: `The minimum row count of a segment required for creating index. +Segments with smaller size than this parameter will not be indexed, and will be searched with brute force.`, + Export: true, } p.MinSegmentSizeToEnableIndex.Init(base.mgr) @@ -1206,7 +1216,7 @@ func (p *proxyConfig) init(base *BaseTable) { Version: "2.2.0", DefaultValue: "200", PanicIfEmpty: true, - Doc: "ms, the interval that proxy synchronize the time tick", + Doc: "The interval at which proxy synchronizes the time tick, unit: ms.", Export: true, } p.TimeTickInterval.Init(base.mgr) @@ -1227,6 +1237,7 @@ func (p *proxyConfig) init(base *BaseTable) { Version: "2.2.0", DefaultValue: "512", PanicIfEmpty: true, + Doc: "The maximum number of messages can be buffered in the timeTick message stream of the proxy when producing messages.", Export: true, } p.MsgStreamTimeTickBufSize.Init(base.mgr) @@ -1236,7 +1247,7 @@ func (p *proxyConfig) init(base *BaseTable) { DefaultValue: "255", Version: "2.0.0", PanicIfEmpty: true, - Doc: "Maximum length of name for a collection or alias", + Doc: "The maximum length of the name or alias that can be created in Milvus, including the collection name, collection alias, partition name, and field name.", Export: true, } p.MaxNameLength.Init(base.mgr) @@ -1270,10 +1281,8 @@ func (p *proxyConfig) init(base *BaseTable) { DefaultValue: "64", Version: "2.0.0", PanicIfEmpty: true, - Doc: `Maximum number of fields in a collection. -As of today (2.2.0 and after) it is strongly DISCOURAGED to set maxFieldNum >= 64. -So adjust at your risk!`, - Export: true, + Doc: "The maximum number of field can be created when creating in a collection. It is strongly DISCOURAGED to set maxFieldNum >= 64.", + Export: true, } p.MaxFieldNum.Init(base.mgr) @@ -1282,7 +1291,7 @@ So adjust at your risk!`, Version: "2.4.0", DefaultValue: "4", PanicIfEmpty: true, - Doc: "Maximum number of vector fields in a collection.", + Doc: "The maximum number of vector fields that can be specified in a collection. Value range: [1, 10].", Export: true, } p.MaxVectorFieldNum.Init(base.mgr) @@ -1296,7 +1305,7 @@ So adjust at your risk!`, DefaultValue: "16", Version: "2.0.0", PanicIfEmpty: true, - Doc: "Maximum number of shards in a collection", + Doc: "The maximum number of shards can be created when creating in a collection.", Export: true, } p.MaxShardNum.Init(base.mgr) @@ -1306,7 +1315,7 @@ So adjust at your risk!`, DefaultValue: "32768", Version: "2.0.0", PanicIfEmpty: true, - Doc: "Maximum dimension of a vector", + Doc: "The maximum number of dimensions of a vector can have when creating in a collection.", Export: true, } p.MaxDimension.Init(base.mgr) @@ -1315,7 +1324,7 @@ So adjust at your risk!`, Key: "proxy.maxTaskNum", Version: "2.2.0", DefaultValue: "10000", - Doc: "max task number of proxy task queue", + Doc: "The maximum number of tasks in the task queue of the proxy.", Export: true, } p.MaxTaskNum.Init(base.mgr) @@ -1366,7 +1375,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.enable", Version: "2.2.0", DefaultValue: "false", - Doc: "if use access log", + Doc: "Whether to enable the access log feature.", Export: true, } p.AccessLog.Enable.Init(base.mgr) @@ -1375,7 +1384,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.minioEnable", Version: "2.2.0", DefaultValue: "false", - Doc: "if upload sealed access log file to minio", + Doc: "Whether to upload local access log files to MinIO. This parameter can be specified when proxy.accessLog.filename is not empty.", Export: true, } p.AccessLog.MinioEnable.Init(base.mgr) @@ -1384,6 +1393,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.localPath", Version: "2.2.0", DefaultValue: "/tmp/milvus_access", + Doc: "The local folder path where the access log file is stored. This parameter can be specified when proxy.accessLog.filename is not empty.", Export: true, } p.AccessLog.LocalPath.Init(base.mgr) @@ -1392,7 +1402,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.filename", Version: "2.2.0", DefaultValue: "", - Doc: "Log filename, leave empty to use stdout.", + Doc: "The name of the access log file. If you leave this parameter empty, access logs will be printed to stdout.", Export: true, } p.AccessLog.Filename.Init(base.mgr) @@ -1401,7 +1411,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.maxSize", Version: "2.2.0", DefaultValue: "64", - Doc: "Max size for a single file, in MB.", + Doc: "The maximum size allowed for a single access log file. If the log file size reaches this limit, a rotation process will be triggered. This process seals the current access log file, creates a new log file, and clears the contents of the original log file. Unit: MB.", Export: true, } p.AccessLog.MaxSize.Init(base.mgr) @@ -1428,7 +1438,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.maxBackups", Version: "2.2.0", DefaultValue: "8", - Doc: "Maximum number of old log files to retain.", + Doc: "The maximum number of sealed access log files that can be retained. If the number of sealed access log files exceeds this limit, the oldest one will be deleted.", } p.AccessLog.MaxBackups.Init(base.mgr) @@ -1436,7 +1446,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.rotatedTime", Version: "2.2.0", DefaultValue: "0", - Doc: "Max time for single access log file in seconds", + Doc: "The maximum time interval allowed for rotating a single access log file. Upon reaching the specified time interval, a rotation process is triggered, resulting in the creation of a new access log file and sealing of the previous one. Unit: seconds", Export: true, } p.AccessLog.RotatedTime.Init(base.mgr) @@ -1445,7 +1455,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.remotePath", Version: "2.2.0", DefaultValue: "access_log/", - Doc: "File path in minIO", + Doc: "The path of the object storage for uploading access log files.", Export: true, } p.AccessLog.RemotePath.Init(base.mgr) @@ -1454,7 +1464,7 @@ please adjust in embedded Milvus: false`, Key: "proxy.accessLog.remoteMaxTime", Version: "2.2.0", DefaultValue: "0", - Doc: "Max time for log file in minIO, in hours", + Doc: "The time interval allowed for uploading access log files. If the upload time of a log file exceeds this interval, the file will be deleted. Setting the value to 0 disables this feature.", Export: true, } p.AccessLog.RemoteMaxTime.Init(base.mgr) @@ -1463,7 +1473,7 @@ please adjust in embedded Milvus: false`, KeyPrefix: "proxy.accessLog.formatters.", Version: "2.3.4", Export: true, - Doc: "access log formatters for specified methods, if not set, use the base formatter.", + Doc: "Access log formatters for specified methods, if not set, use the base formatter.", } p.AccessLog.Formatter.Init(base.mgr) @@ -1717,8 +1727,9 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "true", PanicIfEmpty: true, - Doc: "Enable auto handoff", - Export: true, + Doc: `Switch value to control if to automatically replace a growing segment with the corresponding indexed sealed segment when the growing segment reaches the sealing threshold. +If this parameter is set false, Milvus simply searches the growing segments with brute force.`, + Export: true, } p.AutoHandoff.Init(base.mgr) @@ -1727,7 +1738,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "true", PanicIfEmpty: true, - Doc: "Enable auto balance", + Doc: "Switch value to control if to automatically balance the memory usage among query nodes by distributing segment loading and releasing operations evenly.", Export: true, } p.AutoBalance.Init(base.mgr) @@ -1847,7 +1858,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "90", PanicIfEmpty: true, - Doc: "The threshold percentage that memory overload", + Doc: "The threshold of memory usage (in percentage) in a query node to trigger the sealed segment balancing.", Export: true, } p.OverloadedMemoryThresholdPercentage.Init(base.mgr) @@ -1857,6 +1868,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "60", PanicIfEmpty: true, + Doc: "The interval at which query coord balances the memory usage among query nodes.", Export: true, } p.BalanceIntervalSeconds.Init(base.mgr) @@ -1896,6 +1908,7 @@ func (p *queryCoordConfig) init(base *BaseTable) { Version: "2.0.0", DefaultValue: "30", PanicIfEmpty: true, + Doc: "The threshold of memory usage difference (in percentage) between any two query nodes to trigger the sealed segment balancing.", Export: true, } p.MemoryUsageMaxDifferencePercentage.Init(base.mgr) @@ -2314,7 +2327,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { Key: "queryNode.dataSync.flowGraph.maxQueueLength", Version: "2.0.0", DefaultValue: "16", - Doc: "Maximum length of task queue in flowgraph", + Doc: "The maximum size of task queue cache in flow graph in query node.", Export: true, } p.FlowGraphMaxQueueLength.Init(base.mgr) @@ -2332,7 +2345,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { Key: "queryNode.stats.publishInterval", Version: "2.0.0", DefaultValue: "1000", - Doc: "Interval for querynode to report node information (milliseconds)", + Doc: "The interval that query node publishes the node statistics information, including segment status, cpu usage, memory usage, health status, etc. Unit: ms.", Export: true, } p.StatsPublishInterval.Init(base.mgr) @@ -2366,7 +2379,7 @@ func (p *queryNodeConfig) init(base *BaseTable) { } return v }, - Doc: "The number of vectors in a chunk.", + Doc: "Row count by which Segcore divides a segment into chunks.", Export: true, } p.ChunkRows.Init(base.mgr) @@ -2375,8 +2388,10 @@ func (p *queryNodeConfig) init(base *BaseTable) { Key: "queryNode.segcore.interimIndex.enableIndex", Version: "2.0.0", DefaultValue: "false", - Doc: "Enable segment build with index to accelerate vector search when segment is in growing or binlog.", - Export: true, + Doc: `Whether to create a temporary index for growing segments and sealed segments not yet indexed, improving search performance. +Milvus will eventually seals and indexes all segments, but enabling this optimizes search performance for immediate queries following data insertion. +This defaults to true, indicating that Milvus creates temporary index for growing segments and the sealed segments that are not indexed upon searches.`, + Export: true, } p.EnableTempSegmentIndex.Init(base.mgr) @@ -3108,7 +3123,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { Key: "dataCoord.segment.maxSize", Version: "2.0.0", DefaultValue: "1024", - Doc: "Maximum size of a segment in MB", + Doc: "The maximum size of a segment, unit: MB. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed.", Export: true, } p.SegmentMaxSize.Init(base.mgr) @@ -3126,6 +3141,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { Key: "dataCoord.segment.sealProportion", Version: "2.0.0", DefaultValue: "0.12", + Doc: "The minimum proportion to datacoord.segment.maxSize to seal a segment. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed.", Export: true, } p.SegmentSealProportion.Init(base.mgr) @@ -3143,7 +3159,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { Key: "dataCoord.segment.assignmentExpiration", Version: "2.0.0", DefaultValue: "2000", - Doc: "The time of the assignment expiration in ms", + Doc: "Expiration time of the segment assignment, unit: ms", Export: true, } p.SegAssignmentExpiration.Init(base.mgr) @@ -3210,8 +3226,9 @@ exceeds this threshold, the largest growing segment will be sealed.`, Key: "dataCoord.enableCompaction", Version: "2.0.0", DefaultValue: "true", - Doc: "Enable data segment compaction", - Export: true, + Doc: `Switch value to control if to enable segment compaction. +Compaction merges small-size segments into a large segment, and clears the entities deleted beyond the rentention duration of Time Travel.`, + Export: true, } p.EnableCompaction.Init(base.mgr) @@ -3219,7 +3236,9 @@ exceeds this threshold, the largest growing segment will be sealed.`, Key: "dataCoord.compaction.enableAutoCompaction", Version: "2.0.0", DefaultValue: "true", - Export: true, + Doc: `Switch value to control if to enable automatic segment compaction during which data coord locates and merges compactable segments in the background. +This configuration takes effect only when dataCoord.enableCompaction is set as true.`, + Export: true, } p.EnableAutoCompaction.Init(base.mgr) @@ -3565,7 +3584,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.enableGarbageCollection", Version: "2.0.0", DefaultValue: "true", - Doc: "", + Doc: "Switch value to control if to enable garbage collection to clear the discarded data in MinIO or S3 service.", Export: true, } p.EnableGarbageCollection.Init(base.mgr) @@ -3574,7 +3593,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.gc.interval", Version: "2.0.0", DefaultValue: "3600", - Doc: "meta-based gc scanning interval in seconds", + Doc: "The interval at which data coord performs garbage collection, unit: second.", Export: true, } p.GCInterval.Init(base.mgr) @@ -3593,7 +3612,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.gc.missingTolerance", Version: "2.0.0", DefaultValue: "86400", - Doc: "orphan file gc tolerance duration in seconds (orphan file which last modified time before the tolerance interval ago will be deleted)", + Doc: "The retention duration of the unrecorded binary log (binlog) files. Setting a reasonably large value for this parameter avoids erroneously deleting the newly created binlog files that lack metadata. Unit: second.", Export: true, } p.GCMissingTolerance.Init(base.mgr) @@ -3602,7 +3621,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # Key: "dataCoord.gc.dropTolerance", Version: "2.0.0", DefaultValue: "10800", - Doc: "meta-based gc tolerace duration in seconds (file which meta is marked as dropped before the tolerace interval ago will be deleted)", + Doc: "The retention duration of the binlog files of the deleted segments before they are cleared, unit: second.", Export: true, } p.GCDropTolerance.Init(base.mgr) @@ -3984,8 +4003,10 @@ func (p *dataNodeConfig) init(base *BaseTable) { FallbackKeys: []string{"DATA_NODE_IBUFSIZE"}, DefaultValue: "16777216", PanicIfEmpty: true, - Doc: "Max buffer size to flush for a single segment.", - Export: true, + Doc: `The maximum size of each binlog file in a segment buffered in memory. Binlog files whose size exceeds this value are then flushed to MinIO or S3 service. +Unit: Byte +Setting this parameter too small causes the system to store a small amount of data too frequently. Setting it too large increases the system's demand for memory.`, + Export: true, } p.FlushInsertBufferSize.Init(base.mgr) diff --git a/pkg/util/paramtable/grpc_param.go b/pkg/util/paramtable/grpc_param.go index d75f6f61ffadb..f2afef49007ec 100644 --- a/pkg/util/paramtable/grpc_param.go +++ b/pkg/util/paramtable/grpc_param.go @@ -80,7 +80,7 @@ func (p *grpcConfig) init(domain string, base *BaseTable) { p.IPItem = ParamItem{ Key: p.Domain + ".ip", Version: "2.3.3", - Doc: "if not specified, use the first unicastable address", + Doc: "TCP/IP address of " + p.Domain + ". If not specified, use the first unicastable address", Export: true, } p.IPItem.Init(base.mgr) @@ -90,6 +90,7 @@ func (p *grpcConfig) init(domain string, base *BaseTable) { Key: p.Domain + ".port", Version: "2.0.0", DefaultValue: strconv.FormatInt(ProxyExternalPort, 10), + Doc: "TCP port of " + p.Domain, Export: true, } p.Port.Init(base.mgr) @@ -171,6 +172,7 @@ func (p *GrpcServerConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the " + domain + " can send, unit: byte", Export: true, } p.ServerMaxSendSize.Init(base.mgr) @@ -193,6 +195,7 @@ func (p *GrpcServerConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the " + domain + " can receive, unit: byte", Export: true, } p.ServerMaxRecvSize.Init(base.mgr) @@ -250,6 +253,7 @@ func (p *GrpcClientConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the clients on " + domain + " can send, unit: byte", Export: true, } p.ClientMaxSendSize.Init(base.mgr) @@ -272,6 +276,7 @@ func (p *GrpcClientConfig) Init(domain string, base *BaseTable) { } return v }, + Doc: "The maximum size of each RPC request that the clients on " + domain + " can receive, unit: byte", Export: true, } p.ClientMaxRecvSize.Init(base.mgr) diff --git a/pkg/util/paramtable/quota_param.go b/pkg/util/paramtable/quota_param.go index d7a276514af67..74e6b8d4a1d9b 100644 --- a/pkg/util/paramtable/quota_param.go +++ b/pkg/util/paramtable/quota_param.go @@ -204,6 +204,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.ddl.enabled", Version: "2.2.0", DefaultValue: "false", + Doc: "Whether DDL request throttling is enabled.", Export: true, } p.DDLLimitEnabled.Init(base.mgr) @@ -222,7 +223,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection", + Doc: `Maximum number of collection-related DDL requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 collection-related DDL requests per second, including collection creation requests, collection drop requests, collection load requests, and collection release requests. +To use this setting, set quotaAndLimits.ddl.enabled to true at the same time.`, Export: true, } p.DDLCollectionRate.Init(base.mgr) @@ -260,7 +263,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition", + Doc: `Maximum number of partition-related DDL requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including partition creation requests, partition drop requests, partition load requests, and partition release requests. +To use this setting, set quotaAndLimits.ddl.enabled to true at the same time.`, Export: true, } p.DDLPartitionRate.Init(base.mgr) @@ -288,6 +293,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.indexRate.enabled", Version: "2.2.0", DefaultValue: "false", + Doc: "Whether index-related request throttling is enabled.", Export: true, } p.IndexLimitEnabled.Init(base.mgr) @@ -306,7 +312,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for CreateIndex, DropIndex", + Doc: `Maximum number of index-related requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 partition-related requests per second, including index creation requests and index drop requests. +To use this setting, set quotaAndLimits.indexRate.enabled to true at the same time.`, Export: true, } p.MaxIndexRate.Init(base.mgr) @@ -334,6 +342,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.flushRate.enabled", Version: "2.2.0", DefaultValue: "true", + Doc: "Whether flush request throttling is enabled.", Export: true, } p.FlushLimitEnabled.Init(base.mgr) @@ -352,7 +361,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for flush", + Doc: `Maximum number of flush requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 flush requests per second. +To use this setting, set quotaAndLimits.flushRate.enabled to true at the same time.`, Export: true, } p.MaxFlushRate.Init(base.mgr) @@ -399,6 +410,7 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.compactionRate.enabled", Version: "2.2.0", DefaultValue: "false", + Doc: "Whether manual compaction request throttling is enabled.", Export: true, } p.CompactionLimitEnabled.Init(base.mgr) @@ -417,7 +429,9 @@ seconds, (0 ~ 65536)`, } return v }, - Doc: "qps, default no limit, rate for manualCompaction", + Doc: `Maximum number of manual-compaction requests per second. +Setting this item to 10 indicates that Milvus processes no more than 10 manual-compaction requests per second. +To use this setting, set quotaAndLimits.compaction.enabled to true at the same time.`, Export: true, } p.MaxCompactionRate.Init(base.mgr) @@ -446,9 +460,8 @@ seconds, (0 ~ 65536)`, Key: "quotaAndLimits.dml.enabled", Version: "2.2.0", DefaultValue: "false", - Doc: `dml limit rates, default no limit. -The maximum rate will not be greater than ` + "max" + `.`, - Export: true, + Doc: "Whether DML request throttling is enabled.", + Export: true, } p.DMLLimitEnabled.Init(base.mgr) @@ -470,7 +483,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data insertion rate per second. +Setting this item to 5 indicates that Milvus only allows data insertion at the rate of 5 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxInsertRate.Init(base.mgr) @@ -558,7 +573,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data insertion rate per collection per second. +Setting this item to 5 indicates that Milvus only allows data insertion to any collection at the rate of 5 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxInsertRatePerCollection.Init(base.mgr) @@ -822,7 +839,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data deletion rate per second. +Setting this item to 0.1 indicates that Milvus only allows data deletion at the rate of 0.1 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxDeleteRate.Init(base.mgr) @@ -910,7 +929,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return fmt.Sprintf("%f", rate) }, - Doc: "MB/s, default no limit", + Doc: `Highest data deletion rate per second. +Setting this item to 0.1 indicates that Milvus only allows data deletion from any collection at the rate of 0.1 MB/s. +To use this setting, set quotaAndLimits.dml.enabled to true at the same time.`, Export: true, } p.DMLMaxDeleteRatePerCollection.Init(base.mgr) @@ -1161,9 +1182,8 @@ The maximum rate will not be greater than ` + "max" + `.`, Key: "quotaAndLimits.dql.enabled", Version: "2.2.0", DefaultValue: "false", - Doc: `dql limit rates, default no limit. -The maximum rate will not be greater than ` + "max" + `.`, - Export: true, + Doc: "Whether DQL request throttling is enabled.", + Export: true, } p.DQLLimitEnabled.Init(base.mgr) @@ -1181,7 +1201,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "vps (vectors per second), default no limit", + Doc: `Maximum number of vectors to search per second. +Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second no matter whether these 100 vectors are all in one search or scattered across multiple searches. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxSearchRate.Init(base.mgr) @@ -1261,7 +1283,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "vps (vectors per second), default no limit", + Doc: `Maximum number of vectors to search per collection per second. +Setting this item to 100 indicates that Milvus only allows searching 100 vectors per second per collection no matter whether these 100 vectors are all in one search or scattered across multiple searches. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxSearchRatePerCollection.Init(base.mgr) @@ -1341,7 +1365,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "qps, default no limit", + Doc: `Maximum number of queries per second. +Setting this item to 100 indicates that Milvus only allows 100 queries per second. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxQueryRate.Init(base.mgr) @@ -1421,7 +1447,9 @@ The maximum rate will not be greater than ` + "max" + `.`, } return v }, - Doc: "qps, default no limit", + Doc: `Maximum number of queries per collection per second. +Setting this item to 100 indicates that Milvus only allows 100 queries per collection per second. +To use this setting, set quotaAndLimits.dql.enabled to true at the same time.`, Export: true, } p.DQLMaxQueryRatePerCollection.Init(base.mgr) @@ -1500,6 +1528,7 @@ The maximum rate will not be greater than ` + "max" + `.`, Key: "quotaAndLimits.limits.maxCollectionNumPerDB", Version: "2.2.0", DefaultValue: "65536", + Doc: "Maximum number of collections per database.", Export: true, } p.MaxCollectionNumPerDB.Init(base.mgr) diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index 2a41d30b69072..04d1da31c72cb 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -121,7 +121,10 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "localhost:2379", PanicIfEmpty: true, - Export: true, + Doc: `Endpoints used to access etcd service. You can change this parameter as the endpoints of your own etcd cluster. +Environment variable: ETCD_ENDPOINTS +etcd preferentially acquires valid address from environment variable ETCD_ENDPOINTS when Milvus is started.`, + Export: true, } p.Endpoints.Init(base.mgr) @@ -159,8 +162,12 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "by-dev", PanicIfEmpty: true, - Doc: "The root path where data is stored in etcd", - Export: true, + Doc: `Root prefix of the key to where Milvus stores data in etcd. +It is recommended to change this parameter before starting Milvus for the first time. +To share an etcd instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. +Set an easy-to-identify root path for Milvus if etcd service already exists. +Changing this for an already running Milvus instance may result in failures to read legacy data.`, + Export: true, } p.RootPath.Init(base.mgr) @@ -169,8 +176,10 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "meta", PanicIfEmpty: true, - Doc: "metaRootPath = rootPath + '/' + metaSubPath", - Export: true, + Doc: `Sub-prefix of the key to where Milvus stores metadata-related information in etcd. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended to change this parameter before starting Milvus for the first time.`, + Export: true, } p.MetaSubPath.Init(base.mgr) @@ -186,8 +195,10 @@ func (p *EtcdConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "kv", PanicIfEmpty: true, - Doc: "kvRootPath = rootPath + '/' + kvSubPath", - Export: true, + Doc: `Sub-prefix of the key to where Milvus stores timestamps in etcd. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended not to change this parameter if there is no specific reason.`, + Export: true, } p.KvSubPath.Init(base.mgr) @@ -437,8 +448,10 @@ func (p *LocalStorageConfig) Init(base *BaseTable) { Key: "localStorage.path", Version: "2.0.0", DefaultValue: "/var/lib/milvus/data", - Doc: "please adjust in embedded Milvus: /tmp/milvus/data/", - Export: true, + Doc: `Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended to change this parameter before starting Milvus for the first time.`, + Export: true, } p.Path.Init(base.mgr) } @@ -603,7 +616,7 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.port", Version: "2.0.0", DefaultValue: "6650", - Doc: "Port of Pulsar", + Doc: "Port of Pulsar service.", Export: true, } p.Port.Init(base.mgr) @@ -623,7 +636,11 @@ func (p *PulsarConfig) Init(base *BaseTable) { port, _ := p.Port.get() return "pulsar://" + addr + ":" + port }, - Doc: "Address of pulsar", + Doc: `IP address of Pulsar service. +Environment variable: PULSAR_ADDRESS +pulsar.address and pulsar.port together generate the valid access to Pulsar. +Pulsar preferentially acquires the valid IP address from the environment variable PULSAR_ADDRESS when Milvus is started. +Default value applies when Pulsar is running on the same network with Milvus.`, Export: true, } p.Address.Init(base.mgr) @@ -632,7 +649,7 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.webport", Version: "2.0.0", DefaultValue: "80", - Doc: "Web port of pulsar, if you connect directly without proxy, should use 8080", + Doc: "Web port of of Pulsar service. If you connect direcly without proxy, should use 8080.", Export: true, } p.WebPort.Init(base.mgr) @@ -656,8 +673,10 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.maxMessageSize", Version: "2.0.0", DefaultValue: strconv.Itoa(SuggestPulsarMaxMessageSize), - Doc: "5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.", - Export: true, + Doc: `The maximum size of each message in Pulsar. Unit: Byte. +By default, Pulsar can transmit at most 5 MB of data in a single message. When the size of inserted data is greater than this value, proxy fragments the data into multiple messages to ensure that they can be transmitted correctly. +If the corresponding parameter in Pulsar remains unchanged, increasing this configuration will cause Milvus to fail, and reducing it produces no advantage.`, + Export: true, } p.MaxMessageSize.Init(base.mgr) @@ -665,7 +684,9 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.tenant", Version: "2.2.0", DefaultValue: "public", - Export: true, + Doc: `Pulsar can be provisioned for specific tenants with appropriate capacity allocated to the tenant. +To share a Pulsar instance among multiple Milvus instances, you can change this to an Pulsar tenant rather than the default one for each Milvus instance before you start them. However, if you do not want Pulsar multi-tenancy, you are advised to change msgChannel.chanNamePrefix.cluster to the different value.`, + Export: true, } p.Tenant.Init(base.mgr) @@ -673,6 +694,7 @@ func (p *PulsarConfig) Init(base *BaseTable) { Key: "pulsar.namespace", Version: "2.2.0", DefaultValue: "default", + Doc: "A Pulsar namespace is the administrative unit nomenclature within a tenant.", Export: true, } p.Namespace.Init(base.mgr) @@ -867,8 +889,10 @@ func (r *RocksmqConfig) Init(base *BaseTable) { r.Path = ParamItem{ Key: "rocksmq.path", Version: "2.0.0", - Doc: `The path where the message is stored in rocksmq -please adjust in embedded Milvus: /tmp/milvus/rdb_data`, + Doc: `Prefix of the key to where Milvus stores data in RocksMQ. +Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. +It is recommended to change this parameter before starting Milvus for the first time. +Set an easy-to-identify root key prefix for Milvus if etcd service already exists.`, Export: true, } r.Path.Init(base.mgr) @@ -886,7 +910,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.rocksmqPageSize", DefaultValue: strconv.FormatInt(64<<20, 10), Version: "2.0.0", - Doc: "64 MB, 64 * 1024 * 1024 bytes, The size of each page of messages in rocksmq", + Doc: "The maximum size of messages in each page in RocksMQ. Messages in RocksMQ are checked and cleared (when expired) in batch based on this parameters. Unit: Byte.", Export: true, } r.PageSize.Init(base.mgr) @@ -895,7 +919,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.retentionTimeInMinutes", DefaultValue: "4320", Version: "2.0.0", - Doc: "3 days, 3 * 24 * 60 minutes, The retention time of the message in rocksmq.", + Doc: "The maximum retention time of acked messages in RocksMQ. Acked messages in RocksMQ are retained for the specified period of time and then cleared. Unit: Minute.", Export: true, } r.RetentionTimeInMinutes.Init(base.mgr) @@ -904,7 +928,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.retentionSizeInMB", DefaultValue: "7200", Version: "2.0.0", - Doc: "8 GB, 8 * 1024 MB, The retention size of the message in rocksmq.", + Doc: "The maximum retention size of acked messages of each topic in RocksMQ. Acked messages in each topic are cleared if their size exceed this parameter. Unit: MB.", Export: true, } r.RetentionSizeInMB.Init(base.mgr) @@ -913,7 +937,7 @@ please adjust in embedded Milvus: /tmp/milvus/rdb_data`, Key: "rocksmq.compactionInterval", DefaultValue: "86400", Version: "2.0.0", - Doc: "1 day, trigger rocksdb compaction every day to remove deleted data", + Doc: "Time interval to trigger rocksdb compaction to remove deleted data. Unit: Second", Export: true, } r.CompactionInterval.Init(base.mgr) @@ -959,7 +983,7 @@ func (r *NatsmqConfig) Init(base *BaseTable) { Key: "natsmq.server.port", Version: "2.3.0", DefaultValue: "4222", - Doc: `Port for nats server listening`, + Doc: "Listening port of the NATS server.", Export: true, } r.ServerPort.Init(base.mgr) @@ -1096,7 +1120,7 @@ func (p *MinioConfig) Init(base *BaseTable) { Key: "minio.port", DefaultValue: "9000", Version: "2.0.0", - Doc: "Port of MinIO/S3", + Doc: "Port of MinIO or S3 service.", PanicIfEmpty: true, Export: true, } @@ -1116,7 +1140,11 @@ func (p *MinioConfig) Init(base *BaseTable) { port, _ := p.Port.get() return addr + ":" + port }, - Doc: "Address of MinIO/S3", + Doc: `IP address of MinIO or S3 service. +Environment variable: MINIO_ADDRESS +minio.address and minio.port together generate the valid access to MinIO or S3 service. +MinIO preferentially acquires the valid IP address from the environment variable MINIO_ADDRESS when Milvus is started. +Default value applies when MinIO or S3 is running on the same network with Milvus.`, Export: true, } p.Address.Init(base.mgr) @@ -1126,8 +1154,12 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "minioadmin", PanicIfEmpty: false, // tmp fix, need to be conditional - Doc: "accessKeyID of MinIO/S3", - Export: true, + Doc: `Access key ID that MinIO or S3 issues to user for authorized access. +Environment variable: MINIO_ACCESS_KEY_ID or minio.accessKeyID +minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. +This configuration must be set identical to the environment variable MINIO_ACCESS_KEY_ID, which is necessary for starting MinIO or S3. +The default value applies to MinIO or S3 service that started with the default docker-compose.yml file.`, + Export: true, } p.AccessKeyID.Init(base.mgr) @@ -1136,8 +1168,12 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "minioadmin", PanicIfEmpty: false, // tmp fix, need to be conditional - Doc: "MinIO/S3 encryption string", - Export: true, + Doc: `Secret key used to encrypt the signature string and verify the signature string on server. It must be kept strictly confidential and accessible only to the MinIO or S3 server and users. +Environment variable: MINIO_SECRET_ACCESS_KEY or minio.secretAccessKey +minio.accessKeyID and minio.secretAccessKey together are used for identity authentication to access the MinIO or S3 service. +This configuration must be set identical to the environment variable MINIO_SECRET_ACCESS_KEY, which is necessary for starting MinIO or S3. +The default value applies to MinIO or S3 service that started with the default docker-compose.yml file.`, + Export: true, } p.SecretAccessKey.Init(base.mgr) @@ -1146,7 +1182,7 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "false", PanicIfEmpty: true, - Doc: "Access to MinIO/S3 with SSL", + Doc: "Switch value to control if to access the MinIO or S3 service through SSL.", Export: true, } p.UseSSL.Init(base.mgr) @@ -1164,8 +1200,13 @@ func (p *MinioConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: "a-bucket", PanicIfEmpty: true, - Doc: "Bucket name in MinIO/S3", - Export: true, + Doc: `Name of the bucket where Milvus stores data in MinIO or S3. +Milvus 2.0.0 does not support storing data in multiple buckets. +Bucket with this name will be created if it does not exist. If the bucket already exists and is accessible, it will be used directly. Otherwise, there will be an error. +To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. +The data will be stored in the local Docker if Docker is used to start the MinIO service locally. Ensure that there is sufficient storage space. +A bucket name is globally unique in one MinIO or S3 instance.`, + Export: true, } p.BucketName.Init(base.mgr) @@ -1180,8 +1221,12 @@ func (p *MinioConfig) Init(base *BaseTable) { return path.Clean(rootPath) }, PanicIfEmpty: false, - Doc: "The root path where the message is stored in MinIO/S3", - Export: true, + Doc: `Root prefix of the key to where Milvus stores data in MinIO or S3. +It is recommended to change this parameter before starting Milvus for the first time. +To share an MinIO instance among multiple Milvus instances, consider changing this to a different value for each Milvus instance before you start them. For details, see Operation FAQs. +Set an easy-to-identify root key prefix for Milvus if etcd service already exists. +Changing this for an already running Milvus instance may result in failures to read legacy data.`, + Export: true, } p.RootPath.Init(base.mgr)