From ccc4583aa96bf5feb6c158097f68ece10468d351 Mon Sep 17 00:00:00 2001 From: Nisarg Thakkar Date: Mon, 22 Jul 2024 12:53:08 -0700 Subject: [PATCH] [controller][admin-tool][vpj][test] Add a multi-region config to controllers and infer many configs 1. Use multi.region config to control certain features instead of controlling them via explicit configs. * Native-replication - Enabled in multi-region mode. Disabled in single-region mode. * Admin channel consumption - Enabled in multi-region mode. Disabled in single-region mode. * This is still allowed to be disabled via LiveConfig for store migration purposes. * Controller allowing BATCH push via API - Disabled in multi-region mode. Enabled in single-region mode. * Active-active replication enabled on a controller - Enabled in multi-region mode. Disabled in single-region mode. 2. Commands and code to enable and disable Native-replication for a cluster has been removed as that is the only mode. 3. The following configs are now obsolete, and removed: * enable.native.replication.for.batch.only * enable.native.replication.for.batch.only * enable.native.replication.for.hybrid * enable.native.replication.as.default.for.batch.only * enable.native.replication.as.default.for.hybrid * enable.active.active.replication.as.default.for.batch.only.store * enable.active.active.replication.as.default.for.batch.only.store * admin.topic.remote.consumption.enabled * child.controller.admin.topic.consumption.enabled * Only removed from controller config * It is still used in LiveConfigs during store migration * active.active.enabled.on.controller * controller.enable.batch.push.from.admin.in.child 4. The following configs were unused and now removed: * kafka.min.log.compaction.lag.ms * topic.cleanup.send.concurrent.delete.requests.enabled * topic.deletion.status.poll.interval.ms * topic.creation.throttling.time.window.ms * topic.manager.kafka.operation.timeout.ms * admin.consumption.timeout.minute * amplification.factor * server.ingestion.isolation.metric.request.timeout.seconds --- clients/venice-admin-tool/README | 756 +++++++++++++----- .../java/com/linkedin/venice/AdminTool.java | 38 +- .../java/com/linkedin/venice/Command.java | 10 - .../linkedin/venice/hadoop/VenicePushJob.java | 2 +- .../venice/hadoop/VenicePushJobConstants.java | 5 - .../TestKafkaFormatTopicAutoDiscover.java | 2 +- .../hadoop/TestVenicePushJobCheckpoints.java | 2 +- .../venice/hadoop/VenicePushJobTest.java | 2 +- .../single-dc-configs/controller.properties | 2 +- .../com/linkedin/venice/VeniceConstants.java | 4 - .../java/com/linkedin/venice/ConfigKeys.java | 103 +-- .../controllerapi/ControllerClient.java | 14 - .../venice/controllerapi/ControllerRoute.java | 3 - .../java/com/linkedin/venice/utils/Utils.java | 13 +- .../com/linkedin/venice/utils/UtilsTest.java | 12 + .../AbstractTestVeniceHelixAdmin.java | 23 +- ...LevelConfigForActiveActiveReplication.java | 235 ++---- ...lusterLevelConfigForNativeReplication.java | 3 - .../controller/TestControllerEnforceSSL.java | 1 - .../controller/TestD2ControllerClient.java | 1 - .../controller/TestDelayedRebalance.java | 2 - ...niceHelixAdminWithIsolatedEnvironment.java | 3 +- .../venice/controller/TestFabricBuildout.java | 12 +- ...niceHelixAdminWithIsolatedEnvironment.java | 2 +- ...VeniceHelixAdminWithSharedEnvironment.java | 1 - .../endToEnd/DaVinciClientDiskFullTest.java | 1 - .../TestPushJobWithNativeReplication.java | 168 ---- ...tPushJobWithSourceGridFabricSelection.java | 13 +- ...iceClientBasedMetadataIntegrationTest.java | 1 - .../integration/utils/ServiceFactory.java | 3 - .../utils/VeniceClusterCreateOptions.java | 35 +- .../utils/VeniceClusterWrapper.java | 3 +- .../utils/VeniceControllerCreateOptions.java | 43 +- .../utils/VeniceControllerWrapper.java | 9 +- .../VeniceMultiClusterCreateOptions.java | 40 +- .../utils/VeniceMultiClusterWrapper.java | 11 +- ...woLayerMultiRegionMultiClusterWrapper.java | 41 +- .../kafka/EndToEndKafkaWithSASLTest.java | 1 - .../TestRestartServerDuringIngestion.java | 2 +- .../utils/IntegrationTestPushUtils.java | 2 +- .../grpc/VeniceGrpcEndToEndTest.java | 1 - .../linkedin/venice/utils/TestWriteUtils.java | 2 +- .../com/linkedin/venice/controller/Admin.java | 12 - .../HelixVeniceClusterResources.java | 20 +- .../VeniceControllerClusterConfig.java | 331 ++------ .../VeniceControllerMultiClusterConfig.java | 32 +- .../controller/VeniceControllerService.java | 34 +- .../venice/controller/VeniceHelixAdmin.java | 184 +---- .../controller/VeniceParentHelixAdmin.java | 31 +- .../controller/kafka/TopicCleanupService.java | 18 +- .../kafka/consumer/AdminExecutionTask.java | 20 +- .../controller/server/AdminSparkServer.java | 4 - .../controller/server/StoresRoutes.java | 36 - .../PartitionStatusBasedPushMonitor.java | 4 +- .../TestVeniceControllerClusterConfig.java | 111 ++- .../kafka/TestTopicCleanupService.java | 4 +- ...icCleanupServiceForMultiKafkaClusters.java | 3 +- ...opicCleanupServiceForParentController.java | 3 +- .../consumer/TestAdminConsumerService.java | 2 - 59 files changed, 983 insertions(+), 1493 deletions(-) diff --git a/clients/venice-admin-tool/README b/clients/venice-admin-tool/README index f0b2c7dcec..eb8bebc9dd 100644 --- a/clients/venice-admin-tool/README +++ b/clients/venice-admin-tool/README @@ -4,190 +4,588 @@ Venice Admin Tool For directly affecting administrative actions and queries on a Venice cluster ``` -usage: java -jar venice-admin-tool-0.1.jar -- [parameters] +usage: java -jar venice-admin-tool-all.jar -- [parameters] Commands: - --add-schema Requires: --url, --cluster, --store, - --value-schema-file - --cluster-health-instances List the status for every instance. - Requires: --url, --cluster - --cluster-health-stores List the status for every store. - Requires: --url, --cluster - --delete-all-versions Delete all versions in given store.. - Requires: --url, --cluster, --store - --delete-store Delete the given store including both - metadata and all versions in this store.. - Requires: --url, --cluster, --store - --describe-store Requires: --url, --cluster, --store - --describe-stores Requires: --url, --cluster - --disable-store Disable store in both read and write - path.. Requires: --url, --cluster, - --store - --disable-store-read Prevent a store from serving read - requests. Requires: --url, --cluster, - --store - --disable-store-write Prevent a store from accepting new - versions. Requires: --url, --cluster, - --store - --empty-push Do an empty push into an existing store. - Requires: --url, --cluster, --store, - --push-id, --store-size - --enable-store Enable a store in both read and write - path. Requires: --url, --cluster, - --store - --enable-store-read Allow a store to serve read requests - again after reads have been disabled. - Requires: --url, --cluster, --store - --enable-store-write Allow a store to accept new versions - again after being writes have been - disabled. Requires: --url, --cluster, - --store - --get-execution Get the execution status for an async - admin command.. Requires: --url, - --cluster, --execution - --job-status Query the ingest status of a running push - job. Requires: --url, --cluster, - --store, --version - --kill-job Kill a running push job. Requires: - --url, --cluster, --store, --version - --list-storage-nodes Requires: --url, --cluster - --list-stores Requires: --url, --cluster - --new-store Requires: --url, --cluster, --store, - --key-schema-file, --value-schema-file, - --owner - --node-removable A node is removable if all replicas it is - serving are available on other nodes. - Requires: --url, --cluster, - --storage-node - --query Query a store that has a simple key - schema. Requires: --url, --cluster, - --store, --key - --remove-node Remove a storage node from the cluster. - Requires: --url, --cluster, - --storage-node - --replicas-of-store List the location and status of all - replicas for a store. Requires: --url, - --cluster, --store, --version - --replicas-on-storage-node List the store and status of all replicas - on a storage node. Requires: --url, - --cluster, --storage-node - --schemas Show the key and value schemas for a - store. Requires: --url, --cluster, - --store - --set-owner Update owner info of an existing store. - Requires: --url, --cluster, --store, - --owner - --set-partition-count Update the number of partitions of an - existing store. Requires: --url, - --cluster, --store, --partition-count - --set-version Set the version that will be served. - Requires: --url, --cluster, --store, - --version - --skip-admin Skip an admin message. Requires: --url, - --cluster, --offset - --update-store update store metadata. Requires: --url, - --cluster, --store, --owner, - --partition-count, --version, - --readability, --writeability, - --storage-quota, --read-quota - --white-list-add-node Add a storage node into the white list.. - Requires: --url, --cluster, - --storage-node - --white-list-remove-node Remove a storage node from the white - list.. Requires: --url, --cluster, - --storage-node - --node-replicas-readiness Get the status of all current version - replicas that a storage node assigned to. - Requires: --url, --cluster, --storage-node + --abort-migration Kill store migration task and revert to previous state. + Requires: --url --store --cluster-src --cluster-dest + Optional args: --force + --add-derived-schema + Requires: --url --store --value-schema-id --derived-schema-file + Optional args: --cluster + --add-schema + Requires: --url --store --value-schema-file + Optional args: --cluster + --add-schema-to-zk + Requires: --venice-zookeeper-url --store --value-schema-file --value-schema-id + --zk-ssl-config-file + Optional args: --cluster + --add-to-store-acl Add a principal to ACL's for an existing store. + Requires: --url --store --principal + Optional args: --cluster --readability --writeability + --allow-list-add-node Add a storage node into the allowlist. + Requires: --url --cluster --storage-node + --allow-list-remove-node Remove a storage node from the allowlist. + Requires: --url --cluster --storage-node + --backfill-system-stores Create system stores of a given type for user stores in a cluster. + Requires: --url --cluster --system-store-type + --backup-store-metadata-from-graveyard Backup store metadata from graveyard in EI. + Requires: --venice-zookeeper-url --zk-ssl-config-file --backup-folder + --check-fabric-buildout-status Check the status of cluster building in destination fabric. + Requires: --url --cluster --source-fabric --dest-fabric + --cleanup-instance-customized-states Cleanup any lingering instance level customized states. + Requires: --url --cluster + --cluster-health-instances List the status for every instance. + Requires: --url --cluster + Optional args: --enable-disabled-replicas + --cluster-health-stores List the status for every store. + Requires: --url --cluster + --compare-store Compare a store between two fabrics. + Requires: --url --store --fabric-a --fabric-b + Optional args: --cluster + --complete-migration Update cluster discovery in a fabric. + Requires: --url --store --cluster-src --cluster-dest --fabric + --configure-store-view Configure store view of a certain store. + Requires: --url --store --view-name + Optional args: --cluster --view-class --view-params --remove-view + --convert-vson-schema Convert and print out Avro schemas based on input Vson schemas. + Requires: --key-schema-file --value-schema-file + --delete-all-versions Delete all versions in given store. + Requires: --url --store + Optional args: --cluster + --delete-kafka-topic Delete a Kafka topic directly (without interaction with the Venice Controller. + Requires: --kafka-bootstrap-servers --kafka-topic-name + Optional args: --kafka-operation-timeout --kafka-consumer-config-file + --delete-old-version Delete the given version(non current version) in the given store. + Requires: --url --store --version + Optional args: --cluster + --delete-storage-persona Deletes an existing storage persona. + Requires: --url --cluster --storage-persona + --delete-store Delete the given store including both metadata and all versions in this store. + Requires: --url --store + Optional args: --cluster + --delete-store-acl Delete ACL's for an existing store. + Requires: --url --store + Optional args: --cluster + --describe-store Get store details. + Requires: --url --store + Optional args: --cluster + --describe-stores + Requires: --url --cluster + Optional args: --include-system-stores + --disable-active-active-replication-for-cluster disable active active replication for certain stores (batch-only, hybrid-only, + incremental-push, hybrid-or-incremental, all) in a cluster. + Requires: --url --cluster --store-type + Optional args: --regions-filter + --disable-max-capacity-protection Disable the feature that prevent read request usage exceeding the max capacity on all + routers.. + Requires: --url --cluster + --disable-quota-rebalance Disable the feature that quota could be rebalanced once live router count is changed + on all routers. + Requires: --url --cluster --expected-router-count + --disable-store Disable store in both read and write path. + Requires: --url --store + Optional args: --cluster + --disable-store-read Prevent a store from serving read requests. + Requires: --url --store + Optional args: --cluster + --disable-store-write Prevent a store from accepting new versions. + Requires: --url --store + Optional args: --cluster + --disable-throttling Disable the feature that throttling read request on all routers. + Requires: --url --cluster + --dump-admin-messages Dump admin messages. + Requires: --cluster --kafka-bootstrap-servers --starting_offset --message_count + --kafka-consumer-config-file + --dump-control-messages Dump control messages in a partition. + Requires: --kafka-bootstrap-servers --kafka-consumer-config-file --kafka-topic-name + --kafka-topic-partition --starting_offset --message_count + --dump-ingestion-state Dump the real-time ingestion state for a certain store version in a certain storage + node. + Requires: --server-url --store --version + Optional args: --partition + --dump-kafka-topic Dump a Kafka topic for a Venice cluster. If start offset and message count are not + specified, the entire partition will be dumped. PLEASE REFRAIN FROM USING SERVER + CERTIFICATES, IT IS A GDPR VIOLATION, GET ADDED TO THE STORE ACL'S OR GET FAST ACCESS + TO THE KAFKA TOPIC!!. + Requires: --kafka-bootstrap-servers --kafka-consumer-config-file --kafka-topic-name + --cluster --url + --dump-topic-partition-ingestion-context Dump the topic partition ingestion context belong to a certain store version in a + certain storage node. + Requires: --server-url --store --version --kafka-topic-name --kafka-topic-partition + --empty-push Do an empty push into an existing store. + Requires: --url --store --push-id --store-size + Optional args: --cluster + --enable-active-active-replication-for-cluster enable active active replication for certain stores (batch-only, hybrid-only, + incremental-push, hybrid-or-incremental, all) in a cluster. + Requires: --url --cluster --store-type + Optional args: --regions-filter + --enable-max-capacity-protection Enable the feature that prevent read request usage exceeding the max capacity on all + routers. + Requires: --url --cluster + --enable-quota-rebalance Enable the feature that quota could be rebalanced once live router count is changed on + all routers. + Requires: --url --cluster + --enable-store Enable a store in both read and write path. + Requires: --url --store + Optional args: --cluster + --enable-store-read Allow a store to serve read requests again after reads have been disabled. + Requires: --url --store + Optional args: --cluster + --enable-store-write Allow a store to accept new versions again after being writes have been disabled. + Requires: --url --store + Optional args: --cluster + --enable-throttling Enable the feature that throttling read request on all routers. + Requires: --url --cluster + --end-fabric-buildout End the building of a cluster in destination fabric. + Requires: --url --cluster --source-fabric --dest-fabric + --end-migration Send this command to delete the original store. + Requires: --url --store --cluster-src --cluster-dest + --estimate-data-recovery-time Estimates the time it would take to execute data recovery for a group of stores. + ('--stores' overwrites '--cluster' value). + Requires: --url --dest-fabric + Optional args: --stores --cluster + --execute-data-recovery Execute data recovery for a group of stores. ('--stores' overwrites '--cluster' + value). + Requires: --url --recovery-command --source-fabric --dest-fabric --datetime + Optional args: --stores --cluster --extra-command-args --debug --non-interactive + --extract-venice-zk-paths Extract Venice-specific paths from a ZK snapshot input text file to an output text + file. + Requires: --infile --outfile --cluster-list --base-path + --get-all-migration-push-strategies Get migration push strategies for all the voldemort stores. + Requires: --url --cluster + --get-deletable-store-topics Get a list of deletable store topics in the fabric that belongs to the controller + handling the request. + Requires: --url --cluster + --get-execution Get the execution status for an async admin command. + Requires: --url --cluster --execution + --get-kafka-topic-configs Get configs of a topic through controllers. + Requires: --url --kafka-topic-name + --get-migration-push-strategy Get migration push strategy for the specified voldemort store. + Requires: --url --cluster --voldemort-store + --get-routers-cluster-config Get cluster level router's config. + Requires: --url --cluster + --get-storage-persona Gets info on an existing storage persona by name. + Requires: --url --cluster --storage-persona + --get-storage-persona-for-store Gets the storage persona associated with a store name.. + Requires: --url --store + Optional args: --cluster + --get-store-acl Get ACL's for an existing store. + Requires: --url --store + Optional args: --cluster + --job-status Query the ingest status of a running push job. If a version is not specified, the job + status of the last job will be printed.. + Requires: --url --store + Optional args: --cluster --version + --kill-job Kill a running push job. + Requires: --url --store --version + Optional args: --cluster + --list-bootstrapping-versions List all versions which have at least one bootstrapping replica. + Requires: --url --cluster + --list-cluster-stale-stores List all stores in a cluster which have stale replicas.. + Requires: --url --cluster + --list-cluster-storage-personas Lists all storage personas in a cluster.. + Requires: --url --cluster + --list-storage-nodes + Requires: --url --cluster + --list-store-push-info List information about current pushes and push history for a specific store.. + Requires: --url --store + Optional args: --cluster --partition-detail-enabled + --list-stores List all stores present in the given cluster. + Requires: --url --cluster + Optional args: --include-system-stores + --migrate-store Migrate store from one cluster to another within the same fabric. + Requires: --url --store --cluster-src --cluster-dest + --migration-status Get store migration status. + Requires: --url --store --cluster-src --cluster-dest + --monitor-data-recovery Monitor data recovery progress for a group of stores. ('--stores' overwrites + '--cluster' value). + Requires: --url --dest-fabric --datetime + Optional args: --stores --cluster --interval + --new-storage-persona Creates a new storage persona.. + Requires: --url --cluster --storage-persona --storage-quota --store --owner + --new-store + Requires: --url --cluster --store --key-schema-file --value-schema-file + Optional args: --owner --vson_store + --new-store-acl Create a new store with ACL permissions set. + Requires: --url --store --key-schema-file --value-schema-file --acl-perms + Optional args: --cluster --owner --vson_store + --node-removable A node is removable if all replicas it is serving are available on other nodes. + Requires: --url --cluster --storage-node + --node-replicas-readiness Get the readiness of all current replicas on a storage node from a child controller. + Requires: --url --cluster --storage-node + --query Query a store that has a simple key schema. + Requires: --url --store --key + Optional args: --cluster --vson_store --venice-client-ssl-config-file + --query-kafka-topic Query some specific keys from the Venice Topic. + Requires: --kafka-bootstrap-servers --kafka-consumer-config-file --kafka-topic-name + --cluster --url --start-date --key + Optional args: --end-date --progress-interval + --recover-store-metadata Recover store metadata in EI. + Requires: --url --store --venice-zookeeper-url --kafka-bootstrap-servers + --graveyard-clusters + Optional args: --zk-ssl-config-file --kafka-consumer-config-file --recover-cluster + --skip-last-store-creation --repair + --remove-derived-schema remove derived schema for a given store by the value and derived schema Ids. + Requires: --url --store --value-schema-id --derived-schema-id + Optional args: --cluster + --remove-from-store-acl Remove a principal from ACL's for an existing store. + Requires: --url --store --principal + Optional args: --cluster --readability --writeability + --remove-node Remove a storage node from the cluster. + Requires: --url --cluster --storage-node + --replicas-of-store List the location and status of all replicas for a store. + Requires: --url --store --version + Optional args: --cluster + --replicas-on-storage-node List the store and status of all replicas on a storage node. + Requires: --url --cluster --storage-node + --replicate-meta-data Copy a cluster's all stores schemas and store level configs from source fabric to + destination fabric. + Requires: --url --cluster --source-fabric --dest-fabric + --request-based-metadata Get the store's metadata using request based metadata endpoint via a transport client + and a server URL. + Requires: --url --server-url --store + --schemas Show the key and value schemas for a store. + Requires: --url --store + Optional args: --cluster + --send-end-of-push Send this message after Samza reprocessing job to close offline batch push. + Requires: --url --store --version + Optional args: --cluster + --set-migration-push-strategy Setup migration push strategy for the specified voldemort store. + Requires: --url --cluster --voldemort-store --migration-push-strategy + --set-owner Update owner info of an existing store. + Requires: --url --store --owner + Optional args: --cluster + --set-partition-count Update the number of partitions of an existing store. + Requires: --url --store --partition-count + Optional args: --cluster + --set-version Set the version that will be served. + Requires: --url --store --version + Optional args: --cluster + --skip-admin Skip an admin message. + Requires: --url --cluster --offset + Optional args: --skip-div + --start-fabric-buildout Start building a cluster in destination fabric by copying stores metadata and data + from source fabric. + Requires: --url --cluster --source-fabric --dest-fabric + Optional args: --retry + --update-cluster-config Update live cluster configs. + Requires: --url --cluster + Optional args: --fabric --server.kafka.fetch.quota.records.per.second + --allow.store.migration --child.controller.admin.topic.consumption.enabled + --update-kafka-topic-log-compaction Update log compaction config of a topic through controllers. + Requires: --url --kafka-topic-name --kafka-topic-log-compaction-enabled + Optional args: --cluster + --update-kafka-topic-min-in-sync-replica Update minISR of a topic through controllers. + Requires: --url --kafka-topic-name --kafka-topic-min-in-sync-replica + Optional args: --cluster + --update-kafka-topic-retention Update retention config of a topic through controllers. + Requires: --url --kafka-topic-name --kafka-topic-retention-in-ms + Optional args: --cluster + --update-storage-persona Updates an existing storage persona. + Requires: --url --cluster --storage-persona + Optional args: --storage-quota --store --owner + --update-store update store metadata. + Requires: --url --store + Optional args: --cluster --owner --version --largest-used-version --partition-count + --partitioner-class --partitioner-params --readability --writeability --storage-quota + --storage-node-read-quota-enabled --hybrid-store-overhead-bypass --read-quota + --hybrid-rewind-seconds --hybrid-offset-lag --hybrid-time-lag + --hybrid-data-replication-policy --hybrid-buffer-replay-policy --access-control + --compression-strategy --client-decompression-enabled --chunking-enabled + --rmd-chunking-enabled --batch-get-limit --num-versions-to-preserve + --write-computation-enabled --read-computation-enabled --backup-strategy + --auto-schema-register-push-job-enabled --incremental-push-enabled + --bootstrap-to-online-timeout --hybrid-store-disk-quota-enabled + --regular-version-etl-enabled --future-version-etl-enabled --etled-proxy-user-account + --native-replication-enabled --push-stream-source-address + --backup-version-retention-day --replication-factor --native-replication-source-fabric + --replicate-all-configs --active-active-replication-enabled --regions-filter + --disable-meta-store --disable-davinci-push-status-store --storage-persona + --storage-view-configs --latest-superset-schema-id --min-compaction-lag-seconds + --max-compaction-lag-seconds --max-record-size-bytes --enable-unused-schema-deletion + --blob-transfer-enabled + --update-store-acl Update ACL's for an existing store. + Requires: --url --store --acl-perms + Optional args: --cluster + --wipe-cluster Delete data and metadata of a cluster/store/version in a child fabric. + Requires: --url --cluster --fabric + Optional args: --store --version usage: Parameters: - -c,--cluster Name of Venice cluster - -e,--execution Execution ID of async admin command - -f,--filter-json Comma-delimited list of fields to - display from the json output. Omit - to display all fields - -fj,--flat-json Display output as flat json, without - pretty-print indentation and line - breaks - -h,--help Show usage - -ho,--hybrid-offset-lag for hybrid stores, what is the offset - lag threshold for the storage nodes' - consumption to be considered ONLINE - -hr,--hybrid-rewind-seconds for hybrid stores, how far back to - rewind in the nearline stream after a - batch push completes - -k,--key Plain-text key for identifying a - record in a store - -ks,--key-schema-file Path to text file with key schema - -n,--storage-node Helix instance ID for a storage node, - eg. lva1-app1234_1690 - -o,--owner Owner email for new store creation - -of,--offset Kafka offset number - -pid,--push-id Push Id - -pn,--partition-count number of partitions a store has - -rb,--readability store's readability - -rq,--read-quota quota for read request hit this - store. Measurement is capacity unit - -s,--store Name of Venice store - -sq,--storage-quota maximum capacity a store version - could have - -ss,--store-size Size of the store in bytes, used to - calculate partitioning - -u,--url Venice url, eg. http://localhost:1689 - This can be a router or a controller - -v,--version Venice store version number - -vs,--value-schema-file Path to text file with value schema - -wb,--writeability store's writeability + -aa,--active-active-replication-enabled A parameter flag to enable/disable Active/Active replication feature for a + store + -acl,--access-control Enable/disable store-level access control + -ap,--acl-perms Acl permissions for the store + -asm,--allow.store.migration whether stores are allowed to be migrated from/to a cluster + -asp,--auto-schema-register-push-job-enabled whether or not to use auto-schema register for pushjob + -atc,--child.controller.admin.topic.consumption.enabled whether child controller consumes admin topic + -bf,--backup-folder Backup folder path + -bgl,--batch-get-limit Key number limit inside one batch-get request + -bp,--base-path Base path for ZK, eg. /venice-parent + -bt,--blob-transfer-enabled Flag to indicate if the blob transfer is allowed or not + -btot,--bootstrap-to-online-timeout Set the maximum number of hours allowed for the store to transition from + bootstrap to online + -bus,--backup-strategy Strategies to preserve backup versions, eg KEEP_MIN_VERSIONS, + DELETE_ON_NEW_PUSH_START. Default is KEEP_MIN_VERSIONS + -bvrd,--backup-version-retention-day Backup version retention time in day after a new version is promoted to the + current version, if not specified, Venice will use the configured retention + as the default policy + -c,--cluster Name of Venice cluster + -cd,--cluster-dest Store migration destination Venice cluster name + -ce,--chunking-enabled Enable/Disable value chunking, mostly for large value store support + -cl,--cluster-list Comma separated list of cluster names, eg. venice-0, venice-1, ... + -cs,--compression-strategy strategies used to compress/decompress Record's value + -csd,--client-decompression-enabled Enable/Disable client-side record decompression (default: true) + -d,--debug Print debugging messages for execute-data-recovery + -ddvc,--disable-davinci-push-status-store disable davinci push status store. This command sets + daVinciPushStatusStoreEnabled flag to false but does not delete any + resources associated with the push status store. Please use this option + with caution + -df,--dest-fabric The fabric where metadata/data gets copy over into + -did,--derived-schema-id derived schema id + -div,--skip-div Whether or not to only skip DIV for skip admin + -dl,--disable-log Disable logs from internal classes. Only print command output on console + -dms,--disable-meta-store disable meta system store. This command sets storeMetaSystemStoreEnabled + flag to false but does not delete any resources associated with the meta + store. Please use this option with caution + -ds,--derived-schema-file Path to text file with derived schema + -dtm,--datetime Date and time stamp (YYYY-MM-DDTHH:MM:SS) in UTC time zone for data + recovery + -e,--execution Execution ID of async admin command + -eca,--extra-command-args extra command arguments + -ed,--end-date End date in PST. Example: 2020-10-10 10:10:10 + -edr,--enable-disabled-replicas Reenable disabled replicas + -epu,--etled-proxy-user-account if enabled ETL, the proxy user account for HDFS file directory where the + ETLed snapshots will go. + -erc,--expected-router-count How many routers that a cluster should have. + -f,--force Force execute this operation + -fa,--fabric-a The name of the first fabric in store comparison. + -fb,--fabric-b The name of the second fabric in store comparison. + -fc,--fabric Which fabric to execute the admin command. + -flj,--flat-json Display output as flat json, without pretty-print indentation and line + breaks + -fve,--future-version-etl-enabled whether or not to enable future version etl for this store. + -gc,--graveyard-clusters Clusters to scan store graveyard to retrieve metadata, eg. + cluster-1,cluster-2 + -h,--help Show usage + -hbrp,--hybrid-buffer-replay-policy for hybrid stores, how buffer replay start timestamps are calculated. + -hdrp,--hybrid-data-replication-policy for hybrid stores, how real-time Samza data is replicated + -ho,--hybrid-offset-lag for hybrid stores, what is the offset lag threshold for the storage nodes' + consumption to be considered ONLINE + -hr,--hybrid-rewind-seconds for hybrid stores, how far back to rewind in the nearline stream after a + batch push completes + -hsq,--hybrid-store-disk-quota-enabled whether or not enable disk quota for a hybrid store + -ht,--hybrid-time-lag for hybrid stores, servers cannot report ready-to-serve until they see a + message with producer timestamp bigger than (current time - this threshold) + -if,--infile Path to input text file + -ipe,--incremental-push-enabled a flag to see if the store supports incremental push or not + -iss,--include-system-stores Include internal stores maintained by the system. + -itv,--interval monitor data recovery progress at seconds close to the number specified by + the interval parameter until tasks are finished + -k,--key Plain-text key for identifying a record in a store + -kbs,--kafka-bootstrap-servers Kafka bootstrap server URL(s) + -kcc,--kafka-consumer-config-file Configuration file for SSL (optional, if plain-text is available) + -kfq,--server.kafka.fetch.quota.records.per.second The quota of records to fetch from Kafka for the specified fabric. + -kot,--kafka-operation-timeout Timeout in seconds for Kafka operations (default: 30 sec) + -ks,--key-schema-file Path to text file with key schema + -ktlce,--kafka-topic-log-compaction-enabled Enable/disable Kafka log compaction for a topic + -ktmisr,--kafka-topic-min-in-sync-replica Kafka topic minISR config + -ktn,--kafka-topic-name Kafka topic name + -ktp,--kafka-topic-partition Kafka topic partition number + -ktrim,--kafka-topic-retention-in-ms Kafka topic retention time in milliseconds + -ldr,--log-data-record Log the data record for each kafka message on console + -lm,--log-metadata Log the metadata for each kafka message on console + -lrr,--log-rmd-record Log the RMD record for each kafka message on console + -lssi,--latest-superset-schema-id the latest superset schema id for this store + -luv,--largest-used-version Largest used store version number (whether active or not) + -mc,--message_count Max message count when dumping admin messages + -mcls,--min-compaction-lag-seconds Min compaction lag seconds for version topic of hybrid stores + -mpa,--max_poll_attempts The max amount of attempts to poll new data from a Kafka topic (should no + new data be available). + -mrsb,--max-record-size-bytes Store-level max record size in bytes. Used by VeniceWriter to fail batch + push jobs and pause consumption on nearline jobs. + -mxcls,--max-compaction-lag-seconds Max compaction lag seconds for version topic of hybrid stores + -n,--storage-node Helix instance ID for a storage node, eg. lva1-app1234_1690 + -nita,--non-interactive non-interactive mode + -nr,--native-replication-enabled whether or not native replication is enabled for this store. Leader Follow + must also be enabled. + -nrsf,--native-replication-source-fabric The source fabric name to be used in native replication. Remote consumption + will happen from kafka in this fabric. + -nvp,--num-versions-to-preserve Number of version that store should preserve. + -o,--owner Owner email for new store creation + -ob,--hybrid-store-overhead-bypass for hybrid stores, if set to false, updating storage quota will be + multiplied by store db overhead ratio. + -of,--offset Kafka offset number + -p,--partition Partition Id + -pc,--partitioner-class Name of chosen partitioner class + -pde,--partition-detail-enabled A flag to indicate whether to retrieve partition details + -pi,--progress-interval Dump progress after processing this number of messages + -pid,--push-id Push Id + -pn,--partition-count number of partitions a store has + -pod,--parent_output_directory A directory where output can be dumped to. If dumping a kafka topic, the + output will be dumped under this directory. + -pp,--partitioner-params Additional parameters for partitioner. + -ps,--migration-push-strategy Migration push strategy, valid values: [RunBnPOnlyStrategy, + RunVPJOnlyStrategy, RunBnPAndVPJWaitForBothStrategy, + RunBnPAndVPJWaitForBnPOnlyStrategy, RunBnPAndVPJWaitForVPJOnlyStrategy] + -pssa,--push-stream-source-address The url address for the kafka broker which hosts the topic which contains + the push data for this store. + -r,--retry Retry this operation + -rac,--replicate-all-configs Whether all unchanged store configs in parent controller will be replicated + to child controllers + -rb,--readability store's readability + -rc,--recover-cluster Cluster to recover from + -rce,--rmd-chunking-enabled Enable/Disable replication metadata chunking, mostly for Active/Active + replication enabled store with partial update requirement support + -rco,--recovery-command command to execute the data recovery + -re,--repair Repair the store + -regf,--regions-filter A list of regions that will be impacted by the command; can be used by + UpdateStore command + -rf,--replication-factor the number of replica each store version will have + -rq,--read-quota quota for read request hit this store. Measurement is capacity unit + -rv,--remove-view Optional config to specify to disable certain store view + -rve,--regular-version-etl-enabled whether or not to enable regular version etl for this store. + -s,--store Name of Venice store + -scnf,--store-config-name-filter An optional argument in list-store command; pass in a store config to + select stores. If the config name argument is used in the command, users + must specify the config value filter too. + -scp,--ssl-config-path SSl config file path + -scvf,--store-config-value-filter n optional argument in list-store command; if the config name argument is + used in the command, users must specify the config value filter too. + -sd,--start-date Start date in PST. Example: 2020-10-10 10:10:10 + -sf,--source-fabric The fabric where metadata/data copy over starts from + -slsc,--skip-last-store-creation Skip last round of store creation and the following schema manipulation + -snrqe,--storage-node-read-quota-enabled whether storage node read quota is enabled for this store + -so,--starting_offset Starting offset when dumping admin messages, inclusive + -sp,--storage-persona Name of Storage Persona + -sq,--storage-quota maximum capacity a store version or storage persona could have + -ss,--store-size Size of the store in bytes, used to calculate partitioning + -sst,--system-store-type Type of system store to backfill. Supported types are + davinci_push_status_store and meta_store + -st,--store-type the type of the stores. The support type are 'batch_only', hybrid_only', + `incremental_push', 'hybrid_or_incremental', 'system', 'all' + -sts,--stores Name of a group of Venice stores + -su,--server-url Venice server url, eg. http://localhost:1690 This has to be a storage node + -svc,--storage-view-configs Config that describes views to be added for a store. Input is a json map. + Example: {"ExampleView": {"viewClassName": + "com.linkedin.venice.views.ChangeCaptureView","params": {}}} + -u,--url Venice url, eg. http://localhost:1689 This can be a router or a controller + -usde,--enable-unused-schema-deletion Enable unused schema deletion + -v,--version Active store version number + -vc,--view-class Name of a store view class + -vcsc,--venice-client-ssl-config-file Configuration file for querying key in Venice client through SSL. + -vid,--value-schema-id value schema id + -vn,--view-name Name of a store view + -vp,--view-params Additional parameter map of a store view class + -vs,--voldemort-store Voldemort store name + -vson,--vson_store indicate whether it is Vson store or Avro store + -vzu,--venice-zookeeper-url Venice Zookeeper url, eg. localhost:2622 + -wb,--writeability store's writeability + -wc,--write-computation-enabled Whether or not write computation is enabled for a store + -zscf,--zk-ssl-config-file Path to text file with ZK SSL configs Examples: -java -jar venice-admin-tool-0.1.jar --list-stores --url --cluster -java -jar venice-admin-tool-0.1.jar --describe-store --url --cluster --store -java -jar venice-admin-tool-0.1.jar --describe-stores --url --cluster -java -jar venice-admin-tool-0.1.jar --disable-store-write --url --cluster --store -java -jar venice-admin-tool-0.1.jar --enable-store-write --url --cluster --store -java -jar venice-admin-tool-0.1.jar --disable-store-read --url --cluster --store -java -jar venice-admin-tool-0.1.jar --enable-store-read --url --cluster --store -java -jar venice-admin-tool-0.1.jar --disable-store --url --cluster --store -java -jar venice-admin-tool-0.1.jar --enable-store --url --cluster --store -java -jar venice-admin-tool-0.1.jar --job-status --url --cluster --store --version -java -jar venice-admin-tool-0.1.jar --kill-job --url --cluster --store --version -java -jar venice-admin-tool-0.1.jar --skip-admin --url --cluster --offset -java -jar venice-admin-tool-0.1.jar --new-store --url --cluster --store --key-schema-file --value-schema-file --owner --vson_store -java -jar venice-admin-tool-0.1.jar --delete-store --url --cluster --store -java -jar venice-admin-tool-0.1.jar --set-version --url --cluster --store --version -java -jar venice-admin-tool-0.1.jar --add-schema --url --cluster --store --value-schema-file -java -jar venice-admin-tool-0.1.jar --list-storage-nodes --url --cluster -java -jar venice-admin-tool-0.1.jar --cluster-health-instances --url --cluster -java -jar venice-admin-tool-0.1.jar --cluster-health-stores --url --cluster -java -jar venice-admin-tool-0.1.jar --node-removable --url --cluster --storage-node -java -jar venice-admin-tool-0.1.jar --white-list-add-node --url --cluster --storage-node -java -jar venice-admin-tool-0.1.jar --white-list-remove-node --url --cluster --storage-node -java -jar venice-admin-tool-0.1.jar --remove-node --url --cluster --storage-node -java -jar venice-admin-tool-0.1.jar --replicas-of-store --url --cluster --store --version -java -jar venice-admin-tool-0.1.jar --replicas-on-storage-node --url --cluster --storage-node -java -jar venice-admin-tool-0.1.jar --query --url --cluster --store --key -java -jar venice-admin-tool-0.1.jar --schemas --url --cluster --store -java -jar venice-admin-tool-0.1.jar --delete-all-versions --url --cluster --store -java -jar venice-admin-tool-0.1.jar --get-execution --url --cluster --execution -java -jar venice-admin-tool-0.1.jar --set-owner --url --cluster --store --owner -java -jar venice-admin-tool-0.1.jar --set-partition-count --url --cluster --store --partition-count -java -jar venice-admin-tool-0.1.jar --update-store --url --cluster --store --owner --partition-count --version --readability --writeability --storage-quota --read-quota -java -jar venice-admin-tool-0.1.jar --empty-push --url --cluster --store --push-id --store-size -java -jar venice-admin-tool-0.1.jar --enable-throttling --url --cluster -java -jar venice-admin-tool-0.1.jar --disable-throttling --url --cluster -java -jar venice-admin-tool-0.1.jar --enable-max-capacity-protection --url --cluster -java -jar venice-admin-tool-0.1.jar --disable-max-capacity-protection --url --cluster -java -jar venice-admin-tool-0.1.jar --enable-quota-rebalance --url --cluster --expected-router-count -java -jar venice-admin-tool-0.1.jar --disable-quota-rebalance --url --cluster -java -jar venice-admin-tool-0.1.jar --get-routers-cluster-config --url --cluster -java -jar venice-admin-tool-0.1.jar --convert-vson-schema --key-schema-file --value-schema-file -java -jar venice-admin-tool-0.1.jar --get-all-migration-push-strategies --url --cluster -java -jar venice-admin-tool-0.1.jar --get-migration-push-strategy --url --cluster --voldemort-store -java -jar venice-admin-tool-0.1.jar --set-migration-push-strategy --url --cluster --voldemort-store --migration-push-strategy -java -jar venice-admin-tool-0.1.jar --node-replicas-readiness --url --cluster --storage-node - - - - +java -jar venice-admin-tool-all.jar --abort-migration --url --store --cluster-src --cluster-dest [--force ] +java -jar venice-admin-tool-all.jar --add-derived-schema --url --store --value-schema-id --derived-schema-file [--cluster ] +java -jar venice-admin-tool-all.jar --add-schema --url --store --value-schema-file [--cluster ] +java -jar venice-admin-tool-all.jar --add-schema-to-zk --venice-zookeeper-url --store --value-schema-file --value-schema-id --zk-ssl-config-file [--cluster ] +java -jar venice-admin-tool-all.jar --add-to-store-acl --url --store --principal [--cluster ] [--readability ] [--writeability ] +java -jar venice-admin-tool-all.jar --allow-list-add-node --url --cluster --storage-node +java -jar venice-admin-tool-all.jar --allow-list-remove-node --url --cluster --storage-node +java -jar venice-admin-tool-all.jar --backfill-system-stores --url --cluster --system-store-type +java -jar venice-admin-tool-all.jar --backup-store-metadata-from-graveyard --venice-zookeeper-url --zk-ssl-config-file --backup-folder +java -jar venice-admin-tool-all.jar --check-fabric-buildout-status --url --cluster --source-fabric --dest-fabric +java -jar venice-admin-tool-all.jar --cleanup-instance-customized-states --url --cluster +java -jar venice-admin-tool-all.jar --cluster-health-instances --url --cluster [--enable-disabled-replicas ] +java -jar venice-admin-tool-all.jar --cluster-health-stores --url --cluster +java -jar venice-admin-tool-all.jar --compare-store --url --store --fabric-a --fabric-b [--cluster ] +java -jar venice-admin-tool-all.jar --complete-migration --url --store --cluster-src --cluster-dest --fabric +java -jar venice-admin-tool-all.jar --configure-store-view --url --store --view-name [--cluster ] [--view-class ] [--view-params ] [--remove-view ] +java -jar venice-admin-tool-all.jar --convert-vson-schema --key-schema-file --value-schema-file +java -jar venice-admin-tool-all.jar --delete-all-versions --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --delete-kafka-topic --kafka-bootstrap-servers --kafka-topic-name [--kafka-operation-timeout ] [--kafka-consumer-config-file ] +java -jar venice-admin-tool-all.jar --delete-old-version --url --store --version [--cluster ] +java -jar venice-admin-tool-all.jar --delete-storage-persona --url --cluster --storage-persona +java -jar venice-admin-tool-all.jar --delete-store --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --delete-store-acl --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --describe-store --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --describe-stores --url --cluster [--include-system-stores ] +java -jar venice-admin-tool-all.jar --disable-active-active-replication-for-cluster --url --cluster --store-type [--regions-filter ] +java -jar venice-admin-tool-all.jar --disable-max-capacity-protection --url --cluster +java -jar venice-admin-tool-all.jar --disable-quota-rebalance --url --cluster --expected-router-count +java -jar venice-admin-tool-all.jar --disable-store --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --disable-store-read --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --disable-store-write --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --disable-throttling --url --cluster +java -jar venice-admin-tool-all.jar --dump-admin-messages --cluster --kafka-bootstrap-servers --starting_offset --message_count --kafka-consumer-config-file +java -jar venice-admin-tool-all.jar --dump-control-messages --kafka-bootstrap-servers --kafka-consumer-config-file --kafka-topic-name --kafka-topic-partition --starting_offset --message_count +java -jar venice-admin-tool-all.jar --dump-ingestion-state --server-url --store --version [--partition ] +java -jar venice-admin-tool-all.jar --dump-kafka-topic --kafka-bootstrap-servers --kafka-consumer-config-file --kafka-topic-name --cluster --url +java -jar venice-admin-tool-all.jar --dump-topic-partition-ingestion-context --server-url --store --version --kafka-topic-name --kafka-topic-partition +java -jar venice-admin-tool-all.jar --empty-push --url --store --push-id --store-size [--cluster ] +java -jar venice-admin-tool-all.jar --enable-active-active-replication-for-cluster --url --cluster --store-type [--regions-filter ] +java -jar venice-admin-tool-all.jar --enable-max-capacity-protection --url --cluster +java -jar venice-admin-tool-all.jar --enable-quota-rebalance --url --cluster +java -jar venice-admin-tool-all.jar --enable-store --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --enable-store-read --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --enable-store-write --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --enable-throttling --url --cluster +java -jar venice-admin-tool-all.jar --end-fabric-buildout --url --cluster --source-fabric --dest-fabric +java -jar venice-admin-tool-all.jar --end-migration --url --store --cluster-src --cluster-dest +java -jar venice-admin-tool-all.jar --estimate-data-recovery-time --url --dest-fabric [--stores ] [--cluster ] +java -jar venice-admin-tool-all.jar --execute-data-recovery --url --recovery-command --source-fabric --dest-fabric --datetime [--stores ] [--cluster ] [--extra-command-args ] [--debug ] [--non-interactive ] +java -jar venice-admin-tool-all.jar --extract-venice-zk-paths --infile --outfile --cluster-list --base-path +java -jar venice-admin-tool-all.jar --get-all-migration-push-strategies --url --cluster +java -jar venice-admin-tool-all.jar --get-deletable-store-topics --url --cluster +java -jar venice-admin-tool-all.jar --get-execution --url --cluster --execution +java -jar venice-admin-tool-all.jar --get-kafka-topic-configs --url --kafka-topic-name +java -jar venice-admin-tool-all.jar --get-migration-push-strategy --url --cluster --voldemort-store +java -jar venice-admin-tool-all.jar --get-routers-cluster-config --url --cluster +java -jar venice-admin-tool-all.jar --get-storage-persona --url --cluster --storage-persona +java -jar venice-admin-tool-all.jar --get-storage-persona-for-store --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --get-store-acl --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --job-status --url --store [--cluster ] [--version ] +java -jar venice-admin-tool-all.jar --kill-job --url --store --version [--cluster ] +java -jar venice-admin-tool-all.jar --list-bootstrapping-versions --url --cluster +java -jar venice-admin-tool-all.jar --list-cluster-stale-stores --url --cluster +java -jar venice-admin-tool-all.jar --list-cluster-storage-personas --url --cluster +java -jar venice-admin-tool-all.jar --list-storage-nodes --url --cluster +java -jar venice-admin-tool-all.jar --list-store-push-info --url --store [--cluster ] [--partition-detail-enabled ] +java -jar venice-admin-tool-all.jar --list-stores --url --cluster [--include-system-stores ] +java -jar venice-admin-tool-all.jar --migrate-store --url --store --cluster-src --cluster-dest +java -jar venice-admin-tool-all.jar --migration-status --url --store --cluster-src --cluster-dest +java -jar venice-admin-tool-all.jar --monitor-data-recovery --url --dest-fabric --datetime [--stores ] [--cluster ] [--interval ] +java -jar venice-admin-tool-all.jar --new-storage-persona --url --cluster --storage-persona --storage-quota --store --owner +java -jar venice-admin-tool-all.jar --new-store --url --cluster --store --key-schema-file --value-schema-file [--owner ] [--vson_store ] +java -jar venice-admin-tool-all.jar --new-store-acl --url --store --key-schema-file --value-schema-file --acl-perms [--cluster ] [--owner ] [--vson_store ] +java -jar venice-admin-tool-all.jar --node-removable --url --cluster --storage-node +java -jar venice-admin-tool-all.jar --node-replicas-readiness --url --cluster --storage-node +java -jar venice-admin-tool-all.jar --query --url --store --key [--cluster ] [--vson_store ] [--venice-client-ssl-config-file ] +java -jar venice-admin-tool-all.jar --query-kafka-topic --kafka-bootstrap-servers --kafka-consumer-config-file --kafka-topic-name --cluster --url --start-date --key [--end-date ] [--progress-interval ] +java -jar venice-admin-tool-all.jar --recover-store-metadata --url --store --venice-zookeeper-url --kafka-bootstrap-servers --graveyard-clusters [--zk-ssl-config-file ] [--kafka-consumer-config-file ] [--recover-cluster ] [--skip-last-store-creation ] [--repair ] +java -jar venice-admin-tool-all.jar --remove-derived-schema --url --store --value-schema-id --derived-schema-id [--cluster ] +java -jar venice-admin-tool-all.jar --remove-from-store-acl --url --store --principal [--cluster ] [--readability ] [--writeability ] +java -jar venice-admin-tool-all.jar --remove-node --url --cluster --storage-node +java -jar venice-admin-tool-all.jar --replicas-of-store --url --store --version [--cluster ] +java -jar venice-admin-tool-all.jar --replicas-on-storage-node --url --cluster --storage-node +java -jar venice-admin-tool-all.jar --replicate-meta-data --url --cluster --source-fabric --dest-fabric +java -jar venice-admin-tool-all.jar --request-based-metadata --url --server-url --store +java -jar venice-admin-tool-all.jar --schemas --url --store [--cluster ] +java -jar venice-admin-tool-all.jar --send-end-of-push --url --store --version [--cluster ] +java -jar venice-admin-tool-all.jar --set-migration-push-strategy --url --cluster --voldemort-store --migration-push-strategy +java -jar venice-admin-tool-all.jar --set-owner --url --store --owner [--cluster ] +java -jar venice-admin-tool-all.jar --set-partition-count --url --store --partition-count [--cluster ] +java -jar venice-admin-tool-all.jar --set-version --url --store --version [--cluster ] +java -jar venice-admin-tool-all.jar --skip-admin --url --cluster --offset [--skip-div ] +java -jar venice-admin-tool-all.jar --start-fabric-buildout --url --cluster --source-fabric --dest-fabric [--retry ] +java -jar venice-admin-tool-all.jar --update-cluster-config --url --cluster [--fabric ] [--server.kafka.fetch.quota.records.per.second ] [--allow.store.migration ] [--child.controller.admin.topic.consumption.enabled ] +java -jar venice-admin-tool-all.jar --update-kafka-topic-log-compaction --url --kafka-topic-name --kafka-topic-log-compaction-enabled [--cluster ] +java -jar venice-admin-tool-all.jar --update-kafka-topic-min-in-sync-replica --url --kafka-topic-name --kafka-topic-min-in-sync-replica [--cluster ] +java -jar venice-admin-tool-all.jar --update-kafka-topic-retention --url --kafka-topic-name --kafka-topic-retention-in-ms [--cluster ] +java -jar venice-admin-tool-all.jar --update-storage-persona --url --cluster --storage-persona [--storage-quota ] [--store ] [--owner ] +java -jar venice-admin-tool-all.jar --update-store --url --store [--cluster ] [--owner ] [--version ] [--largest-used-version ] [--partition-count ] [--partitioner-class ] [--partitioner-params ] [--readability ] [--writeability ] [--storage-quota ] [--storage-node-read-quota-enabled ] [--hybrid-store-overhead-bypass ] [--read-quota ] [--hybrid-rewind-seconds ] [--hybrid-offset-lag ] [--hybrid-time-lag ] [--hybrid-data-replication-policy ] [--hybrid-buffer-replay-policy ] [--access-control ] [--compression-strategy ] [--client-decompression-enabled ] [--chunking-enabled ] [--rmd-chunking-enabled ] [--batch-get-limit ] [--num-versions-to-preserve ] [--write-computation-enabled ] [--read-computation-enabled ] [--backup-strategy ] [--auto-schema-register-push-job-enabled ] [--incremental-push-enabled ] [--bootstrap-to-online-timeout ] [--hybrid-store-disk-quota-enabled ] [--regular-version-etl-enabled ] [--future-version-etl-enabled ] [--etled-proxy-user-account ] [--native-replication-enabled ] [--push-stream-source-address ] [--backup-version-retention-day ] [--replication-factor ] [--native-replication-source-fabric ] [--replicate-all-configs ] [--active-active-replication-enabled ] [--regions-filter ] [--disable-meta-store ] [--disable-davinci-push-status-store ] [--storage-persona ] [--storage-view-configs ] [--latest-superset-schema-id ] [--min-compaction-lag-seconds ] [--max-compaction-lag-seconds ] [--max-record-size-bytes ] [--enable-unused-schema-deletion ] [--blob-transfer-enabled ] +java -jar venice-admin-tool-all.jar --update-store-acl --url --store --acl-perms [--cluster ] +java -jar venice-admin-tool-all.jar --wipe-cluster --url --cluster --fabric [--store ] [--version ] ``` Summarize the replicas status from the result of command "--replicas-on-storage-node" or "--replicas-of-store": diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java index ed4b94d2cc..b3b392e283 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java @@ -471,12 +471,6 @@ public static void main(String[] args) throws Exception { case REMOVE_FROM_STORE_ACL: removeFromStoreAcl(cmd); break; - case ENABLE_NATIVE_REPLICATION_FOR_CLUSTER: - enableNativeReplicationForCluster(cmd); - break; - case DISABLE_NATIVE_REPLICATION_FOR_CLUSTER: - disableNativeReplicationForCluster(cmd); - break; case ENABLE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER: enableActiveActiveReplicationForCluster(cmd); break; @@ -1810,7 +1804,7 @@ public static void checkMigrationStatus( printSystemStoreMigrationStatus(destControllerClient, storeName, printFunction); } else { // This is a parent controller - System.err.println("\n=================== Parent Controllers ===================="); + printFunction.apply("\n=================== Parent Controllers ===================="); printMigrationStatus(srcControllerClient, storeName, printFunction); printMigrationStatus(destControllerClient, storeName, printFunction); @@ -1821,7 +1815,7 @@ public static void checkMigrationStatus( Map destChildControllerClientMap = getControllerClientMap(destClusterName, response); for (Map.Entry entry: srcChildControllerClientMap.entrySet()) { - System.err.println("\n\n=================== Child Datacenter " + entry.getKey() + " ===================="); + printFunction.apply("\n\n=================== Child Datacenter " + entry.getKey() + " ===================="); ControllerClient srcChildController = entry.getValue(); ControllerClient destChildController = destChildControllerClientMap.get(entry.getKey()); @@ -2586,34 +2580,6 @@ private static void removeFromStoreAcl(CommandLine cmd) throws Exception { } } - private static void enableNativeReplicationForCluster(CommandLine cmd) { - String storeType = getRequiredArgument(cmd, Arg.STORE_TYPE); - String sourceRegionParam = getOptionalArgument(cmd, Arg.NATIVE_REPLICATION_SOURCE_FABRIC); - Optional sourceRegion = - StringUtils.isEmpty(sourceRegionParam) ? Optional.empty() : Optional.of(sourceRegionParam); - String regionsFilterParam = getOptionalArgument(cmd, Arg.REGIONS_FILTER); - Optional regionsFilter = - StringUtils.isEmpty(regionsFilterParam) ? Optional.empty() : Optional.of(regionsFilterParam); - - ControllerResponse response = - controllerClient.configureNativeReplicationForCluster(true, storeType, sourceRegion, regionsFilter); - printObject(response); - } - - private static void disableNativeReplicationForCluster(CommandLine cmd) { - String storeType = getRequiredArgument(cmd, Arg.STORE_TYPE); - String sourceFabricParam = getOptionalArgument(cmd, Arg.NATIVE_REPLICATION_SOURCE_FABRIC); - Optional sourceFabric = - StringUtils.isEmpty(sourceFabricParam) ? Optional.empty() : Optional.of(sourceFabricParam); - String regionsFilterParam = getOptionalArgument(cmd, Arg.REGIONS_FILTER); - Optional regionsFilter = - StringUtils.isEmpty(regionsFilterParam) ? Optional.empty() : Optional.of(regionsFilterParam); - - ControllerResponse response = - controllerClient.configureNativeReplicationForCluster(false, storeType, sourceFabric, regionsFilter); - printObject(response); - } - private static void enableActiveActiveReplicationForCluster(CommandLine cmd) { String storeType = getRequiredArgument(cmd, Arg.STORE_TYPE); String regionsFilterParam = getOptionalArgument(cmd, Arg.REGIONS_FILTER); diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java index 10d2e36a22..6520b020fa 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java @@ -389,16 +389,6 @@ public enum Command { "remove-from-store-acl", "Remove a principal from ACL's for an existing store", new Arg[] { URL, STORE, PRINCIPAL }, new Arg[] { CLUSTER, READABILITY, WRITEABILITY } ), - ENABLE_NATIVE_REPLICATION_FOR_CLUSTER( - "enable-native-replication-for-cluster", - "enable native replication for certain stores (batch-only, hybrid-only, incremental-push, hybrid-or-incremental, all) in a cluster", - new Arg[] { URL, STORE_TYPE }, new Arg[] { CLUSTER, REGIONS_FILTER, NATIVE_REPLICATION_SOURCE_FABRIC } - ), - DISABLE_NATIVE_REPLICATION_FOR_CLUSTER( - "disable-native-replication-for-cluster", - "disable native replication for certain stores (batch-only, hybrid-only, incremental-push, hybrid-or-incremental, all) in a cluster", - new Arg[] { URL, CLUSTER, STORE_TYPE }, new Arg[] { REGIONS_FILTER, NATIVE_REPLICATION_SOURCE_FABRIC } - ), ENABLE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER( "enable-active-active-replication-for-cluster", "enable active active replication for certain stores (batch-only, hybrid-only, incremental-push, hybrid-or-incremental, all) in a cluster", diff --git a/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJob.java b/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJob.java index ff76917318..f4f9343145 100755 --- a/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJob.java +++ b/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJob.java @@ -5,6 +5,7 @@ import static com.linkedin.venice.ConfigKeys.KAFKA_PRODUCER_DELIVERY_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.KAFKA_PRODUCER_REQUEST_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.KAFKA_PRODUCER_RETRIES_CONFIG; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.ConfigKeys.VENICE_PARTITIONERS; import static com.linkedin.venice.VeniceConstants.DEFAULT_SSL_FACTORY_CLASS_NAME; import static com.linkedin.venice.hadoop.VenicePushJobConstants.ALLOW_DUPLICATE_KEY; @@ -47,7 +48,6 @@ import static com.linkedin.venice.hadoop.VenicePushJobConstants.KEY_FIELD_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.LEGACY_AVRO_KEY_FIELD_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.LEGACY_AVRO_VALUE_FIELD_PROP; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.NON_CRITICAL_EXCEPTION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.NOT_SET; import static com.linkedin.venice.hadoop.VenicePushJobConstants.PARENT_CONTROLLER_REGION_NAME; diff --git a/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJobConstants.java b/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJobConstants.java index ae46ea313f..e68d490f41 100644 --- a/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJobConstants.java +++ b/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJobConstants.java @@ -200,11 +200,6 @@ private VenicePushJobConstants() { public static final String REWIND_EPOCH_TIME_BUFFER_IN_SECONDS_OVERRIDE = "rewind.epoch.time.buffer.in.seconds.override"; - /** - * This config specifies if Venice is deployed in a multi-region mode - */ - public static final String MULTI_REGION = "multi.region"; - /** * In single-region mode, this must be a comma-separated list of child controller URLs or {@literal d2://} * In multi-region mode, it must be a comma-separated list of parent controller URLs or {@literal d2://} diff --git a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestKafkaFormatTopicAutoDiscover.java b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestKafkaFormatTopicAutoDiscover.java index 47c8e4579a..07d23703c8 100644 --- a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestKafkaFormatTopicAutoDiscover.java +++ b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestKafkaFormatTopicAutoDiscover.java @@ -1,9 +1,9 @@ package com.linkedin.venice.hadoop; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.D2_ZK_HOSTS_PREFIX; import static com.linkedin.venice.hadoop.VenicePushJobConstants.KAFKA_INPUT_BROKER_URL; import static com.linkedin.venice.hadoop.VenicePushJobConstants.KAFKA_INPUT_TOPIC; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.REWIND_EPOCH_TIME_IN_SECONDS_OVERRIDE; import static com.linkedin.venice.hadoop.VenicePushJobConstants.SOURCE_GRID_FABRIC; import static com.linkedin.venice.hadoop.VenicePushJobConstants.SOURCE_KAFKA; diff --git a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestVenicePushJobCheckpoints.java b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestVenicePushJobCheckpoints.java index 677f4366c0..cb6f295b54 100644 --- a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestVenicePushJobCheckpoints.java +++ b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/TestVenicePushJobCheckpoints.java @@ -1,11 +1,11 @@ package com.linkedin.venice.hadoop; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.COMPRESSION_METRIC_COLLECTION_ENABLED; import static com.linkedin.venice.hadoop.VenicePushJobConstants.COMPRESSION_STRATEGY; import static com.linkedin.venice.hadoop.VenicePushJobConstants.D2_ZK_HOSTS_PREFIX; import static com.linkedin.venice.hadoop.VenicePushJobConstants.INPUT_PATH_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.KEY_FIELD_PROP; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.POLL_JOB_STATUS_INTERVAL_MS; import static com.linkedin.venice.hadoop.VenicePushJobConstants.PUSH_JOB_STATUS_UPLOAD_ENABLE; import static com.linkedin.venice.hadoop.VenicePushJobConstants.SOURCE_GRID_FABRIC; diff --git a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/VenicePushJobTest.java b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/VenicePushJobTest.java index a6a03b274c..2a0295fc17 100644 --- a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/VenicePushJobTest.java +++ b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/VenicePushJobTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.hadoop; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJob.getExecutionStatusFromControllerResponse; import static com.linkedin.venice.hadoop.VenicePushJobConstants.CONTROLLER_REQUEST_RETRY_ATTEMPTS; import static com.linkedin.venice.hadoop.VenicePushJobConstants.D2_ZK_HOSTS_PREFIX; @@ -13,7 +14,6 @@ import static com.linkedin.venice.hadoop.VenicePushJobConstants.KEY_FIELD_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.LEGACY_AVRO_KEY_FIELD_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.LEGACY_AVRO_VALUE_FIELD_PROP; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.PARENT_CONTROLLER_REGION_NAME; import static com.linkedin.venice.hadoop.VenicePushJobConstants.REPUSH_TTL_ENABLE; import static com.linkedin.venice.hadoop.VenicePushJobConstants.REPUSH_TTL_SECONDS; diff --git a/docker/venice-controller/single-dc-configs/controller.properties b/docker/venice-controller/single-dc-configs/controller.properties index b123508d16..2e1f3b514e 100644 --- a/docker/venice-controller/single-dc-configs/controller.properties +++ b/docker/venice-controller/single-dc-configs/controller.properties @@ -22,7 +22,7 @@ enable.hybrid.push.ssl.whitelist=false default.partition.size=100 min.active.replica=1 kafka.replication.factor=1 -child.cluster.allowlist=false +child.cluster.allowlist= default.replica.factor=1 controller.add.version.via.admin.protocol=true controller.ssl.enabled=false diff --git a/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java b/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java index 66de8f7272..474cc0d391 100644 --- a/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java +++ b/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java @@ -68,10 +68,6 @@ public class VeniceConstants { public static final String SYSTEM_PROPERTY_FOR_APP_RUNNING_REGION = "com.linkedin.app.env"; - // public static final String TIMESTAMP_FIELD_NAME = "timestamp"; // - // - // public static final String REPLICATION_CHECKPOINT_VECTOR_FIELD = "replication_checkpoint_vector"; - /** * This is a sentinel value to be used in TopicSwitch message rewindStartTimestamp field between controller and server. * When controller specifies this, Leader server nodes will calculate the rewind start time itself. diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 2f12ccc69a..e81d042d2c 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -108,11 +108,6 @@ private ConfigKeys() { */ public static final String KAFKA_LOG_COMPACTION_FOR_HYBRID_STORES = "kafka.log.compaction.for.hybrid.stores"; - /** - * For log compaction enabled topics, this config will define the minimum time a message will remain uncompacted in the log. - */ - public static final String KAFKA_MIN_LOG_COMPACTION_LAG_MS = "kafka.min.log.compaction.lag.ms"; - /** * The minimum number of in sync replicas to set for store version topics. * @@ -144,39 +139,6 @@ private ConfigKeys() { */ public static final String KAFKA_REPLICATION_FACTOR_RT_TOPICS = "kafka.replication.factor.rt.topics"; - /** - * TODO: the following 3 configs will be deprecated after the native replication migration is changed to a two-step - * process: 1. Turn on the cluster level config that takes care of newly created stores; 2. Run admin command - * to convert existing stores to native replication. - */ - /** - * Cluster-level config to enable native replication for all batch-only stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY = "enable.native.replication.for.batch.only"; - - /** - * Cluster-level config to enable native replication for all hybrid stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_FOR_HYBRID = "enable.native.replication.for.hybrid"; - - /** - * Cluster-level config to enable native replication for new batch-only stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY = - "enable.native.replication.as.default.for.batch.only"; - - /** - * Cluster-level config to enable native replication for new hybrid stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID = - "enable.native.replication.as.default.for.hybrid"; - - /** - * Cluster-level config to enable active-active replication for new batch-only stores. - */ - public static final String ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE = - "enable.active.active.replication.as.default.for.batch.only.store"; - /** * Cluster-level config to enable active-active replication for new hybrid stores. */ @@ -189,7 +151,7 @@ private ConfigKeys() { public static final String ENABLE_BLOB_TRANSFER = "enable.blob.transfer"; /** - * Sets the default for whether or not do schema validation for all stores + * Sets the default for whether to do schema validation or not for all stores */ public static final String CONTROLLER_SCHEMA_VALIDATION_ENABLED = "controller.schema.validation.enabled"; @@ -216,7 +178,6 @@ private ConfigKeys() { public static final String PARTITION_COUNT_ROUND_UP_SIZE = "partition.count.round.up.size"; public static final String OFFLINE_JOB_START_TIMEOUT_MS = "offline.job.start.timeout.ms"; public static final String DELAY_TO_REBALANCE_MS = "delay.to.rebalance.ms"; - public static final String MIN_ACTIVE_REPLICA = "min.active.replica"; public static final String CLUSTER_TO_D2 = "cluster.to.d2"; public static final String CLUSTER_TO_SERVER_D2 = "cluster.to.server.d2"; public static final String HELIX_SEND_MESSAGE_TIMEOUT_MS = "helix.send.message.timeout.ms"; @@ -262,13 +223,6 @@ private ConfigKeys() { public static final String TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS = "topic.cleanup.sleep.interval.between.topic.list.fetch.ms"; public static final String TOPIC_CLEANUP_DELAY_FACTOR = "topic.cleanup.delay.factor"; - public static final String TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS = - "topic.cleanup.send.concurrent.delete.requests.enabled"; - - /** - * Sleep interval for polling topic deletion status from ZK. - */ - public static final String TOPIC_DELETION_STATUS_POLL_INTERVAL_MS = "topic.deletion.status.poll.interval.ms"; /** * The following config is to control the default retention time in milliseconds if it is not specified in store level. @@ -304,9 +258,9 @@ private ConfigKeys() { public static final String CONTROLLER_ENFORCE_SSL = "controller.enforce.ssl"; /** - * Whether child controllers will directly consume the source admin topic in the parent Kafka cluster. + * This config specifies if Venice is deployed in a multi-region mode */ - public static final String ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED = "admin.topic.remote.consumption.enabled"; + public static final String MULTI_REGION = "multi.region"; /** * This config defines the source region name of the admin topic @@ -314,7 +268,8 @@ private ConfigKeys() { public static final String ADMIN_TOPIC_SOURCE_REGION = "admin.topic.source.region"; /** - * This following config defines whether admin consumption should be enabled or not, and this config will only control the behavior in Child Controller. + * This following config defines whether admin consumption should be enabled or not, and this config will only control + * the behavior in Child Controller. This is used for store migration. */ public static final String CHILD_CONTROLLER_ADMIN_TOPIC_CONSUMPTION_ENABLED = "child.controller.admin.topic.consumption.enabled"; @@ -744,12 +699,6 @@ private ConfigKeys() { public static final String SERVER_INGESTION_ISOLATION_HEARTBEAT_REQUEST_TIMEOUT_SECONDS = "server.ingestion.isolation.heartbeat.request.timeout.seconds"; - /** - * Timeout for single metric request sent from main process to forked ingestion process. - */ - public static final String SERVER_INGESTION_ISOLATION_METRIC_REQUEST_TIMEOUT_SECONDS = - "server.ingestion.isolation.metric.request.timeout.seconds"; - /** * whether to enable checksum verification in the ingestion path from kafka to database persistency. If enabled it will * keep a running checksum for all and only PUT kafka data message received in the ingestion task and periodically @@ -1087,15 +1036,6 @@ private ConfigKeys() { * */ public static final String CONTROLLER_CLUSTER_REPLICA = "controller.cluster.replica"; - /** - * The time window in ms used to throttle the Kafka topic creation, during the time window, only 1 topic is allowed to - * be created. - */ - public static final String TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS = "topic.creation.throttling.time.window.ms"; - - /** Timeout for create topic and delete topic operations. */ - public static final String TOPIC_MANAGER_KAFKA_OPERATION_TIMEOUT_MS = "topic.manager.kafka.operation.timeout.ms"; - /** * This is the minimum number of Kafka topics that are guaranteed to be preserved by the leaky topic clean * up routine. The topics with the highest version numbers will be favored by this preservative behavior. @@ -1255,15 +1195,9 @@ private ConfigKeys() { public static final String PARENT_KAFKA_CLUSTER_FABRIC_LIST = "parent.kafka.cluster.fabric.list"; /** - * Whether A/A is enabled on the controller. When it is true, all A/A required config (e.g. - * {@link #ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST}) must be set. - */ - public static final String ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER = "active.active.enabled.on.controller"; - - /** - * A list of fabrics that are source(s) of the active active real time replication. When active-active replication - * is enabled on the controller {@link #ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER} is true, this list should contain fabrics - * where the Venice server should consume from when it accepts the TS (TopicSwitch) message. + * A list of regions that are source(s) of the Active/Active real time replication. When running in a multi-region + * mode, this list should contain region names where the Venice server should consume from when it accepts the + * TS (TopicSwitch) message. * Example value of this config: "dc-0, dc-1, dc-2". */ public static final String ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST = "active.active.real.time.source.fabric.list"; @@ -1276,12 +1210,6 @@ private ConfigKeys() { public static final String PARENT_CONTROLLER_WAITING_TIME_FOR_CONSUMPTION_MS = "parent.controller.waiting.time.for.consumption.ms"; - /** - * If there is a failure in consuming from the admin topic, skip the message after retrying for this many minutes - * Default 5 days - */ - public static final String ADMIN_CONSUMPTION_TIMEOUT_MINUTES = "admin.consumption.timeout.minute"; - /** * The maximum time allowed for worker threads to execute admin messages in one cycle. A cycle is the processing of * delegated admin messages by some number of worker thread(s) defined by {@code ADMIN_CONSUMPTION_MAX_WORKER_THREAD_POOL_SIZE}. @@ -1561,16 +1489,6 @@ private ConfigKeys() { */ public static final String CONTROLLER_HAAS_SUPER_CLUSTER_NAME = "controller.haas.super.cluster.name"; - /** - * Whether to enable batch push (including GF job) from Admin in Child Controller. - * In theory, we should disable batch push in Child Controller no matter what, but the fact is that today there are - * many tests, which are doing batch pushes to an individual cluster setup (only Child Controller), so disabling batch push from Admin - * in Child Controller will require a lot of refactoring. - * So the current strategy is to enable it by default, but disable it in EI and PROD. - */ - public static final String CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD = - "controller.enable.batch.push.from.admin.in.child"; - /** * A config that turns the key/value profiling stats on and off. This config can be placed in both Router and SNs and it * is off by default. When switching it on, We will emit a fine grained histogram that reflects the distribution of @@ -1589,11 +1507,6 @@ private ConfigKeys() { * A config specifies which partitioning scheme should be used by VenicePushJob. */ public static final String PARTITIONER_CLASS = "partitioner.class"; - /** - * A configs of over-partitioning factor - * number of Kafka partitions in each partition - */ - public static final String AMPLIFICATION_FACTOR = "amplification.factor"; /** * A unique id that can represent this instance diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java index 0e708e6b7b..b1fe560ec5 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java @@ -29,7 +29,6 @@ import static com.linkedin.venice.controllerapi.ControllerApiConstants.LOCKED_NODE_ID_LIST_SEPARATOR; import static com.linkedin.venice.controllerapi.ControllerApiConstants.LOCKED_STORAGE_NODE_IDS; import static com.linkedin.venice.controllerapi.ControllerApiConstants.NAME; -import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OFFSET; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OPERATION; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OWNER; @@ -1101,19 +1100,6 @@ public SystemStoreHeartbeatResponse getHeartbeatFromSystemStore(String storeName SystemStoreHeartbeatResponse.class); } - public ControllerResponse configureNativeReplicationForCluster( - boolean enableNativeReplication, - String storeType, - Optional sourceFabric, - Optional regionsFilter) { - // Verify the input storeType is valid - VeniceUserStoreType.valueOf(storeType.toUpperCase()); - QueryParams params = newParams().add(STATUS, enableNativeReplication).add(STORE_TYPE, storeType); - sourceFabric.ifPresent(s -> params.add(NATIVE_REPLICATION_SOURCE_FABRIC, s)); - regionsFilter.ifPresent(f -> params.add(REGIONS_FILTER, f)); - return request(ControllerRoute.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER, params, ControllerResponse.class); - } - public ControllerResponse configureActiveActiveReplicationForCluster( boolean enableActiveActiveReplication, String storeType, diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java index 037131ab96..83d2352b7f 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java @@ -231,9 +231,6 @@ public enum ControllerRoute { UPDATE_ACL("/update_acl", HttpMethod.POST, Arrays.asList(CLUSTER, NAME, ACCESS_PERMISSION)), GET_ACL("/get_acl", HttpMethod.GET, Arrays.asList(CLUSTER, NAME)), DELETE_ACL("/delete_acl", HttpMethod.GET, Arrays.asList(CLUSTER, NAME)), - CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER( - "/configure_native_replication_for_cluster", HttpMethod.POST, Arrays.asList(CLUSTER, STORE_TYPE, STATUS) - ), CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER( "/configure_active_active_replication_for_cluster", HttpMethod.POST, Arrays.asList(CLUSTER, STORE_TYPE, STATUS) ), GET_DELETABLE_STORE_TOPICS("/get_deletable_store_topics", HttpMethod.GET, Collections.emptyList()), diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java index 7d6c988799..b66e79b222 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java @@ -461,15 +461,14 @@ public static boolean verifyTransition(ExecutionStatus newStatus, ExecutionStatu } public static List parseCommaSeparatedStringToList(String rawString) { - String[] strArray = rawString.split(",\\s*"); - if (strArray.length < 1) { - throw new VeniceException("Invalid input: " + rawString); + if (StringUtils.isEmpty(rawString)) { + return Collections.emptyList(); } - return Arrays.asList(strArray); + return Arrays.asList(rawString.split(",\\s*")); } public static Set parseCommaSeparatedStringToSet(String rawString) { - if (rawString == null || rawString.length() == 0) { + if (StringUtils.isEmpty(rawString)) { return Collections.emptySet(); } return Utils.setOf(rawString.split(",\\s*")); @@ -736,10 +735,6 @@ public static Set mutableSetOf(T... objs) { return new HashSet<>(Arrays.asList(objs)); } - public static long calculateDurationMs(Time time, long startTimeMs) { - return time.getMilliseconds() - startTimeMs; - } - public static void closeQuietlyWithErrorLogged(Closeable... closeables) { if (closeables == null) { return; diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java index 24f3cc4acf..1459858577 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java @@ -15,6 +15,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import org.testng.Assert; import org.testng.annotations.Test; @@ -183,4 +184,15 @@ public void testParseMap() { public void testSanitizingStringForLogger() { Assert.assertEquals(Utils.getSanitizedStringForLogger(".abc.123."), "_abc_123_"); } + + @Test + public void testParseCommaSeparatedStringToSet() { + Assert.assertTrue(Utils.parseCommaSeparatedStringToSet(null).isEmpty()); + Assert.assertTrue(Utils.parseCommaSeparatedStringToSet("").isEmpty()); + Set set = Utils.parseCommaSeparatedStringToSet("a,b,c"); + Assert.assertEquals(set.size(), 3); + Assert.assertTrue(set.contains("a")); + Assert.assertTrue(set.contains("b")); + Assert.assertTrue(set.contains("c")); + } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java index 14215ff7b5..746392059b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AbstractTestVeniceHelixAdmin.java @@ -1,7 +1,6 @@ package com.linkedin.venice.controller; import static com.linkedin.venice.ConfigKeys.ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED; -import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; import static com.linkedin.venice.ConfigKeys.CLUSTER_TO_D2; import static com.linkedin.venice.ConfigKeys.CLUSTER_TO_SERVER_D2; @@ -13,7 +12,6 @@ import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; import static com.linkedin.venice.ConfigKeys.KAFKA_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED; -import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS; import static com.linkedin.venice.ConfigKeys.UNREGISTER_METRIC_FOR_DELETED_STORE_ENABLED; import static com.linkedin.venice.ConfigKeys.ZOOKEEPER_ADDRESS; @@ -82,14 +80,6 @@ class AbstractTestVeniceHelixAdmin { final PubSubTopicRepository pubSubTopicRepository = new PubSubTopicRepository(); - public void setupCluster() throws Exception { - setupCluster(true, new MetricsRepository()); - } - - public void setupCluster(boolean createParticipantStore) throws Exception { - setupCluster(createParticipantStore, new MetricsRepository()); - } - public void setupCluster(boolean createParticipantStore, MetricsRepository metricsRepository) throws Exception { Utils.thisIsLocalhost(); zkServerWrapper = ServiceFactory.getZkServer(); @@ -118,12 +108,7 @@ public void setupCluster(boolean createParticipantStore, MetricsRepository metri if (createParticipantStore) { // Wait for participant store to finish materializing - TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, () -> { - Store store = - veniceAdmin.getStore(clusterName, VeniceSystemStoreUtils.getParticipantStoreNameForCluster(clusterName)); - Assert.assertNotNull(store); - Assert.assertEquals(store.getCurrentVersion(), 1); - }); + verifyParticipantMessageStoreSetup(); } } @@ -210,9 +195,7 @@ Properties getControllerProperties(String clusterName) throws IOException { properties.put(CONTROLLER_ADD_VERSION_VIA_ADMIN_PROTOCOL, true); properties.put(ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED, false); properties.put(PARTICIPANT_MESSAGE_STORE_ENABLED, true); - properties.put(TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS, true); properties.put(CONTROLLER_SYSTEM_SCHEMA_CLUSTER_NAME, clusterName); - properties.put(CHILD_CLUSTER_ALLOWLIST, "dc-0"); properties.put(CONTROLLER_SSL_ENABLED, false); properties.putAll(PubSubBrokerWrapper.getBrokerDetailsForClients(Collections.singletonList(pubSubBrokerWrapper))); return properties; @@ -263,9 +246,9 @@ VeniceHelixAdmin getFollower(List admins, String cluster) { /** * Participant store should be set up by child controller. */ - void verifyParticipantMessageStoreSetup() { + private void verifyParticipantMessageStoreSetup() { String participantStoreName = VeniceSystemStoreUtils.getParticipantStoreNameForCluster(clusterName); - TestUtils.waitForNonDeterministicAssertion(5, TimeUnit.SECONDS, () -> { + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, () -> { Store store = veniceAdmin.getStore(clusterName, participantStoreName); Assert.assertNotNull(store); Assert.assertEquals(store.getVersions().size(), 1); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java index f6e7699d38..ab23710c5a 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java @@ -1,8 +1,6 @@ package com.linkedin.venice.controller; -import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; import static com.linkedin.venice.utils.TestUtils.assertCommand; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -20,6 +18,7 @@ import java.io.IOException; import java.util.Properties; import java.util.concurrent.TimeUnit; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -27,195 +26,63 @@ public class TestClusterLevelConfigForActiveActiveReplication { private static final long TEST_TIMEOUT = 120 * Time.MS_PER_SECOND; + private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; + private ControllerClient parentControllerClient; + @BeforeClass(alwaysRun = true) public void setUp() { Utils.thisIsLocalhost(); + Properties parentControllerProps = new Properties(); + parentControllerProps + .setProperty(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, Boolean.toString(true)); + multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfRouters(1) + .numberOfServers(1) + .parentControllerProperties(parentControllerProps) + .build()); + String clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; + parentControllerClient = + new ControllerClient(clusterName, multiRegionMultiClusterWrapper.getControllerConnectString()); } - @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelActiveActiveReplicationConfigForNewHybridStores() throws IOException { - Properties parentControllerProps = getActiveActiveControllerProperties(true, false); - try ( - VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) - .numberOfParentControllers(1) - .numberOfChildControllers(1) - .numberOfRouters(1) - .numberOfServers(1) - .parentControllerProperties(parentControllerProps) - .build()); - ControllerClient parentControllerClient = new ControllerClient( - multiRegionMultiClusterWrapper.getClusterNames()[0], - multiRegionMultiClusterWrapper.getControllerConnectString())) { - String storeName = Utils.getUniqueString("test-store-hybrid"); - String pushJobId1 = "test-push-job-id-1"; - parentControllerClient.createNewStore(storeName, "test-owner", "\"string\"", "\"string\""); - parentControllerClient.emptyPush(storeName, pushJobId1, 1); - - // Version 1 should exist. - StoreInfo store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); - assertEquals(store.getVersions().size(), 1); - - // Check store level Active/Active is enabled or not - assertFalse(store.isActiveActiveReplicationEnabled()); - - // Convert to hybrid store - assertCommand( - parentControllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(1000L).setHybridOffsetLagThreshold(1000L))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertTrue(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); - }); - - // Reverting hybrid configs disables A/A mode - assertCommand( - parentControllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(-1).setHybridOffsetLagThreshold(-1))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertFalse(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); - }); - } + @AfterClass(alwaysRun = true) + public void cleanUp() { + Utils.closeQuietlyWithErrorLogged(multiRegionMultiClusterWrapper); } @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelActiveActiveReplicationConfigForNewIncrementalPushStores() throws IOException { - Properties parentControllerProps = getActiveActiveControllerProperties(true, false); - try ( - VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) - .numberOfParentControllers(1) - .numberOfChildControllers(1) - .numberOfRouters(1) - .numberOfServers(1) - .parentControllerProperties(parentControllerProps) - .build()); - ControllerClient parentControllerClient = new ControllerClient( - multiRegionMultiClusterWrapper.getClusterNames()[0], - multiRegionMultiClusterWrapper.getControllerConnectString())) { - String storeName = Utils.getUniqueString("test-store-incremental"); - String pushJobId1 = "test-push-job-id-1"; - parentControllerClient.createNewStore(storeName, "test-owner", "\"string\"", "\"string\""); - parentControllerClient.emptyPush(storeName, pushJobId1, 1); - - // Version 1 should exist. - StoreInfo store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); - assertEquals(store.getVersions().size(), 1); - assertFalse(store.isIncrementalPushEnabled()); - assertFalse(store.isActiveActiveReplicationEnabled()); - - // Disabling incremental push on a store that has inc push disabled should not have any side effects - assertCommand( - parentControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setIncrementalPushEnabled(false))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertEquals(parentControllerClient.getStore(storeName).getStore().getVersions().size(), 1); - }); - store = parentControllerClient.getStore(storeName).getStore(); - assertFalse(store.isIncrementalPushEnabled()); - assertFalse(store.isActiveActiveReplicationEnabled()); - - // Enable inc-push - assertCommand( - parentControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setIncrementalPushEnabled(true))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertTrue(parentControllerClient.getStore(storeName).getStore().isIncrementalPushEnabled()); - }); - store = parentControllerClient.getStore(storeName).getStore(); - assertTrue(store.isActiveActiveReplicationEnabled()); - - // After inc push is disabled, even though default A/A config for pure hybrid store is false, - // the store's A/A config is retained. - assertCommand( - parentControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setIncrementalPushEnabled(false))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertFalse(parentControllerClient.getStore(storeName).getStore().isIncrementalPushEnabled()); - }); - store = parentControllerClient.getStore(storeName).getStore(); - assertTrue(store.isActiveActiveReplicationEnabled()); - } - } - - @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelActiveActiveReplicationConfigForNewBatchOnlyStores() throws IOException { - Properties parentControllerProps = getActiveActiveControllerProperties(false, true); - try ( - VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) - .numberOfParentControllers(1) - .numberOfChildControllers(1) - .numberOfRouters(1) - .numberOfServers(1) - .parentControllerProperties(parentControllerProps) - .build()); - ControllerClient parentControllerClient = new ControllerClient( - multiRegionMultiClusterWrapper.getClusterNames()[0], - multiRegionMultiClusterWrapper.getControllerConnectString())) { - String storeName = Utils.getUniqueString("test-store-batch-only"); - String pushJobId1 = "test-push-job-id-1"; - parentControllerClient.createNewStore(storeName, "test-owner", "\"string\"", "\"string\""); - parentControllerClient.emptyPush(storeName, pushJobId1, 1); - - // Version 1 should exist. - StoreInfo store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); - assertEquals(store.getVersions().size(), 1); - - // Check store level Active/Active is enabled or not - assertTrue(store.isActiveActiveReplicationEnabled()); - - // After updating the store to have incremental push enabled, it's A/A is still enabled - assertCommand( - parentControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setIncrementalPushEnabled(true))); - store = parentControllerClient.getStore(storeName).getStore(); - assertTrue(parentControllerClient.getStore(storeName).getStore().isIncrementalPushEnabled()); - assertTrue(store.isActiveActiveReplicationEnabled()); - - // Let's disable the A/A config for the store. - assertCommand( - parentControllerClient - .updateStore(storeName, new UpdateStoreQueryParams().setActiveActiveReplicationEnabled(false))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertFalse(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); - }); - - // After updating the store back to a batch-only store, it's A/A becomes enabled again - assertCommand( - parentControllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setIncrementalPushEnabled(false) - .setHybridRewindSeconds(-1) - .setHybridOffsetLagThreshold(-1))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertTrue(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); - }); - - // After updating the store to be a hybrid store, it's A/A should still be enabled. - assertCommand( - parentControllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(1000L).setHybridOffsetLagThreshold(1000L))); - TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { - assertTrue(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); - }); - } - } - - private Properties getActiveActiveControllerProperties( - boolean enableActiveActiveForHybrid, - boolean enableActiveActiveForBatchOnly) { - Properties props = new Properties(); - props.setProperty(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, "true"); - // Enable Active/Active replication for hybrid stores through cluster-level config - props.setProperty( - ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, - Boolean.toString(enableActiveActiveForHybrid)); - // Enable Active/Active replication for batch-only stores through cluster-level config - props.setProperty( - ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE, - Boolean.toString(enableActiveActiveForBatchOnly)); - return props; + public void testClusterLevelActiveActiveReplicationConfigForNewHybridStores() throws IOException { + String storeName = Utils.getUniqueString("test-store-hybrid"); + String pushJobId1 = "test-push-job-id-1"; + parentControllerClient.createNewStore(storeName, "test-owner", "\"string\"", "\"string\""); + parentControllerClient.emptyPush(storeName, pushJobId1, 1); + + // Version 1 should exist. + StoreInfo store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); + assertEquals(store.getVersions().size(), 1); + + // Check store level Active/Active is enabled or not + assertFalse(store.isActiveActiveReplicationEnabled()); + + // Convert to hybrid store + assertCommand( + parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000L).setHybridOffsetLagThreshold(1000L))); + TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { + assertTrue(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); + }); + + // Reverting hybrid configs disables A/A mode + assertCommand( + parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(-1).setHybridOffsetLagThreshold(-1))); + TestUtils.waitForNonDeterministicAssertion(TEST_TIMEOUT, TimeUnit.MILLISECONDS, () -> { + assertFalse(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); + }); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java index a4fd0d2d06..3cc1fc34ec 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java @@ -1,6 +1,5 @@ package com.linkedin.venice.controller; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES; import static com.linkedin.venice.utils.TestUtils.assertCommand; @@ -34,8 +33,6 @@ public class TestClusterLevelConfigForNativeReplication { public void setUp() { Utils.thisIsLocalhost(); Properties parentControllerProps = new Properties(); - // enable native replication for batch-only stores through cluster-level config - parentControllerProps.setProperty(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, "true"); parentControllerProps.setProperty(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, "dc-batch"); parentControllerProps.setProperty(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES, "dc-hybrid"); multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestControllerEnforceSSL.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestControllerEnforceSSL.java index 8b89d7c7dd..7b8a56a046 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestControllerEnforceSSL.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestControllerEnforceSSL.java @@ -46,7 +46,6 @@ public void testInsecureRouteFailWhenEnforcingSSL() { new VeniceControllerCreateOptions.Builder(CLUSTER_NAME, zkServer, pubSubBrokerWrapper).replicationFactor(1) .partitionSize(10) .rebalanceDelayMs(0) - .minActiveReplica(1) .sslToKafka(true) .extraProperties(extraProperties) .regionName(STANDALONE_REGION_NAME) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestD2ControllerClient.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestD2ControllerClient.java index 345ec29495..2bd6884f4d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestD2ControllerClient.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestD2ControllerClient.java @@ -35,7 +35,6 @@ public void testD2ControllerClientEnd2End() { new VeniceControllerCreateOptions.Builder(CLUSTER_NAME, zkServer, pubSubBrokerWrapper).replicationFactor(1) .partitionSize(10) .rebalanceDelayMs(0) - .minActiveReplica(1) .sslToKafka(true) .d2Enabled(true) .clusterToD2(Collections.singletonMap(CLUSTER_NAME, clusterD2Service)) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestDelayedRebalance.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestDelayedRebalance.java index b333c9a025..c5e30bd86c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestDelayedRebalance.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestDelayedRebalance.java @@ -30,7 +30,6 @@ public class TestDelayedRebalance { // Ensure delayed rebalance time out is larger than test timeout to avoid doing rebalance due to // waitForNonDeterministicCompletion long delayRebalanceMS = testTimeOutMS * 2; - int minActiveReplica = replicaFactor - 1; @BeforeMethod(alwaysRun = true) public void setUp() { @@ -41,7 +40,6 @@ public void setUp() { .replicationFactor(replicaFactor) .partitionSize(partitionSize) .rebalanceDelayMs(delayRebalanceMS) - .minActiveReplica(minActiveReplica) .build(); cluster = ServiceFactory.getVeniceCluster(options); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestEnablingSSLPushInVeniceHelixAdminWithIsolatedEnvironment.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestEnablingSSLPushInVeniceHelixAdminWithIsolatedEnvironment.java index 935635d976..bb5350adbd 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestEnablingSSLPushInVeniceHelixAdminWithIsolatedEnvironment.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestEnablingSSLPushInVeniceHelixAdminWithIsolatedEnvironment.java @@ -3,6 +3,7 @@ import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.utils.TestUtils; +import io.tehuti.metrics.MetricsRepository; import java.io.IOException; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -23,7 +24,7 @@ public class TestEnablingSSLPushInVeniceHelixAdminWithIsolatedEnvironment extend @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { - setupCluster(false); + setupCluster(false, new MetricsRepository()); } @AfterMethod(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java index 59ea1f13e6..6bb682135b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java @@ -1,8 +1,6 @@ package com.linkedin.venice.controller; import static com.linkedin.venice.ConfigKeys.ALLOW_CLUSTER_WIPE; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; import com.linkedin.venice.AdminTool; @@ -29,9 +27,17 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; +/** + * Note: These tests use an unsupported mode during setup - by creating stores and running pushes + * to them directly in child regions. This is not how new regions will be added. We need the test setup to support + * adding blank regions so that we can simulate the true fabric buildout process. Because of this, I've disabled these + * tests for now. + */ +@Ignore public class TestFabricBuildout { private static final int TEST_TIMEOUT = 90_000; // ms @@ -47,9 +53,7 @@ public class TestFabricBuildout { @BeforeClass public void setUp() { Properties childControllerProperties = new Properties(); - childControllerProperties.setProperty(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, "true"); childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true"); - childControllerProperties.setProperty(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, "true"); Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithIsolatedEnvironment.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithIsolatedEnvironment.java index 8bcd22546c..18bbad20d8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithIsolatedEnvironment.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithIsolatedEnvironment.java @@ -42,7 +42,7 @@ public class TestVeniceHelixAdminWithIsolatedEnvironment extends AbstractTestVeniceHelixAdmin { @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { - setupCluster(false); + setupCluster(false, new MetricsRepository()); } @AfterMethod(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java index b4a0e020ca..b635940e8e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java @@ -101,7 +101,6 @@ public class TestVeniceHelixAdminWithSharedEnvironment extends AbstractTestVenic @BeforeClass(alwaysRun = true) public void setUp() throws Exception { setupCluster(true, metricsRepository); - verifyParticipantMessageStoreSetup(); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java index 5201f366af..dda4f1fb6d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java @@ -94,7 +94,6 @@ public void setUp() { .numberOfRouters(1) .replicationFactor(1) .partitionSize(100) - .minActiveReplica(0) .sslToStorageNodes(false) .sslToKafka(false) .extraProperties(clusterConfig) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index 3745f1bf00..2d3accea01 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -61,7 +61,6 @@ import com.linkedin.venice.common.VeniceSystemStoreUtils; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.MultiSchemaResponse; -import com.linkedin.venice.controllerapi.NewStoreResponse; import com.linkedin.venice.controllerapi.StoreResponse; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; @@ -82,7 +81,6 @@ import com.linkedin.venice.meta.Instance; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; -import com.linkedin.venice.meta.VeniceUserStoreType; import com.linkedin.venice.meta.Version; import com.linkedin.venice.offsets.OffsetRecord; import com.linkedin.venice.pubsub.PubSubTopicPartitionImpl; @@ -577,172 +575,6 @@ public void testActiveActiveForHeartbeatSystemStores() throws Exception { }); } - @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelAdminCommandForNativeReplication() throws Exception { - motherOfAllTests( - "testClusterLevelAdminCommandForNativeReplication", - updateStoreQueryParams -> updateStoreQueryParams.setPartitionCount(1), - 10, - (parentControllerClient, clusterName, batchOnlyStoreName, props, inputDir) -> { - // Create a hybrid store - String hybridStoreName = Utils.getUniqueString("hybrid-store"); - NewStoreResponse newStoreResponse = parentControllerClient - .createNewStore(hybridStoreName, "", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - Assert.assertFalse(newStoreResponse.isError()); - UpdateStoreQueryParams updateStoreParams = - new UpdateStoreQueryParams().setHybridRewindSeconds(10).setHybridOffsetLagThreshold(2); - assertCommand(parentControllerClient.updateStore(hybridStoreName, updateStoreParams)); - - /** - * Create an incremental push enabled store - */ - String incrementPushStoreName = Utils.getUniqueString("incremental-push-store"); - newStoreResponse = parentControllerClient - .createNewStore(incrementPushStoreName, "", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - Assert.assertFalse(newStoreResponse.isError()); - updateStoreParams = new UpdateStoreQueryParams().setIncrementalPushEnabled(true); - assertCommand(parentControllerClient.updateStore(incrementPushStoreName, updateStoreParams)); - - final Optional defaultNativeReplicationSource = Optional.of(DEFAULT_NATIVE_REPLICATION_SOURCE); - final Optional newNativeReplicationSource = Optional.of("new-nr-source"); - /** - * Run admin command to disable native replication for all batch-only stores in the cluster - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - false, - VeniceUserStoreType.BATCH_ONLY.toString(), - Optional.empty(), - Optional.empty())); - - childDatacenters.get(0) - .getClusters() - .get(clusterName) - .useControllerClient( - dc0Client -> childDatacenters.get(1).getClusters().get(clusterName).useControllerClient(dc1Client -> { - List allControllerClients = - Arrays.asList(parentControllerClient, dc0Client, dc1Client); - - /** - * Batch-only stores should have native replication enabled; hybrid stores or incremental push stores - * have native replication enabled with dc-0 as source. - */ - NativeReplicationTestUtils - .verifyDCConfigNativeRepl(allControllerClients, batchOnlyStoreName, false); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - hybridStoreName, - true, - defaultNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - incrementPushStoreName, - true, - defaultNativeReplicationSource); - - /** - * Second test: - * 1. Revert the cluster to previous state - * 2. Test the cluster level command that converts all hybrid stores to native replication - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - true, - VeniceUserStoreType.BATCH_ONLY.toString(), - newNativeReplicationSource, - Optional.empty())); - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - false, - VeniceUserStoreType.HYBRID_ONLY.toString(), - Optional.empty(), - Optional.empty())); - - /** - * Hybrid stores shouldn't have native replication enabled; batch-only stores should have native replication - * enabled with the new source fabric and incremental push stores should have native replication enabled - * with original source fabric. - */ - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - batchOnlyStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl(allControllerClients, hybridStoreName, false); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - incrementPushStoreName, - true, - defaultNativeReplicationSource); - - /** - * Third test: - * 1. Revert the cluster to previous state - * 2. Test the cluster level command that disables native replication for all incremental push stores - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - true, - VeniceUserStoreType.HYBRID_ONLY.toString(), - newNativeReplicationSource, - Optional.empty())); - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - false, - VeniceUserStoreType.INCREMENTAL_PUSH.toString(), - Optional.empty(), - Optional.empty())); - - /** - * Incremental push stores shouldn't have native replication enabled; batch-only stores and hybrid stores - * should have native replication enabled with the new source fabric. - */ - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - batchOnlyStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - hybridStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils - .verifyDCConfigNativeRepl(allControllerClients, incrementPushStoreName, false); - - /** - * Fourth test: - * Test the cluster level command that enables native replication for all incremental push stores - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - true, - VeniceUserStoreType.INCREMENTAL_PUSH.toString(), - newNativeReplicationSource, - Optional.empty())); - - /** - * All stores should have native replication enabled with the new source fabric - */ - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - batchOnlyStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - hybridStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - incrementPushStoreName, - true, - newNativeReplicationSource); - })); - }); - } - @Test(timeOut = TEST_TIMEOUT) public void testMultiDataCenterRePushWithIncrementalPush() throws Exception { motherOfAllTests( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java index b748b96784..80bef8122f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java @@ -102,18 +102,9 @@ public void testPushJobWithSourceGridFabricSelection(int recordCount, int partit String storeName = Utils.getUniqueString("store"); String parentControllerUrls = multiRegionMultiClusterWrapper.getControllerConnectString(); - // Enable NR in all regions and A/A in parent region and 1 child region only. The NR source fabric cluster level - // config is - // dc-0 by default. + // Enable A/A in parent region and 1 child region only + // The NR source fabric cluster level config is dc-0 by default. try (ControllerClient parentControllerClient = new ControllerClient(clusterName, parentControllerUrls)) { - Assert.assertFalse( - parentControllerClient - .configureNativeReplicationForCluster( - true, - VeniceUserStoreType.BATCH_ONLY.toString(), - Optional.empty(), - Optional.of(String.join(",", parentControllerRegionName, dcNames[0], dcNames[1]))) - .isError()); Assert.assertFalse( parentControllerClient .configureActiveActiveReplicationForCluster( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/VeniceClientBasedMetadataIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/VeniceClientBasedMetadataIntegrationTest.java index 8c87144036..e341103337 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/VeniceClientBasedMetadataIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/VeniceClientBasedMetadataIntegrationTest.java @@ -52,7 +52,6 @@ public void setUp() throws Exception { .numberOfRouters(1) .replicationFactor(2) .partitionSize(100) - .minActiveReplica(1) .sslToStorageNodes(true) .sslToKafka(false) .build(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java index 460e90097f..f4e8516cef 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java @@ -345,7 +345,6 @@ public static VeniceClusterWrapper getVeniceCluster( .numberOfServers(numberOfServers) .numberOfRouters(numberOfRouters) .replicationFactor(replicationFactor) - .minActiveReplica(replicationFactor - 1) .build(); return getVeniceCluster(options); } @@ -366,7 +365,6 @@ public static VeniceClusterWrapper getVeniceCluster( .numberOfRouters(numberOfRouters) .replicationFactor(replicationFactor) .partitionSize(partitionSize) - .minActiveReplica(replicationFactor - 1) .sslToStorageNodes(sslToStorageNodes) .sslToKafka(sslToKafka) .extraProperties(extraProperties) @@ -389,7 +387,6 @@ public static VeniceClusterWrapper getVeniceCluster( .numberOfRouters(numberOfRouters) .replicationFactor(replicationFactor) .partitionSize(partitionSize) - .minActiveReplica(replicationFactor - 1) .sslToStorageNodes(sslToStorageNodes) .sslToKafka(sslToKafka) .build(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java index aa019c3d18..c67a8227a4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java @@ -21,6 +21,7 @@ public class VeniceClusterCreateOptions { private final String clusterName; private final String regionName; + private final boolean multiRegion; private final Map clusterToD2; private final Map clusterToServerD2; private final int numberOfControllers; @@ -30,7 +31,6 @@ public class VeniceClusterCreateOptions { private final int partitionSize; private final int numberOfPartitions; private final int maxNumberOfPartitions; - private final int minActiveReplica; private final long rebalanceDelayMs; private final boolean standalone; private final boolean enableAllowlist; @@ -47,6 +47,7 @@ public class VeniceClusterCreateOptions { private VeniceClusterCreateOptions(Builder builder) { this.clusterName = builder.clusterName; this.regionName = builder.regionName; + this.multiRegion = builder.multiRegion; this.clusterToD2 = builder.clusterToD2; this.clusterToServerD2 = builder.clusterToServerD2; this.numberOfControllers = builder.numberOfControllers; @@ -56,7 +57,6 @@ private VeniceClusterCreateOptions(Builder builder) { this.partitionSize = builder.partitionSize; this.numberOfPartitions = builder.numberOfPartitions; this.maxNumberOfPartitions = builder.maxNumberOfPartitions; - this.minActiveReplica = builder.minActiveReplica; this.rebalanceDelayMs = builder.rebalanceDelayMs; this.standalone = builder.standalone; this.enableAllowlist = builder.enableAllowlist; @@ -79,6 +79,10 @@ public String getRegionName() { return regionName; } + public boolean isMultiRegion() { + return multiRegion; + } + public Map getClusterToD2() { return clusterToD2; } @@ -115,10 +119,6 @@ public int getMaxNumberOfPartitions() { return maxNumberOfPartitions; } - public int getMinActiveReplica() { - return minActiveReplica; - } - public long getRebalanceDelayMs() { return rebalanceDelayMs; } @@ -176,6 +176,9 @@ public String toString() { .append("standalone:") .append(standalone) .append(", ") + .append("multiRegion:") + .append(multiRegion) + .append(", ") .append("regionName:") .append(regionName) .append(", ") @@ -203,9 +206,6 @@ public String toString() { .append("maxNumberOfPartitions:") .append(maxNumberOfPartitions) .append(", ") - .append("minActiveReplica:") - .append(minActiveReplica) - .append(", ") .append("enableAllowlist:") .append(enableAllowlist) .append(", ") @@ -247,6 +247,7 @@ public String toString() { public static class Builder { private String clusterName; private String regionName; + private boolean multiRegion = false; private Map clusterToD2 = null; private Map clusterToServerD2 = null; private int numberOfControllers = DEFAULT_NUMBER_OF_CONTROLLERS; @@ -256,7 +257,6 @@ public static class Builder { private int partitionSize = DEFAULT_PARTITION_SIZE_BYTES; private int numberOfPartitions = DEFAULT_NUMBER_OF_PARTITIONS; private int maxNumberOfPartitions = DEFAULT_MAX_NUMBER_OF_PARTITIONS; - private int minActiveReplica; private long rebalanceDelayMs = DEFAULT_DELAYED_TO_REBALANCE_MS; private boolean standalone = true; // set to false for multi-cluster private boolean enableAllowlist; @@ -264,7 +264,6 @@ public static class Builder { private boolean sslToStorageNodes = DEFAULT_SSL_TO_STORAGE_NODES; private boolean sslToKafka = DEFAULT_SSL_TO_KAFKA; private boolean forkServer; - private boolean isMinActiveReplicaSet = false; private boolean enableGrpc = false; private Properties extraProperties; private Map> kafkaClusterMap; @@ -281,6 +280,11 @@ public Builder regionName(String regionName) { return this; } + public Builder multiRegion(boolean multiRegion) { + this.multiRegion = multiRegion; + return this; + } + public Builder clusterToD2(Map clusterToD2) { this.clusterToD2 = clusterToD2; return this; @@ -326,12 +330,6 @@ public Builder maxNumberOfPartitions(int maxNumberOfPartitions) { return this; } - public Builder minActiveReplica(int minActiveReplica) { - this.minActiveReplica = minActiveReplica; - this.isMinActiveReplicaSet = true; - return this; - } - public Builder enableGrpc(boolean enableGrpc) { this.enableGrpc = enableGrpc; return this; @@ -399,9 +397,6 @@ private void verifyAndAddDefaults() { if (regionName == null || regionName.isEmpty()) { regionName = STANDALONE_REGION_NAME; } - if (!isMinActiveReplicaSet) { - minActiveReplica = replicationFactor - 1; - } if (extraProperties == null) { extraProperties = new Properties(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java index 230ab1a136..ff7ab71b01 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java @@ -220,12 +220,12 @@ static ServiceProvider generateService(VeniceClusterCreate VeniceControllerWrapper veniceControllerWrapper = ServiceFactory.getVeniceController( new VeniceControllerCreateOptions.Builder(options.getClusterName(), zkServerWrapper, pubSubBrokerWrapper) + .multiRegion(options.isMultiRegion()) .replicationFactor(options.getReplicationFactor()) .partitionSize(options.getPartitionSize()) .numberOfPartitions(options.getNumberOfPartitions()) .maxNumberOfPartitions(options.getMaxNumberOfPartitions()) .rebalanceDelayMs(options.getRebalanceDelayMs()) - .minActiveReplica(options.getMinActiveReplica()) .clusterToD2(clusterToD2) .clusterToServerD2(clusterToServerD2) .sslToKafka(options.isSslToKafka()) @@ -553,7 +553,6 @@ public VeniceControllerWrapper addVeniceController(Properties properties) { .numberOfPartitions(options.getNumberOfPartitions()) .maxNumberOfPartitions(options.getMaxNumberOfPartitions()) .rebalanceDelayMs(options.getRebalanceDelayMs()) - .minActiveReplica(options.getMinActiveReplica()) .sslToKafka(options.isSslToKafka()) .clusterToD2(clusterToD2) .clusterToServerD2(clusterToServerD2) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java index 2de94622c0..305c49b76d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java @@ -19,6 +19,7 @@ public class VeniceControllerCreateOptions { + private final boolean multiRegion; private final boolean isParent; private final boolean sslToKafka; private final boolean d2Enabled; @@ -26,7 +27,6 @@ public class VeniceControllerCreateOptions { private final int partitionSize; private final int numberOfPartitions; private final int maxNumberOfPartitions; - private final int minActiveReplica; private final long rebalanceDelayMs; private final String[] clusterNames; private final Map clusterToD2; @@ -39,13 +39,13 @@ public class VeniceControllerCreateOptions { private final String regionName; private VeniceControllerCreateOptions(Builder builder) { + multiRegion = builder.multiRegion; sslToKafka = builder.sslToKafka; d2Enabled = builder.d2Enabled; replicationFactor = builder.replicationFactor; partitionSize = builder.partitionSize; numberOfPartitions = builder.numberOfPartitions; maxNumberOfPartitions = builder.maxNumberOfPartitions; - minActiveReplica = builder.minActiveReplica; rebalanceDelayMs = builder.rebalanceDelayMs; clusterNames = builder.clusterNames; clusterToD2 = builder.clusterToD2; @@ -64,6 +64,9 @@ public String toString() { return new StringBuilder().append("regionName:") .append(regionName) .append(", ") + .append("multiRegion:") + .append(multiRegion) + .append(", ") .append("isParent:") .append(isParent) .append(", ") @@ -82,9 +85,6 @@ public String toString() { .append("maxNumberOfPartitions:") .append(maxNumberOfPartitions) .append(", ") - .append("minActiveReplica:") - .append(minActiveReplica) - .append(", ") .append("rebalanceDelayMs:") .append(rebalanceDelayMs) .append(", ") @@ -124,6 +124,10 @@ private String getAddressesOfChildControllers() { .toString(); } + public boolean isMultiRegion() { + return multiRegion; + } + public boolean isParent() { return isParent; } @@ -152,10 +156,6 @@ public int getMaxNumberOfPartitions() { return maxNumberOfPartitions; } - public int getMinActiveReplica() { - return minActiveReplica; - } - public long getRebalanceDelayMs() { return rebalanceDelayMs; } @@ -197,17 +197,16 @@ public String getRegionName() { } public static class Builder { + private boolean multiRegion = false; private final String[] clusterNames; private final ZkServerWrapper zkServer; private final PubSubBrokerWrapper kafkaBroker; private boolean sslToKafka = false; private boolean d2Enabled = false; - private boolean isMinActiveReplicaSet = false; private int replicationFactor = DEFAULT_REPLICATION_FACTOR; private int partitionSize = DEFAULT_PARTITION_SIZE_BYTES; private int numberOfPartitions = DEFAULT_NUMBER_OF_PARTITIONS; private int maxNumberOfPartitions = DEFAULT_MAX_NUMBER_OF_PARTITIONS; - private int minActiveReplica; private long rebalanceDelayMs = DEFAULT_DELAYED_TO_REBALANCE_MS; private Map clusterToD2 = null; private Map clusterToServerD2 = null; @@ -227,6 +226,11 @@ public Builder(String clusterName, ZkServerWrapper zkServer, PubSubBrokerWrapper this(new String[] { clusterName }, zkServer, kafkaBroker); } + public Builder multiRegion(boolean multiRegion) { + this.multiRegion = multiRegion; + return this; + } + public Builder sslToKafka(boolean sslToKafka) { this.sslToKafka = sslToKafka; return this; @@ -257,12 +261,6 @@ public Builder maxNumberOfPartitions(int maxNumberOfPartitions) { return this; } - public Builder minActiveReplica(int minActiveReplica) { - this.minActiveReplica = minActiveReplica; - this.isMinActiveReplicaSet = true; - return this; - } - public Builder rebalanceDelayMs(long rebalanceDelayMs) { this.rebalanceDelayMs = rebalanceDelayMs; return this; @@ -299,9 +297,6 @@ public Builder regionName(String regionName) { } private void verifyAndAddParentControllerSpecificDefaults() { - if (!isMinActiveReplicaSet) { - minActiveReplica = replicationFactor > 1 ? replicationFactor - 1 : replicationFactor; - } extraProperties.setProperty(LOCAL_REGION_NAME, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); if (!extraProperties.containsKey(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE)) { extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "true"); @@ -315,12 +310,6 @@ private void verifyAndAddParentControllerSpecificDefaults() { } } - private void verifyAndAddChildControllerSpecificDefaults() { - if (!isMinActiveReplicaSet) { - minActiveReplica = replicationFactor; - } - } - private void addDefaults() { if (extraProperties == null) { extraProperties = new Properties(); @@ -328,8 +317,6 @@ private void addDefaults() { if (childControllers != null && childControllers.length != 0) { verifyAndAddParentControllerSpecificDefaults(); - } else { - verifyAndAddChildControllerSpecificDefaults(); } if (regionName == null || regionName.isEmpty()) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java index 89aca58448..19171da3c5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java @@ -34,7 +34,7 @@ import static com.linkedin.venice.ConfigKeys.KAFKA_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.KAFKA_SECURITY_PROTOCOL; import static com.linkedin.venice.ConfigKeys.LOCAL_REGION_NAME; -import static com.linkedin.venice.ConfigKeys.MIN_ACTIVE_REPLICA; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.ConfigKeys.OFFLINE_JOB_START_TIMEOUT_MS; @@ -49,9 +49,7 @@ import static com.linkedin.venice.ConfigKeys.STORAGE_ENGINE_OVERHEAD_RATIO; import static com.linkedin.venice.ConfigKeys.SYSTEM_SCHEMA_INITIALIZATION_AT_START_TIME_ENABLED; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_DELAY_FACTOR; -import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS; import static com.linkedin.venice.SSLConfig.DEFAULT_CONTROLLER_SSL_ENABLED; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.CHILD_REGION_NAME_PREFIX; @@ -173,6 +171,7 @@ static StatefulServiceProvider generateService(VeniceCo // TODO: Validate that these configs are all still used. // TODO: Centralize default config values in a single place PropertyBuilder builder = new PropertyBuilder().put(clusterProps.toProperties()) + .put(MULTI_REGION, options.isMultiRegion()) .put(KAFKA_REPLICATION_FACTOR, 1) .put(ADMIN_TOPIC_REPLICATION_FACTOR, 1) .put(CONTROLLER_NAME, "venice-controller") // Why is this configurable? @@ -184,8 +183,6 @@ static StatefulServiceProvider generateService(VeniceCo .put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, options.getMaxNumberOfPartitions()) .put(CONTROLLER_PARENT_MODE, options.isParent()) .put(DELAY_TO_REBALANCE_MS, options.getRebalanceDelayMs()) - .put(MIN_ACTIVE_REPLICA, options.getMinActiveReplica()) - .put(TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS, 100) .put(STORAGE_ENGINE_OVERHEAD_RATIO, DEFAULT_STORAGE_ENGINE_OVERHEAD_RATIO) .put(CLUSTER_TO_D2, TestUtils.getClusterToD2String(clusterToD2)) .put(CLUSTER_TO_SERVER_D2, TestUtils.getClusterToD2String(clusterToServerD2)) @@ -204,7 +201,6 @@ static StatefulServiceProvider generateService(VeniceCo .put(CONTROLLER_ADD_VERSION_VIA_ADMIN_PROTOCOL, true) // The first cluster will always be the one to host system schemas... .put(CONTROLLER_SYSTEM_SCHEMA_CLUSTER_NAME, options.getClusterNames()[0]) - .put(TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS, false) .put(CONTROLLER_ZK_SHARED_META_SYSTEM_SCHEMA_STORE_AUTO_CREATION_ENABLED, true) .put(CONTROLLER_ZK_SHARED_DAVINCI_PUSH_STATUS_SYSTEM_SCHEMA_STORE_AUTO_CREATION_ENABLED, true) .put(PUSH_STATUS_STORE_ENABLED, true) @@ -236,7 +232,6 @@ static StatefulServiceProvider generateService(VeniceCo if (options.isParent()) { // Parent controller needs config to route per-cluster requests such as job status // This dummy parent controller won't support such requests until we make this config configurable. - // go/inclusivecode deferred(Reference will be removed when clients have migrated) fabricAllowList = extraProps.getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_WHITELIST, StringUtils.EMPTY); } else { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java index e8eef6d188..807aa17aa5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java @@ -17,20 +17,19 @@ public class VeniceMultiClusterCreateOptions { private final String regionName; + private final boolean multiRegion; private final int numberOfClusters; private final int numberOfControllers; private final int numberOfServers; private final int numberOfRouters; private final int replicationFactor; private final int partitionSize; - private final int minActiveReplica; private final long rebalanceDelayMs; private final boolean enableAllowlist; private final boolean enableAutoJoinAllowlist; private final boolean sslToStorageNodes; private final boolean sslToKafka; private final boolean randomizeClusterName; - private final boolean multiRegionSetup; private final boolean forkServer; private final Map> kafkaClusterMap; private final ZkServerWrapper zkServerWrapper; @@ -66,10 +65,6 @@ public int getPartitionSize() { return partitionSize; } - public int getMinActiveReplica() { - return minActiveReplica; - } - public long getRebalanceDelayMs() { return rebalanceDelayMs; } @@ -94,8 +89,8 @@ public boolean isRandomizeClusterName() { return randomizeClusterName; } - public boolean isMultiRegionSetup() { - return multiRegionSetup; + public boolean isMultiRegion() { + return multiRegion; } public boolean isForkServer() { @@ -128,6 +123,9 @@ public String toString() { .append("regionName:") .append(regionName) .append(", ") + .append("multiRegion:") + .append(multiRegion) + .append(", ") .append("clusters:") .append(numberOfClusters) .append(", ") @@ -149,9 +147,6 @@ public String toString() { .append("partitionSize:") .append(partitionSize) .append(", ") - .append("minActiveReplica:") - .append(minActiveReplica) - .append(", ") .append("enableAllowlist:") .append(enableAllowlist) .append(", ") @@ -167,9 +162,6 @@ public String toString() { .append("forkServer:") .append(forkServer) .append(", ") - .append("multiRegionSetup:") - .append(multiRegionSetup) - .append(", ") .append("randomizeClusterName:") .append(randomizeClusterName) .append(", ") @@ -193,6 +185,7 @@ public String toString() { private VeniceMultiClusterCreateOptions(Builder builder) { regionName = builder.regionName; + multiRegion = builder.multiRegion; numberOfClusters = builder.numberOfClusters; numberOfControllers = builder.numberOfControllers; numberOfServers = builder.numberOfServers; @@ -202,11 +195,9 @@ private VeniceMultiClusterCreateOptions(Builder builder) { enableAllowlist = builder.enableAllowlist; enableAutoJoinAllowlist = builder.enableAutoJoinAllowlist; rebalanceDelayMs = builder.rebalanceDelayMs; - minActiveReplica = builder.minActiveReplica; sslToStorageNodes = builder.sslToStorageNodes; sslToKafka = builder.sslToKafka; randomizeClusterName = builder.randomizeClusterName; - multiRegionSetup = builder.multiRegionSetup; zkServerWrapper = builder.zkServerWrapper; pubSubBrokerWrapper = builder.pubSubBrokerWrapper; childControllerProperties = builder.childControllerProperties; @@ -217,22 +208,20 @@ private VeniceMultiClusterCreateOptions(Builder builder) { public static class Builder { private String regionName; + private boolean multiRegion = false; private int numberOfClusters; private int numberOfControllers = DEFAULT_NUMBER_OF_CONTROLLERS; private int numberOfServers = DEFAULT_NUMBER_OF_SERVERS; private int numberOfRouters = DEFAULT_NUMBER_OF_ROUTERS; private int replicationFactor = DEFAULT_REPLICATION_FACTOR; private int partitionSize = DEFAULT_PARTITION_SIZE_BYTES; - private int minActiveReplica; private long rebalanceDelayMs = DEFAULT_DELAYED_TO_REBALANCE_MS; private boolean enableAllowlist = false; private boolean enableAutoJoinAllowlist = false; private boolean sslToStorageNodes = DEFAULT_SSL_TO_STORAGE_NODES; private boolean sslToKafka = DEFAULT_SSL_TO_KAFKA; private boolean randomizeClusterName = true; - private boolean multiRegionSetup = false; private boolean forkServer = false; - private boolean isMinActiveReplicaSet = false; private Map> kafkaClusterMap; private ZkServerWrapper zkServerWrapper; private PubSubBrokerWrapper pubSubBrokerWrapper; @@ -274,12 +263,6 @@ public Builder partitionSize(int partitionSize) { return this; } - public Builder minActiveReplica(int minActiveReplica) { - this.minActiveReplica = minActiveReplica; - this.isMinActiveReplicaSet = true; - return this; - } - public Builder rebalanceDelayMs(long rebalanceDelayMs) { this.rebalanceDelayMs = rebalanceDelayMs; return this; @@ -310,8 +293,8 @@ public Builder randomizeClusterName(boolean randomizeClusterName) { return this; } - public Builder multiRegionSetup(boolean multiRegionSetup) { - this.multiRegionSetup = multiRegionSetup; + public Builder multiRegion(boolean multiRegion) { + this.multiRegion = multiRegion; return this; } @@ -349,9 +332,6 @@ private void addDefaults() { if (numberOfClusters == 0) { numberOfClusters = 1; } - if (!isMinActiveReplicaSet) { - minActiveReplica = replicationFactor - 1; - } if (childControllerProperties == null) { childControllerProperties = new Properties(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java index 6f16f561a4..908a0e4c41 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java @@ -1,6 +1,5 @@ package com.linkedin.venice.integration.utils; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD; import static com.linkedin.venice.ConfigKeys.LOCAL_REGION_NAME; import static com.linkedin.venice.ConfigKeys.SYSTEM_SCHEMA_CLUSTER_NAME; @@ -87,11 +86,6 @@ static ServiceProvider generateService(VeniceMultiClu // Create controllers for multi-cluster Properties controllerProperties = options.getChildControllerProperties(); - if (options.isMultiRegionSetup() - && !controllerProperties.containsKey(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD)) { - // In multi-region setup, we don't allow batch push to each individual child region, but just parent region - controllerProperties.put(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, "false"); - } if (options.getRegionName() != null) { controllerProperties.setProperty(LOCAL_REGION_NAME, options.getRegionName()); } @@ -112,11 +106,11 @@ static ServiceProvider generateService(VeniceMultiClu pubBrokerDetails.forEach((key, value) -> controllerProperties.putIfAbsent(key, value)); VeniceControllerCreateOptions controllerCreateOptions = new VeniceControllerCreateOptions.Builder(clusterNames, zkServerWrapper, pubSubBrokerWrapper) + .multiRegion(options.isMultiRegion()) .regionName(options.getRegionName()) .replicationFactor(options.getReplicationFactor()) .partitionSize(options.getPartitionSize()) .rebalanceDelayMs(options.getRebalanceDelayMs()) - .minActiveReplica(options.getMinActiveReplica()) .clusterToD2(clusterToD2) .clusterToServerD2(clusterToServerD2) .sslToKafka(false) @@ -134,6 +128,7 @@ static ServiceProvider generateService(VeniceMultiClu pubBrokerDetails.forEach((key, value) -> extraProperties.putIfAbsent(key, value)); VeniceClusterCreateOptions.Builder vccBuilder = new VeniceClusterCreateOptions.Builder().regionName(options.getRegionName()) + .multiRegion(options.isMultiRegion()) .standalone(false) .zkServerWrapper(zkServerWrapper) .kafkaBrokerWrapper(pubSubBrokerWrapper) @@ -147,7 +142,7 @@ static ServiceProvider generateService(VeniceMultiClu .enableAllowlist(options.isEnableAllowlist()) .enableAutoJoinAllowlist(options.isEnableAutoJoinAllowlist()) .rebalanceDelayMs(options.getRebalanceDelayMs()) - .minActiveReplica(options.getMinActiveReplica()) + .sslToKafka(options.isSslToKafka()) .sslToStorageNodes(options.isSslToStorageNodes()) .extraProperties(extraProperties) .forkServer(options.isForkServer()) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java index 76a9638536..dabdfd2f43 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java @@ -1,14 +1,9 @@ package com.linkedin.venice.integration.utils; import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST; -import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.AGGREGATE_REAL_TIME_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_HYBRID; import static com.linkedin.venice.ConfigKeys.KAFKA_CLUSTER_MAP_KEY_NAME; import static com.linkedin.venice.ConfigKeys.KAFKA_CLUSTER_MAP_KEY_OTHER_URLS; import static com.linkedin.venice.ConfigKeys.KAFKA_CLUSTER_MAP_KEY_URL; @@ -82,7 +77,6 @@ static ServiceProvider generateSer */ Properties defaultParentControllerProps = new Properties(); defaultParentControllerProps.setProperty(PARTICIPANT_MESSAGE_STORE_ENABLED, "true"); - defaultParentControllerProps.setProperty(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, "false"); ZkServerWrapper zkServer = null; PubSubBrokerWrapper parentPubSubBrokerWrapper = null; @@ -123,10 +117,6 @@ static ServiceProvider generateSer Map zkServerByRegionName = new HashMap<>(childRegionName.size()); Map pubSubBrokerByRegionName = new HashMap<>(childRegionName.size()); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY, true); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, true); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_FOR_HYBRID, true); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID, true); defaultParentControllerProps .put(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, childRegionName.get(0)); defaultParentControllerProps @@ -163,8 +153,6 @@ static ServiceProvider generateSer defaultChildControllerProps.putAll(finalParentControllerProperties); defaultChildControllerProps.putAll(nativeReplicationRequiredChildControllerProps); defaultChildControllerProps.putAll(activeActiveRequiredChildControllerProps); - defaultChildControllerProps.setProperty(PARTICIPANT_MESSAGE_STORE_ENABLED, "true"); - defaultChildControllerProps.setProperty(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, "true"); final Properties finalChildControllerProperties = new Properties(); finalChildControllerProperties.putAll(defaultChildControllerProps); @@ -190,20 +178,19 @@ static ServiceProvider generateSer } additionalServerProps.putAll(pubSubBrokerProps); - VeniceMultiClusterCreateOptions.Builder builder = - new VeniceMultiClusterCreateOptions.Builder().numberOfClusters(options.getNumberOfClusters()) - .numberOfControllers(options.getNumberOfChildControllers()) - .numberOfServers(options.getNumberOfServers()) - .numberOfRouters(options.getNumberOfRouters()) - .replicationFactor(options.getReplicationFactor()) - .randomizeClusterName(false) - .multiRegionSetup(true) - .childControllerProperties(finalChildControllerProperties) - .extraProperties(additionalServerProps) - .sslToStorageNodes(options.isSslToStorageNodes()) - .sslToKafka(options.isSslToKafka()) - .forkServer(options.isForkServer()) - .kafkaClusterMap(kafkaClusterMap); + VeniceMultiClusterCreateOptions.Builder builder = new VeniceMultiClusterCreateOptions.Builder().multiRegion(true) + .numberOfClusters(options.getNumberOfClusters()) + .numberOfControllers(options.getNumberOfChildControllers()) + .numberOfServers(options.getNumberOfServers()) + .numberOfRouters(options.getNumberOfRouters()) + .replicationFactor(options.getReplicationFactor()) + .randomizeClusterName(false) + .childControllerProperties(finalChildControllerProperties) + .extraProperties(additionalServerProps) + .sslToStorageNodes(options.isSslToStorageNodes()) + .sslToKafka(options.isSslToKafka()) + .forkServer(options.isForkServer()) + .kafkaClusterMap(kafkaClusterMap); // Create multi-clusters for (int i = 0; i < options.getNumberOfRegions(); i++) { String regionName = childRegionName.get(i); @@ -226,7 +213,7 @@ static ServiceProvider generateSer VeniceControllerWrapper.PARENT_D2_CLUSTER_NAME, VeniceControllerWrapper.PARENT_D2_SERVICE_NAME); VeniceControllerCreateOptions parentControllerCreateOptions = - new VeniceControllerCreateOptions.Builder(clusterNames, zkServer, parentPubSubBrokerWrapper) + new VeniceControllerCreateOptions.Builder(clusterNames, zkServer, parentPubSubBrokerWrapper).multiRegion(true) .replicationFactor(options.getReplicationFactor()) .childControllers(childControllers) .extraProperties(finalParentControllerProperties) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/EndToEndKafkaWithSASLTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/EndToEndKafkaWithSASLTest.java index d8b865d599..325c892865 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/EndToEndKafkaWithSASLTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/EndToEndKafkaWithSASLTest.java @@ -75,7 +75,6 @@ public void setUp() throws VeniceClientException { .numberOfRouters(1) .replicationFactor(1) .partitionSize(1) - .minActiveReplica(0) .sslToStorageNodes(false) .sslToKafka(false) .zkServerWrapper(zkServer) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java index bcb7c09d0c..b0c1b8754d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java @@ -1,6 +1,6 @@ package com.linkedin.venice.restart; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.VENICE_DISCOVER_URL_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.VENICE_STORE_NAME_PROP; import static com.linkedin.venice.utils.TestUtils.generateInput; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java index 1990f6a446..236c6424a4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java @@ -2,12 +2,12 @@ import static com.linkedin.venice.CommonConfigKeys.SSL_ENABLED; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.VeniceConstants.DEFAULT_PER_ROUTER_READ_QUOTA; import static com.linkedin.venice.hadoop.VenicePushJobConstants.D2_ZK_HOSTS_PREFIX; import static com.linkedin.venice.hadoop.VenicePushJobConstants.DEFAULT_KEY_FIELD_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.DEFAULT_VALUE_FIELD_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.KEY_FIELD_PROP; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.PARENT_CONTROLLER_REGION_NAME; import static com.linkedin.venice.hadoop.VenicePushJobConstants.SOURCE_GRID_FABRIC; import static com.linkedin.venice.hadoop.VenicePushJobConstants.VALUE_FIELD_PROP; diff --git a/internal/venice-test-common/src/integrationtest/java/com/linkedin/venice/fastclient/grpc/VeniceGrpcEndToEndTest.java b/internal/venice-test-common/src/integrationtest/java/com/linkedin/venice/fastclient/grpc/VeniceGrpcEndToEndTest.java index 46514c7ea7..fe8b54b5e5 100644 --- a/internal/venice-test-common/src/integrationtest/java/com/linkedin/venice/fastclient/grpc/VeniceGrpcEndToEndTest.java +++ b/internal/venice-test-common/src/integrationtest/java/com/linkedin/venice/fastclient/grpc/VeniceGrpcEndToEndTest.java @@ -75,7 +75,6 @@ public void setUp() throws Exception { .partitionSize(1000) .numberOfPartitions(2) .maxNumberOfPartitions(5) - .minActiveReplica(1) .numberOfRouters(1) .numberOfServers(2) .sslToStorageNodes(true) diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java index a1630602e3..644b7582f4 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestWriteUtils.java @@ -1,5 +1,6 @@ package com.linkedin.venice.utils; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.CONTROLLER_REQUEST_RETRY_ATTEMPTS; import static com.linkedin.venice.hadoop.VenicePushJobConstants.D2_ZK_HOSTS_PREFIX; import static com.linkedin.venice.hadoop.VenicePushJobConstants.DEFAULT_KEY_FIELD_PROP; @@ -7,7 +8,6 @@ import static com.linkedin.venice.hadoop.VenicePushJobConstants.INPUT_PATH_PROP; import static com.linkedin.venice.hadoop.VenicePushJobConstants.KEY_INPUT_FILE_DATA_SIZE; import static com.linkedin.venice.hadoop.VenicePushJobConstants.KEY_ZSTD_COMPRESSION_DICTIONARY; -import static com.linkedin.venice.hadoop.VenicePushJobConstants.MULTI_REGION; import static com.linkedin.venice.hadoop.VenicePushJobConstants.PARENT_CONTROLLER_REGION_NAME; import static com.linkedin.venice.hadoop.VenicePushJobConstants.POLL_JOB_STATUS_INTERVAL_MS; import static com.linkedin.venice.hadoop.VenicePushJobConstants.PUSH_JOB_STATUS_UPLOAD_ENABLE; diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java index 0e3ff596d1..991eab19f2 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java @@ -800,18 +800,6 @@ void updateRoutersClusterConfig( */ List getClustersLeaderOf(); - /** - * Enable/disable native replications for certain stores (batch only, hybrid only, incremental push, hybrid or incremental push, - * all) in a cluster. If storeName is not empty, only the specified store might be updated. - */ - void configureNativeReplication( - String cluster, - VeniceUserStoreType storeType, - Optional storeName, - boolean enableNativeReplicationForCluster, - Optional newSourceFabric, - Optional regionsFilter); - /** * Enable/disable active active replications for certain stores (batch only, hybrid only, incremental push, hybrid or incremental push, * all) in a cluster. If storeName is not empty, only the specified store might be updated. diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/HelixVeniceClusterResources.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/HelixVeniceClusterResources.java index efc2927265..0a1182c9be 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/HelixVeniceClusterResources.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/HelixVeniceClusterResources.java @@ -33,8 +33,6 @@ import com.linkedin.venice.utils.locks.AutoCloseableLock; import com.linkedin.venice.utils.locks.ClusterLockManager; import io.tehuti.metrics.MetricsRepository; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; @@ -163,7 +161,7 @@ public HelixVeniceClusterResources( realTimeTopicSwitcher, clusterLockManager, aggregateRealTimeSourceKafkaUrl, - getActiveActiveRealTimeSourceKafkaURLs(config), + config.getActiveActiveRealTimeSourceKafkaURLs(), helixAdminClient, config, admin.getPushStatusStoreReader(), @@ -209,22 +207,6 @@ public HelixVeniceClusterResources( new StoragePersonaRepository(clusterName, this.storeMetadataRepository, adapterSerializer, zkClient); } - private List getActiveActiveRealTimeSourceKafkaURLs(VeniceControllerClusterConfig config) { - List kafkaURLs = new ArrayList<>(config.getActiveActiveRealTimeSourceFabrics().size()); - for (String fabric: config.getActiveActiveRealTimeSourceFabrics()) { - String kafkaURL = config.getChildDataCenterKafkaUrlMap().get(fabric); - if (kafkaURL == null) { - throw new VeniceException( - String.format( - "No A/A source Kafka URL found for fabric %s in %s", - fabric, - config.getChildDataCenterKafkaUrlMap())); - } - kafkaURLs.add(kafkaURL); - } - return Collections.unmodifiableList(kafkaURLs); - } - /** * This function is used to repair all the stores with replication factor: 0. * And these stores will be updated to use the replication factor configured in cluster level. diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java index 4ad4dbd9b6..5705a9e27a 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java @@ -2,17 +2,14 @@ import static com.linkedin.venice.CommonConfigKeys.SSL_FACTORY_CLASS_NAME; import static com.linkedin.venice.ConfigConstants.DEFAULT_PUSH_STATUS_STORE_HEARTBEAT_EXPIRATION_TIME_IN_SECONDS; -import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER; import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST; import static com.linkedin.venice.ConfigKeys.ADMIN_CHECK_READ_METHOD_FOR_KAFKA; import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_MAX_WORKER_THREAD_POOL_SIZE; -import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_TIMEOUT_MINUTES; import static com.linkedin.venice.ConfigKeys.ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_HOSTNAME; import static com.linkedin.venice.ConfigKeys.ADMIN_PORT; import static com.linkedin.venice.ConfigKeys.ADMIN_SECURE_PORT; -import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.AGGREGATE_REAL_TIME_SOURCE_REGION; @@ -22,7 +19,6 @@ import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_D2_SERVICE_NAME; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_URL_PREFIX; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_WHITELIST; -import static com.linkedin.venice.ConfigKeys.CHILD_CONTROLLER_ADMIN_TOPIC_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; @@ -48,7 +44,6 @@ import static com.linkedin.venice.ConfigKeys.CONTROLLER_DISABLE_PARENT_REQUEST_TOPIC_FOR_STREAM_PUSHES; import static com.linkedin.venice.ConfigKeys.CONTROLLER_DISABLE_PARENT_TOPIC_TRUNCATION_UPON_COMPLETION; import static com.linkedin.venice.ConfigKeys.CONTROLLER_EARLY_DELETE_BACKUP_ENABLED; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD; import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_DISABLED_REPLICA_ENABLED; import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENFORCE_SSL; import static com.linkedin.venice.ConfigKeys.CONTROLLER_HAAS_SUPER_CLUSTER_NAME; @@ -91,15 +86,10 @@ import static com.linkedin.venice.ConfigKeys.DEPRECATED_TOPIC_MAX_RETENTION_MS; import static com.linkedin.venice.ConfigKeys.DEPRECATED_TOPIC_RETENTION_MS; import static com.linkedin.venice.ConfigKeys.EMERGENCY_SOURCE_REGION; -import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_HYBRID_PUSH_SSL_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.ENABLE_HYBRID_PUSH_SSL_WHITELIST; import static com.linkedin.venice.ConfigKeys.ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_HYBRID; import static com.linkedin.venice.ConfigKeys.ENABLE_OFFLINE_PUSH_SSL_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.ENABLE_OFFLINE_PUSH_SSL_WHITELIST; import static com.linkedin.venice.ConfigKeys.ENABLE_PARTIAL_UPDATE_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES; @@ -112,27 +102,23 @@ import static com.linkedin.venice.ConfigKeys.HELIX_REBALANCE_ALG; import static com.linkedin.venice.ConfigKeys.HELIX_SEND_MESSAGE_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.IDENTITY_PARSER_CLASS; -import static com.linkedin.venice.ConfigKeys.KAFKA_ADMIN_CLASS; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; import static com.linkedin.venice.ConfigKeys.KAFKA_LOG_COMPACTION_FOR_HYBRID_STORES; import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_IN_SYNC_REPLICAS; import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_IN_SYNC_REPLICAS_ADMIN_TOPICS; import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_IN_SYNC_REPLICAS_RT_TOPICS; -import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_LOG_COMPACTION_LAG_MS; import static com.linkedin.venice.ConfigKeys.KAFKA_OVER_SSL; -import static com.linkedin.venice.ConfigKeys.KAFKA_READ_ONLY_ADMIN_CLASS; import static com.linkedin.venice.ConfigKeys.KAFKA_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.KAFKA_REPLICATION_FACTOR_RT_TOPICS; import static com.linkedin.venice.ConfigKeys.KAFKA_SECURITY_PROTOCOL; -import static com.linkedin.venice.ConfigKeys.KAFKA_WRITE_ONLY_ADMIN_CLASS; import static com.linkedin.venice.ConfigKeys.KME_REGISTRATION_FROM_MESSAGE_HEADER_ENABLED; import static com.linkedin.venice.ConfigKeys.LEAKED_PUSH_STATUS_CLEAN_UP_SERVICE_SLEEP_INTERVAL_MS; import static com.linkedin.venice.ConfigKeys.LEAKED_RESOURCE_ALLOWED_LINGER_TIME_MS; import static com.linkedin.venice.ConfigKeys.META_STORE_WRITER_CLOSE_CONCURRENCY; import static com.linkedin.venice.ConfigKeys.META_STORE_WRITER_CLOSE_TIMEOUT_MS; -import static com.linkedin.venice.ConfigKeys.MIN_ACTIVE_REPLICA; import static com.linkedin.venice.ConfigKeys.MIN_NUMBER_OF_STORE_VERSIONS_TO_PRESERVE; import static com.linkedin.venice.ConfigKeys.MIN_NUMBER_OF_UNUSED_KAFKA_TOPICS_TO_PRESERVE; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_WHITELIST; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC; @@ -162,11 +148,7 @@ import static com.linkedin.venice.ConfigKeys.SYSTEM_SCHEMA_INITIALIZATION_AT_START_TIME_ENABLED; import static com.linkedin.venice.ConfigKeys.TERMINAL_STATE_TOPIC_CHECK_DELAY_MS; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_DELAY_FACTOR; -import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_DELETION_STATUS_POLL_INTERVAL_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_MANAGER_KAFKA_OPERATION_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.UNREGISTER_METRIC_FOR_DELETED_STORE_ENABLED; import static com.linkedin.venice.ConfigKeys.USE_DA_VINCI_SPECIFIC_EXECUTION_STATUS_FOR_ERROR; import static com.linkedin.venice.ConfigKeys.USE_PUSH_STATUS_STORE_FOR_INCREMENTAL_PUSH; @@ -175,9 +157,7 @@ import static com.linkedin.venice.SSLConfig.DEFAULT_CONTROLLER_SSL_ENABLED; import static com.linkedin.venice.VeniceConstants.DEFAULT_PER_ROUTER_READ_QUOTA; import static com.linkedin.venice.VeniceConstants.DEFAULT_SSL_FACTORY_CLASS_NAME; -import static com.linkedin.venice.pubsub.PubSubConstants.DEFAULT_KAFKA_MIN_LOG_COMPACTION_LAG_MS; import static com.linkedin.venice.pubsub.PubSubConstants.DEFAULT_KAFKA_REPLICATION_FACTOR; -import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_TOPIC_DELETION_STATUS_POLL_INTERVAL_MS_DEFAULT_VALUE; import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_TOPIC_MANAGER_METADATA_FETCHER_CONSUMER_POOL_SIZE_DEFAULT_VALUE; import static com.linkedin.venice.utils.ByteUtils.BYTES_PER_MB; import static com.linkedin.venice.utils.ByteUtils.generateHumanReadableByteCountString; @@ -194,7 +174,6 @@ import com.linkedin.venice.meta.RoutingStrategy; import com.linkedin.venice.pubsub.PubSubAdminAdapterFactory; import com.linkedin.venice.pubsub.PubSubClientsFactory; -import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapter; import com.linkedin.venice.pushmonitor.LeakedPushStatusCleanUpService; import com.linkedin.venice.pushmonitor.PushMonitorType; import com.linkedin.venice.status.BatchJobHeartbeatConfigs; @@ -216,7 +195,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.commons.lang.StringUtils; import org.apache.helix.controller.rebalancer.strategy.CrushRebalanceStrategy; import org.apache.kafka.common.protocol.SecurityProtocol; import org.apache.logging.log4j.LogManager; @@ -241,8 +219,8 @@ public class VeniceControllerClusterConfig { // Name of the Helix cluster for controllers private final String controllerClusterName; private final String controllerClusterZkAddress; + private final boolean multiRegion; private final boolean parent; - private final List childDataCenterAllowlist; private final Map childDataCenterControllerUrlMap; private final String d2ServiceName; private final String clusterDiscoveryD2ServiceName; @@ -254,11 +232,9 @@ public class VeniceControllerClusterConfig { // heartbeat. private final Duration batchJobHeartbeatTimeout; private final Duration batchJobHeartbeatInitialBufferTime; - private final long adminConsumptionTimeoutMinute; private final long adminConsumptionCycleTimeoutMs; private final int adminConsumptionMaxWorkerThreadPoolSize; private final double storageEngineOverheadRatio; - private final long topicCreationThrottlingTimeWindowMs; private final long deprecatedJobTopicRetentionMs; private final long fatalDataValidationFailureRetentionMs; @@ -268,7 +244,6 @@ public class VeniceControllerClusterConfig { private final long disabledReplicaEnablerServiceIntervalMs; private final int topicCleanupDelayFactor; - private final int topicManagerKafkaOperationTimeOutMs; private final int topicManagerMetadataFetcherConsumerPoolSize; private final int topicManagerMetadataFetcherThreadPoolSize; private final int minNumberOfUnusedKafkaTopicsToPreserve; @@ -277,21 +252,14 @@ public class VeniceControllerClusterConfig { private final String pushJobStatusStoreClusterName; private final boolean participantMessageStoreEnabled; private final String systemSchemaClusterName; - private final int topicDeletionStatusPollIntervalMs; private final boolean adminHelixMessagingChannelEnabled; private final boolean isControllerClusterLeaderHAAS; private final boolean isVeniceClusterLeaderHAAS; private final String controllerHAASSuperClusterName; private final boolean earlyDeleteBackUpEnabled; - private final boolean sendConcurrentTopicDeleteRequestsEnabled; - private final boolean enableBatchPushFromAdminInChildController; private final boolean adminCheckReadMethodForKafka; - private final String kafkaAdminClass; - private final String kafkaWriteOnlyClass; - private final String kafkaReadOnlyClass; private final Map childDataCenterKafkaUrlMap; - private final boolean activeActiveEnabledOnController; - private final Set activeActiveRealTimeSourceFabrics; + private final List activeActiveRealTimeSourceKafkaURLs; private final String nativeReplicationSourceFabric; private final int errorPartitionAutoResetLimit; private final long errorPartitionProcessingCycleDelay; @@ -375,8 +343,6 @@ public class VeniceControllerClusterConfig { private final boolean allowClusterWipe; - private final boolean childControllerAdminTopicConsumptionEnabled; - private final boolean concurrentInitRoutinesEnabled; private final boolean controllerInAzureFabric; @@ -452,35 +418,6 @@ public class VeniceControllerClusterConfig { private final boolean enableNearlinePushSSLAllowlist; private final List pushSSLAllowlist; - /** - * TODO: the follower 3 cluster level configs remains in the code base in case the new cluster level configs are not - * working as expected. Once the new cluster level configs for native replication have been tested in prod, retire - * the following configs. - */ - /** - * When this option is enabled, all new batch-only store versions created will have native replication enabled so long - * as the store has leader follower also enabled. - */ - private final boolean nativeReplicationEnabledForBatchOnly; - - /** - * When this option is enabled, all new hybrid store versions created will have native replication enabled so long - * as the store has leader follower also enabled. - */ - private final boolean nativeReplicationEnabledForHybrid; - - /** - * When this option is enabled, all new batch-only stores will have native replication enabled in store config so long - * as the store has leader follower also enabled. - */ - private final boolean nativeReplicationEnabledAsDefaultForBatchOnly; - - /** - * When this option is enabled, all new hybrid stores will have native replication enabled in store config so long - * as the store has leader follower also enabled. - */ - private final boolean nativeReplicationEnabledAsDefaultForHybrid; - private final String nativeReplicationSourceFabricAsDefaultForBatchOnly; private final String nativeReplicationSourceFabricAsDefaultForHybrid; @@ -493,12 +430,6 @@ public class VeniceControllerClusterConfig { private final boolean enablePartialUpdateForHybridActiveActiveUserStores; private final boolean enablePartialUpdateForHybridNonActiveActiveUserStores; - /** - * When this option is enabled, all new batch-only stores will have active-active replication enabled in store config so long - * as the store has leader follower also enabled. - */ - private final boolean activeActiveReplicationEnabledAsDefaultForBatchOnly; - /** * When this option is enabled, all new hybrid stores will have active-active replication enabled in store config so long * as the store has leader follower also enabled. @@ -514,12 +445,6 @@ public class VeniceControllerClusterConfig { * After server disconnecting for delayToRebalanceMS, helix would trigger the re-balance immediately. */ private final long delayToRebalanceMS; - /** - * If the replica count smaller than minActiveReplica, helix would trigger the re-balance immediately. - * This config is deprecated. Replication factor config is moved to store/version-level config - */ - @Deprecated - private final int minActiveReplica; /** * kafka Bootstrap Urls . IF there is more than one url, they are separated by commas @@ -538,7 +463,6 @@ public class VeniceControllerClusterConfig { private final Optional minInSyncReplicasRealTimeTopics; private final Optional minInSyncReplicasAdminTopics; private final boolean kafkaLogCompactionForHybridStores; - private final long kafkaMinLogCompactionLagInMs; /** * Alg used by helix to decide the mapping between replicas and nodes. @@ -576,7 +500,7 @@ public class VeniceControllerClusterConfig { private final boolean errorLeaderReplicaFailOverEnabled; - private final String childDatacenters; + private final Set childDatacenters; public VeniceControllerClusterConfig(VeniceProperties props) { this.props = props; @@ -589,8 +513,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.minInSyncReplicasRealTimeTopics = props.getOptionalInt(KAFKA_MIN_IN_SYNC_REPLICAS_RT_TOPICS); this.minInSyncReplicasAdminTopics = props.getOptionalInt(KAFKA_MIN_IN_SYNC_REPLICAS_ADMIN_TOPICS); this.kafkaLogCompactionForHybridStores = props.getBoolean(KAFKA_LOG_COMPACTION_FOR_HYBRID_STORES, true); - this.kafkaMinLogCompactionLagInMs = - props.getLong(KAFKA_MIN_LOG_COMPACTION_LAG_MS, DEFAULT_KAFKA_MIN_LOG_COMPACTION_LAG_MS); this.replicationFactor = props.getInt(DEFAULT_REPLICA_FACTOR); this.minNumberOfPartitions = props.getInt(DEFAULT_NUMBER_OF_PARTITION); this.minNumberOfPartitionsForHybrid = props.getInt(DEFAULT_NUMBER_OF_PARTITION_FOR_HYBRID, minNumberOfPartitions); @@ -604,9 +526,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.offLineJobWaitTimeInMilliseconds = props.getLong(OFFLINE_JOB_START_TIMEOUT_MS, 120000); // By default, delayed rebalance is disabled. this.delayToRebalanceMS = props.getLong(DELAY_TO_REBALANCE_MS, 0); - // By default, the min active replica is replica factor minus one, which means if more than one server failed, - // helix would trigger re-balance immediately. - this.minActiveReplica = props.getInt(MIN_ACTIVE_REPLICA, replicationFactor - 1); if (props.containsKey(PERSISTENCE_TYPE)) { this.persistenceType = PersistenceType.valueOf(props.getString(PERSISTENCE_TYPE)); } else { @@ -628,18 +547,10 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.routingStrategy = RoutingStrategy.CONSISTENT_HASH; } - this.nativeReplicationEnabledForBatchOnly = props.getBoolean(ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY, false); - this.nativeReplicationEnabledAsDefaultForBatchOnly = - props.getBoolean(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, false); - this.nativeReplicationEnabledForHybrid = props.getBoolean(ENABLE_NATIVE_REPLICATION_FOR_HYBRID, false); - this.nativeReplicationEnabledAsDefaultForHybrid = - props.getBoolean(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID, false); this.nativeReplicationSourceFabricAsDefaultForBatchOnly = props.getString(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, ""); this.nativeReplicationSourceFabricAsDefaultForHybrid = props.getString(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES, ""); - this.activeActiveReplicationEnabledAsDefaultForBatchOnly = - props.getBoolean(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE, false); this.activeActiveReplicationEnabledAsDefaultForHybrid = props.getBoolean(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, false); this.controllerSchemaValidationEnabled = props.getBoolean(CONTROLLER_SCHEMA_VALIDATION_ENABLED, true); @@ -704,11 +615,13 @@ public VeniceControllerClusterConfig(VeniceProperties props) { "Default max record size must be at least " + generateHumanReadableByteCountString(BYTES_PER_MB)); } this.replicationMetadataVersion = props.getInt(REPLICATION_METADATA_VERSION, 1); - this.childDatacenters = props.getString(CHILD_CLUSTER_ALLOWLIST); + // go/inclusivecode deferred(Will be replaced when clients have migrated) + this.childDatacenters = Utils.parseCommaSeparatedStringToSet( + props.getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_WHITELIST, (String) null)); this.errorLeaderReplicaFailOverEnabled = props.getBoolean(FORCE_LEADER_ERROR_REPLICA_FAIL_OVER_ENABLED, true); this.adminPort = props.getInt(ADMIN_PORT); - this.adminHostname = props.getString(ADMIN_HOSTNAME, () -> Utils.getHostName()); + this.adminHostname = props.getString(ADMIN_HOSTNAME, Utils::getHostName); this.adminSecurePort = props.getInt(ADMIN_SECURE_PORT); /** * Override the config to false if the "Read" method check is not working as expected. @@ -717,54 +630,87 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.controllerClusterName = props.getString(CONTROLLER_CLUSTER, "venice-controllers"); this.controllerClusterReplica = props.getInt(CONTROLLER_CLUSTER_REPLICA, 3); this.controllerClusterZkAddress = props.getString(CONTROLLER_CLUSTER_ZK_ADDRESSS, getZkAddress()); - this.topicCreationThrottlingTimeWindowMs = - props.getLong(TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS, 10 * Time.MS_PER_SECOND); this.parent = props.getBoolean(CONTROLLER_PARENT_MODE, false); - this.activeActiveEnabledOnController = props.getBoolean(ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER, false); - this.activeActiveRealTimeSourceFabrics = - Utils.parseCommaSeparatedStringToSet(props.getString(ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, "")); - validateActiveActiveConfigs(); - // go/inclusivecode deferred(Will be replaced when clients have migrated) - String dataCenterAllowlist = props.getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_WHITELIST); - if (dataCenterAllowlist.isEmpty()) { + if (childDatacenters.isEmpty()) { this.childDataCenterControllerUrlMap = Collections.emptyMap(); this.childDataCenterControllerD2Map = Collections.emptyMap(); - this.childDataCenterAllowlist = Collections.emptyList(); } else { - this.childDataCenterControllerUrlMap = parseClusterMap(props, dataCenterAllowlist); - this.childDataCenterControllerD2Map = parseClusterMap(props, dataCenterAllowlist, true); - this.childDataCenterAllowlist = Arrays.asList(dataCenterAllowlist.split(LIST_SEPARATOR)); + this.childDataCenterControllerUrlMap = parseClusterMap(props, childDatacenters); + this.childDataCenterControllerD2Map = parseClusterMap(props, childDatacenters, true); } + + Set nativeReplicationSourceFabricAllowlist = Utils.parseCommaSeparatedStringToSet( + props.getStringWithAlternative( + NATIVE_REPLICATION_FABRIC_ALLOWLIST, + // go/inclusivecode deferred(will be removed once all configs have migrated) + NATIVE_REPLICATION_FABRIC_WHITELIST, + (String) null)); + this.d2ServiceName = childDataCenterControllerD2Map.isEmpty() ? null : props.getString(CHILD_CLUSTER_D2_SERVICE_NAME); if (this.parent) { if (childDataCenterControllerUrlMap.isEmpty() && childDataCenterControllerD2Map.isEmpty()) { throw new VeniceException("child controller list can not be empty"); } - String parentFabricList = props.getString(PARENT_KAFKA_CLUSTER_FABRIC_LIST, ""); - this.parentFabrics = Utils.parseCommaSeparatedStringToSet(parentFabricList); - String nativeReplicationSourceFabricAllowlist = props.getStringWithAlternative( - NATIVE_REPLICATION_FABRIC_ALLOWLIST, - // go/inclusivecode deferred(will be removed once all configs have migrated) - NATIVE_REPLICATION_FABRIC_WHITELIST, - dataCenterAllowlist); + this.parentFabrics = + Utils.parseCommaSeparatedStringToSet(props.getString(PARENT_KAFKA_CLUSTER_FABRIC_LIST, (String) null)); this.childDataCenterKafkaUrlMap = parseChildDataCenterKafkaUrl(props, nativeReplicationSourceFabricAllowlist); } else { this.parentFabrics = Collections.emptySet(); - String nativeReplicationSourceFabricAllowlist = props.getStringWithAlternative( - NATIVE_REPLICATION_FABRIC_ALLOWLIST, - // go/inclusivecode deferred(will be removed once all configs have migrated) - NATIVE_REPLICATION_FABRIC_WHITELIST, - ""); - if (nativeReplicationSourceFabricAllowlist == null || nativeReplicationSourceFabricAllowlist.length() == 0) { + if (nativeReplicationSourceFabricAllowlist.isEmpty()) { this.childDataCenterKafkaUrlMap = Collections.emptyMap(); } else { this.childDataCenterKafkaUrlMap = parseChildDataCenterKafkaUrl(props, nativeReplicationSourceFabricAllowlist); } } + + Set activeActiveRealTimeSourceFabrics = Utils + .parseCommaSeparatedStringToSet(props.getString(ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, (String) null)); + + for (String fabric: activeActiveRealTimeSourceFabrics) { + if (!childDataCenterKafkaUrlMap.containsKey(fabric)) { + throw new VeniceException( + String.format("No A/A source Kafka URL found for fabric %s in %s", fabric, childDataCenterKafkaUrlMap)); + } + } + + this.activeActiveRealTimeSourceKafkaURLs = activeActiveRealTimeSourceFabrics.stream() + .map(childDataCenterKafkaUrlMap::get) + .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + this.nativeReplicationSourceFabric = props.getString(NATIVE_REPLICATION_SOURCE_FABRIC, ""); + + if (props.containsKey(MULTI_REGION)) { + this.multiRegion = props.getBoolean(MULTI_REGION); + } else { + LOGGER.warn("Config {} is not set. Inferring multi-region setup from other configs.", MULTI_REGION); + + /** + * Historically, {@link MULTI_REGION} was not a supported config. It was handled on a case-by-case basis by + * carefully setting feature configs. While this works for ramping new features, it makes it hard to remove the + * feature flags once the feature is fully ramped. Ideally, this should be a mandatory config, but that would break + * backward compatibility and hence, we infer the multi-region setup through the presence of other configs. + */ + if (parent) { + // Parent controllers only run in multi-region mode + LOGGER.info("Inferring multi-region mode since this is the parent controller."); + this.multiRegion = true; + } else if (!childDatacenters.isEmpty()) { + LOGGER.info("Inferring multi-region mode since there are child regions configured."); + this.multiRegion = true; + } else if (!childDataCenterKafkaUrlMap.isEmpty()) { + LOGGER.info("Inferring multi-region mode since PubSub URLs are set for child regions."); + // This is implicitly a mandatory config for child controllers in multi-region mode since Admin topic remote + // consumption is the only supported mode in multi-region setup, and that needs PubSub URLs to be set. + this.multiRegion = true; + } else { + LOGGER.info("Inferring single-region mode."); + this.multiRegion = false; + } + } + this.parentControllerWaitingTimeForConsumptionMs = props.getInt(PARENT_CONTROLLER_WAITING_TIME_FOR_CONSUMPTION_MS, 30 * Time.MS_PER_SECOND); this.batchJobHeartbeatStoreCluster = props.getString( @@ -781,7 +727,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { props.getLong( BatchJobHeartbeatConfigs.HEARTBEAT_CONTROLLER_INITIAL_DELAY_CONFIG.getConfigName(), BatchJobHeartbeatConfigs.HEARTBEAT_CONTROLLER_INITIAL_DELAY_CONFIG.getDefaultValue())); - this.adminConsumptionTimeoutMinute = props.getLong(ADMIN_CONSUMPTION_TIMEOUT_MINUTES, TimeUnit.DAYS.toMinutes(5)); this.adminConsumptionCycleTimeoutMs = props.getLong(ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS, TimeUnit.MINUTES.toMillis(30)); // A value of one will result in a bad message for one store to block the admin message consumption of other stores. @@ -813,8 +758,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.disabledReplicaEnablerServiceIntervalMs = props.getLong(CONTROLLER_DISABLED_REPLICA_ENABLER_INTERVAL_MS, TimeUnit.HOURS.toMillis(16)); - this.topicManagerKafkaOperationTimeOutMs = - props.getInt(TOPIC_MANAGER_KAFKA_OPERATION_TIMEOUT_MS, 30 * Time.MS_PER_SECOND); this.topicManagerMetadataFetcherConsumerPoolSize = props.getInt( PUBSUB_TOPIC_MANAGER_METADATA_FETCHER_CONSUMER_POOL_SIZE, PUBSUB_TOPIC_MANAGER_METADATA_FETCHER_CONSUMER_POOL_SIZE_DEFAULT_VALUE); @@ -840,8 +783,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { } this.systemSchemaClusterName = props.getString(CONTROLLER_SYSTEM_SCHEMA_CLUSTER_NAME, ""); this.earlyDeleteBackUpEnabled = props.getBoolean(CONTROLLER_EARLY_DELETE_BACKUP_ENABLED, true); - this.topicDeletionStatusPollIntervalMs = props - .getInt(TOPIC_DELETION_STATUS_POLL_INTERVAL_MS, PUBSUB_TOPIC_DELETION_STATUS_POLL_INTERVAL_MS_DEFAULT_VALUE); // 2s this.isControllerClusterLeaderHAAS = props.getBoolean(CONTROLLER_CLUSTER_LEADER_HAAS, false); this.isVeniceClusterLeaderHAAS = props.getBoolean(VENICE_STORAGE_CLUSTER_LEADER_HAAS, false); this.controllerHAASSuperClusterName = props.getString(CONTROLLER_HAAS_SUPER_CLUSTER_NAME, ""); @@ -850,13 +791,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { CONTROLLER_HAAS_SUPER_CLUSTER_NAME + " is required for " + CONTROLLER_CLUSTER_LEADER_HAAS + " or " + VENICE_STORAGE_CLUSTER_LEADER_HAAS + " to be set to true"); } - this.sendConcurrentTopicDeleteRequestsEnabled = - props.getBoolean(TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS, false); - this.enableBatchPushFromAdminInChildController = - props.getBoolean(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, true); - this.kafkaAdminClass = props.getString(KAFKA_ADMIN_CLASS, ApacheKafkaAdminAdapter.class.getName()); - this.kafkaWriteOnlyClass = props.getString(KAFKA_WRITE_ONLY_ADMIN_CLASS, kafkaAdminClass); - this.kafkaReadOnlyClass = props.getString(KAFKA_READ_ONLY_ADMIN_CLASS, kafkaAdminClass); this.errorPartitionAutoResetLimit = props.getInt(ERROR_PARTITION_AUTO_RESET_LIMIT, 0); this.errorPartitionProcessingCycleDelay = props.getLong(ERROR_PARTITION_PROCESSING_CYCLE_DELAY, 5 * Time.MS_PER_MINUTE); @@ -901,7 +835,7 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.regionName = RegionUtils.getLocalRegionName(props, parent); LOGGER.info("Final region name for this node: {}", this.regionName); this.disabledRoutes = parseControllerRoutes(props, CONTROLLER_DISABLED_ROUTES, Collections.emptyList()); - this.adminTopicRemoteConsumptionEnabled = props.getBoolean(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, false); + this.adminTopicRemoteConsumptionEnabled = !this.parent && this.multiRegion; if (adminTopicRemoteConsumptionEnabled && childDataCenterKafkaUrlMap.isEmpty()) { throw new VeniceException("Admin topic remote consumption is enabled but Kafka url map is empty"); } @@ -916,8 +850,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { this.metaStoreWriterCloseConcurrency = props.getInt(META_STORE_WRITER_CLOSE_CONCURRENCY, -1); this.emergencySourceRegion = props.getString(EMERGENCY_SOURCE_REGION, ""); this.allowClusterWipe = props.getBoolean(ALLOW_CLUSTER_WIPE, false); - this.childControllerAdminTopicConsumptionEnabled = - props.getBoolean(CHILD_CONTROLLER_ADMIN_TOPIC_CONSUMPTION_ENABLED, true); this.concurrentInitRoutinesEnabled = props.getBoolean(CONCURRENT_INIT_ROUTINES_ENABLED, false); this.controllerInAzureFabric = props.getBoolean(CONTROLLER_IN_AZURE_FABRIC, false); this.unregisterMetricForDeletedStoreEnabled = props.getBoolean(UNREGISTER_METRIC_FOR_DELETED_STORE_ENABLED, false); @@ -956,31 +888,6 @@ public VeniceControllerClusterConfig(VeniceProperties props) { props.getInt(CONTROLLER_DANGLING_TOPIC_OCCURRENCE_THRESHOLD_FOR_CLEANUP, 3); } - private void validateActiveActiveConfigs() { - if (this.activeActiveEnabledOnController && this.activeActiveRealTimeSourceFabrics.isEmpty()) { - throw new VeniceException( - String.format( - "The config %s cannot be empty when the child controller has A/A enabled " + "(%s == true).", - ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, - ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER)); - - } else if (this.activeActiveEnabledOnController) { - LOGGER.info( - "A/A is enabled on a child controller and {} == {}", - ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, - this.activeActiveRealTimeSourceFabrics); - } else { - LOGGER.info( - "A/A is not enabled on child controller. {}", - !this.activeActiveRealTimeSourceFabrics.isEmpty() - ? String.format( - " But %s is still set to %s.", - ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, - this.activeActiveRealTimeSourceFabrics) - : ""); - } - } - public VeniceProperties getProps() { return props; } @@ -1080,11 +987,6 @@ public long getDelayToRebalanceMS() { return delayToRebalanceMS; } - @Deprecated - public int getMinActiveReplica() { - return minActiveReplica; - } - /** * @return kafka Bootstrap Urls. If there is more than one url, they are separated by commas. */ @@ -1152,10 +1054,6 @@ public int getAdminTopicReplicationFactor() { return adminTopicReplicationFactor; } - public PushMonitorType getPushMonitorType() { - return pushMonitorType; - } - public Optional getMinInSyncReplicas() { return minInSyncReplicas; } @@ -1172,30 +1070,6 @@ public boolean isKafkaLogCompactionForHybridStoresEnabled() { return kafkaLogCompactionForHybridStores; } - public long getKafkaMinLogCompactionLagInMs() { - return kafkaMinLogCompactionLagInMs; - } - - public boolean isNativeReplicationEnabledForBatchOnly() { - return nativeReplicationEnabledForBatchOnly; - } - - public boolean isNativeReplicationEnabledAsDefaultForBatchOnly() { - return nativeReplicationEnabledAsDefaultForBatchOnly; - } - - public boolean isNativeReplicationEnabledForHybrid() { - return nativeReplicationEnabledForHybrid; - } - - public boolean isNativeReplicationEnabledAsDefaultForHybrid() { - return nativeReplicationEnabledAsDefaultForHybrid; - } - - public boolean isActiveActiveReplicationEnabledAsDefaultForBatchOnly() { - return activeActiveReplicationEnabledAsDefaultForBatchOnly; - } - public boolean isActiveActiveReplicationEnabledAsDefaultForHybrid() { return activeActiveReplicationEnabledAsDefaultForHybrid; } @@ -1228,7 +1102,7 @@ public int getReplicationMetadataVersion() { return replicationMetadataVersion; } - public String getChildDatacenters() { + public Set getChildDatacenters() { return childDatacenters; } @@ -1272,12 +1146,12 @@ public String getControllerClusterZkAddress() { return controllerClusterZkAddress; } - public boolean isParent() { - return parent; + public boolean isMultiRegion() { + return multiRegion; } - public long getTopicCreationThrottlingTimeWindowMs() { - return topicCreationThrottlingTimeWindowMs; + public boolean isParent() { + return parent; } public long getDeprecatedJobTopicRetentionMs() { @@ -1354,12 +1228,8 @@ public Map getChildDataCenterKafkaUrlMap() { return childDataCenterKafkaUrlMap; } - public List getChildDataCenterAllowlist() { - return childDataCenterAllowlist; - } - - public Set getActiveActiveRealTimeSourceFabrics() { - return activeActiveRealTimeSourceFabrics; + public List getActiveActiveRealTimeSourceKafkaURLs() { + return activeActiveRealTimeSourceKafkaURLs; } public String getNativeReplicationSourceFabric() { @@ -1390,10 +1260,6 @@ public Duration getBatchJobHeartbeatInitialBufferTime() { return batchJobHeartbeatInitialBufferTime; } - public long getAdminConsumptionTimeoutMinutes() { - return adminConsumptionTimeoutMinute; - } - public boolean isEnableDisabledReplicaEnabled() { return enableDisabledReplicaEnabled; } @@ -1406,7 +1272,7 @@ public int getAdminConsumptionMaxWorkerThreadPoolSize() { return adminConsumptionMaxWorkerThreadPoolSize; } - public static Map parseClusterMap(VeniceProperties clusterPros, String datacenterAllowlist) { + static Map parseClusterMap(VeniceProperties clusterPros, Set datacenterAllowlist) { return parseClusterMap(clusterPros, datacenterAllowlist, false); } @@ -1414,10 +1280,6 @@ public double getStorageEngineOverheadRatio() { return storageEngineOverheadRatio; } - public int getTopicManagerKafkaOperationTimeOutMs() { - return topicManagerKafkaOperationTimeOutMs; - } - public int getTopicManagerMetadataFetcherConsumerPoolSize() { return topicManagerMetadataFetcherConsumerPoolSize; } @@ -1454,10 +1316,6 @@ public String getSystemSchemaClusterName() { return systemSchemaClusterName; } - public int getTopicDeletionStatusPollIntervalMs() { - return topicDeletionStatusPollIntervalMs; - } - public boolean isAdminHelixMessagingChannelEnabled() { return adminHelixMessagingChannelEnabled; } @@ -1478,26 +1336,6 @@ public boolean isEarlyDeleteBackUpEnabled() { return earlyDeleteBackUpEnabled; } - public boolean isConcurrentTopicDeleteRequestsEnabled() { - return sendConcurrentTopicDeleteRequestsEnabled; - } - - public boolean isEnableBatchPushFromAdminInChildController() { - return enableBatchPushFromAdminInChildController; - } - - public String getKafkaAdminClass() { - return kafkaAdminClass; - } - - public String getKafkaWriteOnlyClass() { - return kafkaWriteOnlyClass; - } - - public String getKafkaReadOnlyClass() { - return kafkaReadOnlyClass; - } - public int getErrorPartitionAutoResetLimit() { return errorPartitionAutoResetLimit; } @@ -1629,10 +1467,6 @@ public boolean isClusterWipeAllowed() { return allowClusterWipe; } - public boolean isChildControllerAdminTopicConsumptionEnabled() { - return childControllerAdminTopicConsumptionEnabled; - } - public boolean isConcurrentInitRoutinesEnabled() { return concurrentInitRoutinesEnabled; } @@ -1721,9 +1555,9 @@ public PubSubAdminAdapterFactory getSourceOfTruthAdminAdapterFactory() { * @param datacenterAllowlist data centers that are taken into account. * @param D2Routing whether it uses D2 to route or not. */ - public static Map parseClusterMap( + static Map parseClusterMap( VeniceProperties clusterPros, - String datacenterAllowlist, + Set datacenterAllowlist, Boolean D2Routing) { String propsPrefix = D2Routing ? CHILD_CLUSTER_D2_PREFIX : CHILD_CLUSTER_URL_PREFIX; return parseChildDataCenterToValue(propsPrefix, clusterPros, datacenterAllowlist, (m, k, v, errMsg) -> { @@ -1759,7 +1593,7 @@ public static Map parseClusterMap( */ private static Map parseChildDataCenterKafkaUrl( VeniceProperties clusterPros, - String datacenterAllowlist) { + Set datacenterAllowlist) { return parseChildDataCenterToValue( CHILD_DATA_CENTER_KAFKA_URL_PREFIX, clusterPros, @@ -1770,16 +1604,15 @@ private static Map parseChildDataCenterKafkaUrl( private static Map parseChildDataCenterToValue( String configPrefix, VeniceProperties clusterPros, - String datacenterAllowlist, + Set datacenterAllowlist, PutToMap mappingFunction) { Properties childDataCenterKafkaUriProps = clusterPros.clipAndFilterNamespace(configPrefix).toProperties(); - if (StringUtils.isEmpty(datacenterAllowlist)) { + if (datacenterAllowlist == null || datacenterAllowlist.isEmpty()) { throw new VeniceException("child controller list must have a allowlist"); } Map outputMap = new HashMap<>(); - List allowlist = Arrays.asList(datacenterAllowlist.split(LIST_SEPARATOR)); for (Map.Entry uriEntry: childDataCenterKafkaUriProps.entrySet()) { String datacenter = (String) uriEntry.getKey(); @@ -1794,7 +1627,7 @@ private static Map parseChildDataCenterToValue( throw new VeniceException(errMsg + ": found no value for: " + datacenter); } - if (allowlist.contains(datacenter)) { + if (datacenterAllowlist.contains(datacenter)) { mappingFunction.apply(outputMap, datacenter, value, errMsg); } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java index 0c6feac35e..432254e9f5 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java @@ -59,6 +59,10 @@ public boolean adminCheckReadMethodForKafka() { return getCommonConfig().adminCheckReadMethodForKafka(); } + public boolean isMultiRegion() { + return getCommonConfig().isMultiRegion(); + } + public boolean isParent() { return getCommonConfig().isParent(); } @@ -103,10 +107,6 @@ public String getSslFactoryClassName() { return getCommonConfig().getSslFactoryClassName(); } - public boolean isDisableParentRequestTopicForStreamPushes() { - return getCommonConfig().isDisableParentRequestTopicForStreamPushes(); - } - public long getDeprecatedJobTopicRetentionMs() { return getCommonConfig().getDeprecatedJobTopicRetentionMs(); } @@ -155,10 +155,6 @@ public Duration getBatchJobHeartbeatInitialBufferTime() { return getCommonConfig().getBatchJobHeartbeatInitialBufferTime(); } - public long getTopicCreationThrottlingTimeWindowMs() { - return getCommonConfig().getTopicCreationThrottlingTimeWindowMs(); - } - public Map getClusterToD2Map() { return getCommonConfig().getClusterToD2Map(); } @@ -167,22 +163,6 @@ public Map getClusterToServerD2Map() { return getCommonConfig().getClusterToServerD2Map(); } - public int getTopicManagerKafkaOperationTimeOutMs() { - return getCommonConfig().getTopicManagerKafkaOperationTimeOutMs(); - } - - public int getTopicDeletionStatusPollIntervalMs() { - return getCommonConfig().getTopicDeletionStatusPollIntervalMs(); - } - - public boolean isConcurrentTopicDeleteRequestsEnabled() { - return getCommonConfig().isConcurrentTopicDeleteRequestsEnabled(); - } - - public long getKafkaMinLogCompactionLagInMs() { - return getCommonConfig().getKafkaMinLogCompactionLagInMs(); - } - public int getMinNumberOfUnusedKafkaTopicsToPreserve() { return getCommonConfig().getMinNumberOfUnusedKafkaTopicsToPreserve(); } @@ -223,10 +203,6 @@ public String getSystemSchemaClusterName() { return getCommonConfig().getSystemSchemaClusterName(); } - public boolean isEnableBatchPushFromAdminInChildController() { - return getCommonConfig().isEnableBatchPushFromAdminInChildController(); - } - public long getBackupVersionDefaultRetentionMs() { return getCommonConfig().getBackupVersionDefaultRetentionMs(); } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java index 83483589d0..65799d0187 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java @@ -172,15 +172,19 @@ public VeniceControllerService( new LandFillObjectPool<>(KafkaMessageEnvelope::new), new LandFillObjectPool<>(KafkaMessageEnvelope::new)); for (String cluster: multiClusterConfigs.getClusters()) { - AdminConsumerService adminConsumerService = new AdminConsumerService( - internalAdmin, - multiClusterConfigs.getControllerConfig(cluster), - metricsRepository, - pubSubTopicRepository, - pubSubMessageDeserializer); - this.consumerServicesByClusters.put(cluster, adminConsumerService); + VeniceControllerClusterConfig clusterConfig = multiClusterConfigs.getControllerConfig(cluster); + if (clusterConfig.isMultiRegion()) { + // Enable admin channel consumption only for multi-region setups + AdminConsumerService adminConsumerService = new AdminConsumerService( + internalAdmin, + clusterConfig, + metricsRepository, + pubSubTopicRepository, + pubSubMessageDeserializer); + this.consumerServicesByClusters.put(cluster, adminConsumerService); - this.admin.setAdminConsumerService(cluster, adminConsumerService); + this.admin.setAdminConsumerService(cluster, adminConsumerService); + } } } @@ -209,7 +213,9 @@ private LingeringStoreVersionChecker createLingeringStoreVersionChecker( public boolean startInner() { for (String clusterName: multiClusterConfigs.getClusters()) { admin.initStorageCluster(clusterName); - consumerServicesByClusters.get(clusterName).start(); + if (multiClusterConfigs.isMultiRegion()) { + consumerServicesByClusters.get(clusterName).start(); + } LOGGER.info("started cluster: {}", clusterName); } LOGGER.info("Started Venice controller."); @@ -226,10 +232,12 @@ public void stopInner() { // We don't need to lock resources here, as we will acquire the lock during the ST leader->standby, which would // prevent the partial updates. admin.stop(clusterName); - try { - consumerServicesByClusters.get(clusterName).stop(); - } catch (Exception e) { - LOGGER.error("Got exception when stop AdminConsumerService", e); + if (multiClusterConfigs.isMultiRegion()) { + try { + consumerServicesByClusters.get(clusterName).stop(); + } catch (Exception e) { + LOGGER.error("Got exception when stop AdminConsumerService", e); + } } LOGGER.info("Stopped cluster: {}", clusterName); } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java index 06a9637e21..43ae2f9414 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java @@ -634,7 +634,7 @@ public VeniceHelixAdmin( // Participant stores are not read or written in parent colo. Parent controller skips participant store // initialization. - if (!multiClusterConfigs.isParent() && multiClusterConfigs.isParticipantMessageStoreEnabled()) { + if (!isParent() && multiClusterConfigs.isParticipantMessageStoreEnabled()) { initRoutines.add( new PerClusterInternalRTStoreInitializationRoutine( PARTICIPANT_MESSAGE_SYSTEM_STORE_VALUE, @@ -1000,9 +1000,7 @@ public void createStore( } private void configureNewStore(Store newStore, VeniceControllerClusterConfig config, int largestUsedVersionNumber) { - newStore.setNativeReplicationEnabled(config.isNativeReplicationEnabledAsDefaultForBatchOnly()); - newStore.setActiveActiveReplicationEnabled( - config.isActiveActiveReplicationEnabledAsDefaultForBatchOnly() && !newStore.isSystemStore()); + newStore.setNativeReplicationEnabled(config.isMultiRegion()); /** * Initialize default NR source fabric base on default config for different store types. @@ -1328,7 +1326,7 @@ public void writeEndOfPush(String clusterName, String storeName, int versionNumb /** * Test if a store is allowed for a batch push. * @param storeName name of a store. - * @return ture is the store is a participant system store or {@linkplain ConfigKeys#CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD} is enabled. + * @return {@code true} is the store is a participant system store or if Venice is running in single-region mode */ @Override public boolean whetherEnableBatchPushFromAdmin(String storeName) { @@ -1336,8 +1334,7 @@ public boolean whetherEnableBatchPushFromAdmin(String storeName) { * Allow (empty) push to participant system store from child controller directly since participant stores are * independent in different fabrics (different data). */ - return VeniceSystemStoreUtils.isParticipantStore(storeName) - || multiClusterConfigs.isEnableBatchPushFromAdminInChildController(); + return VeniceSystemStoreUtils.isParticipantStore(storeName) || !multiClusterConfigs.isMultiRegion(); } /** @@ -2083,17 +2080,9 @@ public Version addVersionOnly( store = repository.getStore(storeName); version.setPushType(pushType); store.addVersion(version); - // Apply cluster-level native replication configs - VeniceControllerClusterConfig clusterConfig = resources.getConfig(); - - boolean nativeReplicationEnabled = version.isNativeReplicationEnabled(); - if (store.isHybrid()) { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForHybrid(); - } else { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForBatchOnly(); - } - version.setNativeReplicationEnabled(nativeReplicationEnabled); + VeniceControllerClusterConfig clusterConfig = resources.getConfig(); + version.setNativeReplicationEnabled(clusterConfig.isMultiRegion()); if (version.isNativeReplicationEnabled()) { if (remoteKafkaBootstrapServers != null) { @@ -2555,14 +2544,7 @@ private Pair addVersion( store.addVersion(version); } - // Apply cluster-level native replication configs - boolean nativeReplicationEnabled = version.isNativeReplicationEnabled(); - if (store.isHybrid()) { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForHybrid(); - } else { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForBatchOnly(); - } - version.setNativeReplicationEnabled(nativeReplicationEnabled); + version.setNativeReplicationEnabled(store.isNativeReplicationEnabled()); // Check whether native replication is enabled if (version.isNativeReplicationEnabled()) { @@ -4054,7 +4036,7 @@ void preCheckStorePartitionCountUpdate(String clusterName, Store store, int newP TopicManager topicManager; if (isParent()) { // RT might not exist in parent colo. Get RT partition count from a child colo. - String childDatacenter = Utils.parseCommaSeparatedStringToList(clusterConfig.getChildDatacenters()).get(0); + String childDatacenter = clusterConfig.getChildDatacenters().iterator().next(); topicManager = getTopicManager(multiClusterConfigs.getChildDataCenterKafkaUrlMap().get(childDatacenter)); } else { topicManager = getTopicManager(); @@ -4268,7 +4250,6 @@ void setIncrementalPushEnabled(String clusterName, String storeName, boolean inc VeniceControllerClusterConfig config = getHelixVeniceClusterResources(clusterName).getConfig(); if (incrementalPushEnabled || store.isHybrid()) { // Enabling incremental push - store.setNativeReplicationEnabled(config.isNativeReplicationEnabledAsDefaultForHybrid()); store.setNativeReplicationSourceFabric(config.getNativeReplicationSourceFabricAsDefaultForHybrid()); store.setActiveActiveReplicationEnabled( store.isActiveActiveReplicationEnabled() @@ -4276,11 +4257,8 @@ void setIncrementalPushEnabled(String clusterName, String storeName, boolean inc } else { // Disabling incremental push // This is only possible when hybrid settings are set to null before turning of incremental push for the store. - store.setNativeReplicationEnabled(config.isNativeReplicationEnabledAsDefaultForBatchOnly()); store.setNativeReplicationSourceFabric(config.getNativeReplicationSourceFabricAsDefaultForBatchOnly()); - store.setActiveActiveReplicationEnabled( - store.isActiveActiveReplicationEnabled() - || (config.isActiveActiveReplicationEnabledAsDefaultForBatchOnly() && !store.isSystemStore())); + store.setActiveActiveReplicationEnabled(false); } store.setIncrementalPushEnabled(incrementalPushEnabled); @@ -4686,13 +4664,9 @@ private void internalUpdateStore(String clusterName, String storeName, UpdateSto store.setIncrementalPushEnabled(false); // Enable/disable native replication for batch-only stores if the cluster level config for new batch // stores is on - store.setNativeReplicationEnabled(clusterConfig.isNativeReplicationEnabledAsDefaultForBatchOnly()); store.setNativeReplicationSourceFabric( clusterConfig.getNativeReplicationSourceFabricAsDefaultForBatchOnly()); - store.setActiveActiveReplicationEnabled( - store.isActiveActiveReplicationEnabled() - || (clusterConfig.isActiveActiveReplicationEnabledAsDefaultForBatchOnly() - && !store.isSystemStore())); + store.setActiveActiveReplicationEnabled(false); } else { // Batch-only store is being converted to hybrid store. if (!store.isHybrid()) { @@ -4700,7 +4674,6 @@ private void internalUpdateStore(String clusterName, String storeName, UpdateSto * Enable/disable native replication for hybrid stores if the cluster level config * for new hybrid stores is on */ - store.setNativeReplicationEnabled(clusterConfig.isNativeReplicationEnabledAsDefaultForHybrid()); store .setNativeReplicationSourceFabric(clusterConfig.getNativeReplicationSourceFabricAsDefaultForHybrid()); /* @@ -7152,131 +7125,6 @@ public void deleteAclForStore(String clusterName, String storeName) { throw new VeniceUnsupportedOperationException("deleteAclForStore is not supported!"); } - /** - * @see Admin#configureNativeReplication(String, VeniceUserStoreType, Optional, boolean, Optional, Optional) - */ - @Override - public void configureNativeReplication( - String clusterName, - VeniceUserStoreType storeType, - Optional storeName, - boolean enableNativeReplicationForCluster, - Optional newSourceFabric, - Optional regionsFilter) { - /** - * Check whether the command affects this fabric. - */ - if (regionsFilter.isPresent()) { - Set fabrics = parseRegionsFilterList(regionsFilter.get()); - if (!fabrics.contains(multiClusterConfigs.getRegionName())) { - LOGGER.info( - "EnableNativeReplicationForCluster command will be skipped for cluster {}, because the fabrics filter " - + "is {} which doesn't include the current fabric: {}", - clusterName, - fabrics, - multiClusterConfigs.getRegionName()); - return; - } - } - - if (storeName.isPresent()) { - /** - * Legacy stores venice_system_store_davinci_push_status_store_ still exist. - * But {@link com.linkedin.venice.helix.HelixReadOnlyStoreRepositoryAdapter#getStore(String)} cannot find - * them by store names. Skip davinci push status stores until legacy znodes are cleaned up. - */ - VeniceSystemStoreType systemStoreType = VeniceSystemStoreType.getSystemStoreType(storeName.get()); - if (systemStoreType != null && systemStoreType.equals(VeniceSystemStoreType.DAVINCI_PUSH_STATUS_STORE)) { - LOGGER.info("Will not enable native replication for davinci push status store: {}", storeName.get()); - return; - } - - /** - * The function is invoked by {@link com.linkedin.venice.controller.kafka.consumer.AdminExecutionTask} if the - * storeName is present. - */ - Store originalStore = getStore(clusterName, storeName.get()); - if (originalStore == null) { - throw new VeniceNoStoreException(storeName.get(), clusterName); - } - boolean shouldUpdateNativeReplication = false; - switch (storeType) { - case BATCH_ONLY: - shouldUpdateNativeReplication = - !originalStore.isHybrid() && !originalStore.isIncrementalPushEnabled() && !originalStore.isSystemStore(); - break; - case HYBRID_ONLY: - shouldUpdateNativeReplication = - originalStore.isHybrid() && !originalStore.isIncrementalPushEnabled() && !originalStore.isSystemStore(); - break; - case INCREMENTAL_PUSH: - shouldUpdateNativeReplication = originalStore.isIncrementalPushEnabled() && !originalStore.isSystemStore(); - break; - case HYBRID_OR_INCREMENTAL: - shouldUpdateNativeReplication = - (originalStore.isHybrid() || originalStore.isIncrementalPushEnabled()) && !originalStore.isSystemStore(); - break; - case SYSTEM: - shouldUpdateNativeReplication = originalStore.isSystemStore(); - break; - case ALL: - shouldUpdateNativeReplication = true; - break; - default: - throw new VeniceException("Unsupported store type." + storeType); - } - if (shouldUpdateNativeReplication) { - LOGGER.info("Will enable native replication for store: {}", storeName.get()); - setNativeReplicationEnabled(clusterName, storeName.get(), enableNativeReplicationForCluster); - newSourceFabric.ifPresent(f -> setNativeReplicationSourceFabric(clusterName, storeName.get(), f)); - } else { - LOGGER.info("Will not enable native replication for store: {}", storeName.get()); - } - } else { - /** - * The batch update command hits child controller directly; all stores in the cluster will be updated - */ - List storesToBeConfigured; - switch (storeType) { - case BATCH_ONLY: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> (!s.isHybrid() && !s.isIncrementalPushEnabled() && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case HYBRID_ONLY: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> (s.isHybrid() && !s.isIncrementalPushEnabled() && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case INCREMENTAL_PUSH: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> (s.isIncrementalPushEnabled() && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case HYBRID_OR_INCREMENTAL: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> ((s.isHybrid() || s.isIncrementalPushEnabled()) && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case SYSTEM: - storesToBeConfigured = - getAllStores(clusterName).stream().filter(Store::isSystemStore).collect(Collectors.toList()); - break; - case ALL: - storesToBeConfigured = getAllStores(clusterName); - break; - default: - throw new VeniceException("Unsupported store type." + storeType); - } - - storesToBeConfigured.forEach(store -> { - LOGGER.info("Will enable native replication for store: {}", store.getName()); - setNativeReplicationEnabled(clusterName, store.getName(), enableNativeReplicationForCluster); - newSourceFabric.ifPresent(f -> setNativeReplicationSourceFabric(clusterName, storeName.get(), f)); - }); - } - } - /** * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) */ @@ -7781,7 +7629,7 @@ public Optional getEmergencySourceRegion(@Nonnull String clusterName) { @Override public Optional getAggregateRealTimeTopicSource(String clusterName) { String sourceRegion = multiClusterConfigs.getControllerConfig(clusterName).getAggregateRealTimeSourceRegion(); - if (sourceRegion != null && sourceRegion.length() > 0) { + if (!StringUtils.isEmpty(sourceRegion)) { return Optional.of(getNativeReplicationKafkaBootstrapServerAddress(sourceRegion)); } else { return Optional.empty(); @@ -8098,12 +7946,12 @@ public boolean isAdminTopicConsumptionEnabled(String clusterName) { // HelixVeniceClusterResources should exist on leader controller HelixVeniceClusterResources resources = getHelixVeniceClusterResources(clusterName); try (AutoCloseableLock ignore = resources.getClusterLockManager().createClusterReadLock()) { - HelixReadWriteLiveClusterConfigRepository clusterConfigRepository = + HelixReadWriteLiveClusterConfigRepository liveConfigRepository = getReadWriteLiveClusterConfigRepository(clusterName); - // Enable child controller admin topic consumption when both cfg2 config and live config are true - adminTopicConsumptionEnabled = - clusterConfigRepository.getConfigs().isChildControllerAdminTopicConsumptionEnabled() - && multiClusterConfigs.getControllerConfig(clusterName).isChildControllerAdminTopicConsumptionEnabled(); + // Enable child controller admin topic consumption when configs applied at startup and live config are true. + // The live configs are used during store migration. + adminTopicConsumptionEnabled = liveConfigRepository.getConfigs().isChildControllerAdminTopicConsumptionEnabled() + && multiClusterConfigs.getControllerConfig(clusterName).isMultiRegion(); } return adminTopicConsumptionEnabled; } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java index 73e071711c..6d5a8ed66c 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java @@ -92,7 +92,6 @@ import com.linkedin.venice.controller.kafka.protocol.admin.AddVersion; import com.linkedin.venice.controller.kafka.protocol.admin.AdminOperation; import com.linkedin.venice.controller.kafka.protocol.admin.ConfigureActiveActiveReplicationForCluster; -import com.linkedin.venice.controller.kafka.protocol.admin.ConfigureNativeReplicationForCluster; import com.linkedin.venice.controller.kafka.protocol.admin.CreateStoragePersona; import com.linkedin.venice.controller.kafka.protocol.admin.DeleteAllVersions; import com.linkedin.venice.controller.kafka.protocol.admin.DeleteOldVersion; @@ -3859,7 +3858,7 @@ private boolean whetherToCreateNewDataRecoveryVersion( * e.g. {@link StoreBackupVersionCleanupService#cleanupBackupVersion(Store, String)}. */ return destStore.getHybridStoreConfig() == null && versionNumber <= destStore.getCurrentVersion() - && multiClusterConfigs.getControllerConfig(clusterName).getChildDataCenterAllowlist().contains(destFabric); + && multiClusterConfigs.getControllerConfig(clusterName).getChildDatacenters().contains(destFabric); } /** @@ -4718,32 +4717,6 @@ public void deleteAclForStore(String clusterName, String storeName) { } } - /** - * @see Admin#configureNativeReplication(String, VeniceUserStoreType, Optional, boolean, Optional, Optional) - */ - @Override - public void configureNativeReplication( - String clusterName, - VeniceUserStoreType storeType, - Optional storeName, - boolean enableNativeReplicationForCluster, - Optional newSourceRegion, - Optional regionsFilter) { - ConfigureNativeReplicationForCluster migrateClusterToNativeReplication = - (ConfigureNativeReplicationForCluster) AdminMessageType.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER - .getNewInstance(); - migrateClusterToNativeReplication.clusterName = clusterName; - migrateClusterToNativeReplication.storeType = storeType.toString(); - migrateClusterToNativeReplication.enabled = enableNativeReplicationForCluster; - migrateClusterToNativeReplication.nativeReplicationSourceRegion = newSourceRegion.orElse(null); - migrateClusterToNativeReplication.regionsFilter = regionsFilter.orElse(null); - - AdminOperation message = new AdminOperation(); - message.operationType = AdminMessageType.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER.getValue(); - message.payloadUnion = migrateClusterToNativeReplication; - sendAdminMessageAndWaitForConsumed(clusterName, null, message); - } - /** * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) */ @@ -4796,7 +4769,7 @@ public Map getClusterStaleStores(String clusterName) { StoreDataAudit audit = store.getValue(); Optional currentPushJobTopic = getTopicForCurrentPushJob(clusterName, store.getValue().getStoreName(), false, false); - if (audit.getStaleRegions().size() > 0 && !currentPushJobTopic.isPresent()) { + if (!audit.getStaleRegions().isEmpty() && !currentPushJobTopic.isPresent()) { retMap.put(store.getKey(), audit); } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/TopicCleanupService.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/TopicCleanupService.java index 3da813cc62..e1d2fa2f1c 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/TopicCleanupService.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/TopicCleanupService.java @@ -17,7 +17,6 @@ import com.linkedin.venice.pubsub.manager.TopicManager; import com.linkedin.venice.service.AbstractVeniceService; import com.linkedin.venice.utils.Time; -import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.util.ArrayList; import java.util.Collections; @@ -74,7 +73,7 @@ public class TopicCleanupService extends AbstractVeniceService { protected final int delayFactor; private final int minNumberOfUnusedKafkaTopicsToPreserve; private final AtomicBoolean stop = new AtomicBoolean(false); - private final List childFabricList; + private final Set childRegions; private final Map> multiDataCenterStoreToVersionTopicCount; private final PubSubTopicRepository pubSubTopicRepository; private final TopicCleanupServiceStats topicCleanupServiceStats; @@ -105,12 +104,11 @@ public TopicCleanupService( this.multiClusterConfigs = multiClusterConfigs; this.pubSubTopicRepository = pubSubTopicRepository; this.topicCleanupServiceStats = topicCleanupServiceStats; - this.childFabricList = - Utils.parseCommaSeparatedStringToList(multiClusterConfigs.getCommonConfig().getChildDatacenters()); + this.childRegions = multiClusterConfigs.getCommonConfig().getChildDatacenters(); if (!admin.isParent()) { // Only perform cross fabric VT check for RT deletion in child fabrics. - this.multiDataCenterStoreToVersionTopicCount = new HashMap<>(childFabricList.size()); - for (String datacenter: childFabricList) { + this.multiDataCenterStoreToVersionTopicCount = new HashMap<>(childRegions.size()); + for (String datacenter: childRegions) { multiDataCenterStoreToVersionTopicCount.put(datacenter, new HashMap<>()); } } else { @@ -306,14 +304,14 @@ private void populateDeprecatedTopicQueue(PriorityQueue topics) { private void refreshMultiDataCenterStoreToVersionTopicCountMap(Set localTopics) { if (localDatacenter == null) { String localPubSubBootstrapServer = getTopicManager().getPubSubClusterAddress(); - for (String childFabric: childFabricList) { + for (String childFabric: childRegions) { if (localPubSubBootstrapServer.equals(multiClusterConfigs.getChildDataCenterKafkaUrlMap().get(childFabric))) { localDatacenter = childFabric; break; } } if (localDatacenter == null) { - String childFabrics = String.join(",", childFabricList); + String childFabrics = String.join(",", childRegions); LOGGER.error( "Blocking RT topic deletion. Cannot find local datacenter in child datacenter list: {}", childFabrics); @@ -324,8 +322,8 @@ private void refreshMultiDataCenterStoreToVersionTopicCountMap(Set clearAndPopulateStoreToVersionTopicCountMap( localTopics, multiDataCenterStoreToVersionTopicCount.get(localDatacenter)); - if (childFabricList.size() > 1) { - for (String childFabric: childFabricList) { + if (childRegions.size() > 1) { + for (String childFabric: childRegions) { try { if (childFabric.equals(localDatacenter)) { continue; diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java index d919b9319c..e3aacf847d 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java @@ -546,7 +546,7 @@ private void handleSetStore(UpdateStore message) { if (message.replicateAllConfigs) { finalParams = params; } else { - if (message.updatedConfigsList == null || message.updatedConfigsList.size() == 0) { + if (message.updatedConfigsList == null || message.updatedConfigsList.isEmpty()) { throw new VeniceException( "UpdateStore failed for store " + storeName + ". replicateAllConfigs flag was off " + "but there was no config updates."); @@ -676,21 +676,9 @@ private void handleAddVersion(AddVersion message) { } private void handleEnableNativeReplicationForCluster(ConfigureNativeReplicationForCluster message) { - String clusterName = message.clusterName.toString(); - VeniceUserStoreType storeType = VeniceUserStoreType.valueOf(message.storeType.toString().toUpperCase()); - boolean enableNativeReplication = message.enabled; - Optional nativeReplicationSourceFabric = (message.nativeReplicationSourceRegion == null) - ? Optional.empty() - : Optional.of(message.nativeReplicationSourceRegion.toString()); - Optional regionsFilter = - (message.regionsFilter == null) ? Optional.empty() : Optional.of(message.regionsFilter.toString()); - admin.configureNativeReplication( - clusterName, - storeType, - Optional.of(storeName), - enableNativeReplication, - nativeReplicationSourceFabric, - regionsFilter); + LOGGER.info( + "Received message to configure native replication for cluster: {} but ignoring it as native replication is the only mode", + message.clusterName); } private void handleEnableActiveActiveReplicationForCluster(ConfigureActiveActiveReplicationForCluster message) { diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java index 779fb4ad4e..3765eb4557 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java @@ -15,7 +15,6 @@ import static com.linkedin.venice.controllerapi.ControllerRoute.COMPARE_STORE; import static com.linkedin.venice.controllerapi.ControllerRoute.COMPLETE_MIGRATION; import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER; -import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER; import static com.linkedin.venice.controllerapi.ControllerRoute.CREATE_STORAGE_PERSONA; import static com.linkedin.venice.controllerapi.ControllerRoute.ClUSTER_HEALTH_INSTANCES; import static com.linkedin.venice.controllerapi.ControllerRoute.DATA_RECOVERY; @@ -404,9 +403,6 @@ public boolean startInner() throws Exception { httpService.post(UPLOAD_PUSH_JOB_STATUS.getPath(), jobRoutes.uploadPushJobStatus(admin)); httpService.post(SEND_PUSH_JOB_DETAILS.getPath(), jobRoutes.sendPushJobDetails(admin)); - httpService.post( - CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER.getPath(), - storesRoutes.enableNativeReplicationForCluster(admin)); httpService.post( CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER.getPath(), storesRoutes.enableActiveActiveReplicationForCluster(admin)); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java index 8b4e2b5a3a..6bfa7f51b3 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java @@ -9,7 +9,6 @@ import static com.linkedin.venice.controllerapi.ControllerApiConstants.HEARTBEAT_TIMESTAMP; import static com.linkedin.venice.controllerapi.ControllerApiConstants.INCLUDE_SYSTEM_STORES; import static com.linkedin.venice.controllerapi.ControllerApiConstants.NAME; -import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OPERATION; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OWNER; import static com.linkedin.venice.controllerapi.ControllerApiConstants.PARTITION_DETAIL_ENABLED; @@ -30,7 +29,6 @@ import static com.linkedin.venice.controllerapi.ControllerRoute.COMPARE_STORE; import static com.linkedin.venice.controllerapi.ControllerRoute.COMPLETE_MIGRATION; import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER; -import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER; import static com.linkedin.venice.controllerapi.ControllerRoute.DELETE_ALL_VERSIONS; import static com.linkedin.venice.controllerapi.ControllerRoute.DELETE_KAFKA_TOPIC; import static com.linkedin.venice.controllerapi.ControllerRoute.DELETE_STORE; @@ -785,40 +783,6 @@ public void internalHandle(Request request, StorageEngineOverheadRatioResponse v }; } - /** - * @see Admin#configureNativeReplication(String, VeniceUserStoreType, Optional, boolean, Optional, Optional) - */ - public Route enableNativeReplicationForCluster(Admin admin) { - return new VeniceRouteHandler(ControllerResponse.class) { - @Override - public void internalHandle(Request request, ControllerResponse veniceResponse) { - // Only allow allowlist users to run this command - if (!checkIsAllowListUser(request, veniceResponse, () -> isAllowListUser(request))) { - return; - } - - AdminSparkServer.validateParams(request, CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER.getParams(), admin); - - VeniceUserStoreType storeType = VeniceUserStoreType.valueOf(request.queryParams(STORE_TYPE).toUpperCase()); - - String cluster = request.queryParams(CLUSTER); - boolean enableNativeReplicationForCluster = Utils.parseBooleanFromString(request.queryParams(STATUS), STATUS); - String sourceRegionParams = request.queryParamOrDefault(NATIVE_REPLICATION_SOURCE_FABRIC, null); - String regionsFilterParams = request.queryParamOrDefault(REGIONS_FILTER, null); - - admin.configureNativeReplication( - cluster, - storeType, - Optional.empty(), - enableNativeReplicationForCluster, - Optional.ofNullable(sourceRegionParams), - Optional.ofNullable(regionsFilterParams)); - - veniceResponse.setCluster(cluster); - } - }; - } - /** * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) */ diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/pushmonitor/PartitionStatusBasedPushMonitor.java b/services/venice-controller/src/main/java/com/linkedin/venice/pushmonitor/PartitionStatusBasedPushMonitor.java index 2326bb5643..8929d108ae 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/pushmonitor/PartitionStatusBasedPushMonitor.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/pushmonitor/PartitionStatusBasedPushMonitor.java @@ -34,7 +34,7 @@ public PartitionStatusBasedPushMonitor( RealTimeTopicSwitcher realTimeTopicSwitcher, ClusterLockManager clusterLockManager, String aggregateRealTimeSourceKafkaUrl, - List childDataCenterKafkaUrls, + List activeActiveRealTimeSourceKafkaURLs, HelixAdminClient helixAdminClient, VeniceControllerClusterConfig controllerConfig, PushStatusStoreReader pushStatusStoreReader, @@ -49,7 +49,7 @@ public PartitionStatusBasedPushMonitor( realTimeTopicSwitcher, clusterLockManager, aggregateRealTimeSourceKafkaUrl, - childDataCenterKafkaUrls, + activeActiveRealTimeSourceKafkaURLs, helixAdminClient, controllerConfig, pushStatusStoreReader, diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceControllerClusterConfig.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceControllerClusterConfig.java index 28633252d1..3e6e5ec8fe 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceControllerClusterConfig.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceControllerClusterConfig.java @@ -1,20 +1,45 @@ package com.linkedin.venice.controller; +import static com.linkedin.venice.ConfigKeys.ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED; +import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; +import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_URL_PREFIX; +import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; +import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; +import static com.linkedin.venice.ConfigKeys.CLUSTER_TO_D2; +import static com.linkedin.venice.ConfigKeys.CLUSTER_TO_SERVER_D2; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_ADD_VERSION_VIA_ADMIN_PROTOCOL; import static com.linkedin.venice.ConfigKeys.CONTROLLER_DISABLED_ROUTES; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_MODE; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_SSL_ENABLED; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_SYSTEM_SCHEMA_CLUSTER_NAME; +import static com.linkedin.venice.ConfigKeys.DEFAULT_MAX_NUMBER_OF_PARTITIONS; +import static com.linkedin.venice.ConfigKeys.DEFAULT_PARTITION_SIZE; +import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.LOCAL_REGION_NAME; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; +import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_ALLOWLIST; +import static com.linkedin.venice.ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED; +import static com.linkedin.venice.ConfigKeys.ZOOKEEPER_ADDRESS; import com.linkedin.venice.controllerapi.ControllerRoute; import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.utils.DataProviderUtils; import com.linkedin.venice.utils.PropertyBuilder; +import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.utils.Utils; +import com.linkedin.venice.utils.VeniceProperties; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.Set; import org.testng.Assert; import org.testng.annotations.Test; public class TestVeniceControllerClusterConfig { private static final String DELIMITER = ",\\s*"; - private static final String WHITE_LIST = "dc1,dc2"; + private static final Set REGION_ALLOW_LIST = Utils.setOf("dc1", "dc2"); @Test public void canParseClusterMap() { @@ -22,15 +47,15 @@ public void canParseClusterMap() { builder.put("child.cluster.url.dc1", "http://host:1234, http://host:5678") .put("child.cluster.url.dc2", "http://host:1234, http://host:5678"); - Map map = VeniceControllerClusterConfig.parseClusterMap(builder.build(), WHITE_LIST); + Map map = VeniceControllerClusterConfig.parseClusterMap(builder.build(), REGION_ALLOW_LIST); Assert.assertEquals(map.size(), 2); - Assert.assertTrue(map.keySet().contains("dc1")); - Assert.assertTrue(map.keySet().contains("dc2")); + Assert.assertTrue(map.containsKey("dc1")); + Assert.assertTrue(map.containsKey("dc2")); String[] uris = map.get("dc1").split(DELIMITER); - Assert.assertTrue(uris[0].equals("http://host:1234")); - Assert.assertTrue(uris[1].equals("http://host:5678")); + Assert.assertEquals(uris[0], "http://host:1234"); + Assert.assertEquals(uris[1], "http://host:5678"); } @Test @@ -38,9 +63,9 @@ public void canParseD2ClusterMap() { PropertyBuilder builder = new PropertyBuilder(); builder.put("child.cluster.d2.zkHost.dc1", "zkAddress1").put("child.cluster.d2.zkHost.dc2", "zkAddress2"); - Map map = VeniceControllerClusterConfig.parseClusterMap(builder.build(), WHITE_LIST, true); + Map map = VeniceControllerClusterConfig.parseClusterMap(builder.build(), REGION_ALLOW_LIST, true); Assert.assertEquals(map.get("dc1").split(DELIMITER).length, 1); - Assert.assertTrue(map.get("dc2").split(DELIMITER)[0].equals("zkAddress2")); + Assert.assertEquals(map.get("dc2").split(DELIMITER)[0], "zkAddress2"); } @Test @@ -64,27 +89,89 @@ public void canParseBannedPaths() { public void emptyAllowlist() { PropertyBuilder build = new PropertyBuilder().put("child.cluster.url.dc1", "http://host:1234, http://host:5678") .put("child.cluster.url.dc2", "http://host:1234, http://host:5678"); - VeniceControllerClusterConfig.parseClusterMap(build.build(), ""); + VeniceControllerClusterConfig.parseClusterMap(build.build(), Collections.emptySet()); } @Test(expectedExceptions = VeniceException.class) public void nullAllowlist() { PropertyBuilder build = new PropertyBuilder().put("child.cluster.url.dc1", "http://host:1234, http://host:5678") .put("child.cluster.url.dc2", "http://host:1234, http://host:5678"); - VeniceControllerClusterConfig.parseClusterMap(build.build(), ""); + VeniceControllerClusterConfig.parseClusterMap(build.build(), null); } @Test(expectedExceptions = VeniceException.class) public void errOnMissingScheme() { PropertyBuilder builder = new PropertyBuilder(); builder.put("child.cluster.url.dc1", "host:1234"); - VeniceControllerClusterConfig.parseClusterMap(builder.build(), WHITE_LIST); + VeniceControllerClusterConfig.parseClusterMap(builder.build(), REGION_ALLOW_LIST); } @Test(expectedExceptions = VeniceException.class) public void errOnMissingNodes() { PropertyBuilder builder = new PropertyBuilder(); builder.put("child.cluster.url.dc1", ""); - VeniceControllerClusterConfig.parseClusterMap(builder.build(), WHITE_LIST); + VeniceControllerClusterConfig.parseClusterMap(builder.build(), REGION_ALLOW_LIST); + } + + private Properties getBaseSingleRegionProperties(boolean includeMultiRegionConfig) { + Properties props = TestUtils.getPropertiesForControllerConfig(); + String clusterName = props.getProperty(CLUSTER_NAME); + props.put(LOCAL_REGION_NAME, "dc-0"); + props.put(ZOOKEEPER_ADDRESS, "zkAddress"); + props.put(KAFKA_BOOTSTRAP_SERVERS, "kafkaBootstrapServers"); + props.put(DEFAULT_PARTITION_SIZE, 10); + props.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 16); + props.put(CLUSTER_TO_D2, TestUtils.getClusterToD2String(Collections.singletonMap(clusterName, "dummy_d2"))); + props.put( + CLUSTER_TO_SERVER_D2, + TestUtils.getClusterToD2String(Collections.singletonMap(clusterName, "dummy_server_d2"))); + props.put(CONTROLLER_ADD_VERSION_VIA_ADMIN_PROTOCOL, true); + props.put(ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED, false); + props.put(PARTICIPANT_MESSAGE_STORE_ENABLED, true); + props.put(CONTROLLER_SYSTEM_SCHEMA_CLUSTER_NAME, clusterName); + props.put(CONTROLLER_SSL_ENABLED, false); + if (includeMultiRegionConfig) { + props.put(MULTI_REGION, "false"); + } + return props; + } + + private Properties getBaseMultiRegionProperties(boolean includeMultiRegionConfig) { + Properties props = getBaseSingleRegionProperties(false); + props.put(NATIVE_REPLICATION_FABRIC_ALLOWLIST, "dc-0, dc-1, dc-parent"); + props.put(CHILD_DATA_CENTER_KAFKA_URL_PREFIX + ".dc-0", "kafkaUrlDc0"); + props.put(CHILD_DATA_CENTER_KAFKA_URL_PREFIX + ".dc-1", "kafkaUrlDc1"); + + if (includeMultiRegionConfig) { + props.put(MULTI_REGION, "true"); + } + return props; + } + + private Properties getBaseParentControllerProperties(boolean includeMultiRegionConfig) { + Properties props = getBaseMultiRegionProperties(includeMultiRegionConfig); + props.put(CONTROLLER_PARENT_MODE, "true"); + props.put(CHILD_CLUSTER_ALLOWLIST, "dc-0, dc-1"); + props.put(CHILD_CLUSTER_URL_PREFIX + "dc-0", "http://childControllerUrlDc0"); + props.put(CHILD_CLUSTER_URL_PREFIX + "dc-1", "http://childControllerUrlDc1"); + return props; + } + + @Test(dataProvider = "True-and-False", dataProviderClass = DataProviderUtils.class) + public void testMultiRegionConfig(boolean explicitMultiRegionConfig) { + Properties singleRegionProps = getBaseSingleRegionProperties(explicitMultiRegionConfig); + VeniceControllerClusterConfig singleRegionConfig = + new VeniceControllerClusterConfig(new VeniceProperties(singleRegionProps)); + Assert.assertFalse(singleRegionConfig.isMultiRegion()); + + Properties multiRegionProps = getBaseMultiRegionProperties(explicitMultiRegionConfig); + VeniceControllerClusterConfig multiRegionConfig = + new VeniceControllerClusterConfig(new VeniceProperties(multiRegionProps)); + Assert.assertTrue(multiRegionConfig.isMultiRegion()); + + Properties parentControllerProps = getBaseParentControllerProperties(explicitMultiRegionConfig); + VeniceControllerClusterConfig parentControllerConfig = + new VeniceControllerClusterConfig(new VeniceProperties(parentControllerProps)); + Assert.assertTrue(parentControllerConfig.isMultiRegion()); } } diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupService.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupService.java index 38d0b5d524..e78b738c73 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupService.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupService.java @@ -80,7 +80,7 @@ public void setUp() { VeniceControllerClusterConfig controllerConfig = mock(VeniceControllerClusterConfig.class); doReturn(controllerConfig).when(veniceControllerMultiClusterConfig).getCommonConfig(); - doReturn("local,remote").when(controllerConfig).getChildDatacenters(); + doReturn(Utils.setOf("local", "remote")).when(controllerConfig).getChildDatacenters(); Map dataCenterToBootstrapServerMap = new HashMap<>(); dataCenterToBootstrapServerMap.put("local", "local"); dataCenterToBootstrapServerMap.put("remote", "remote"); @@ -445,7 +445,7 @@ public void testCleanVeniceTopicsBlockRTTopicDeletionWhenMisconfigured() { // Mis-configured where local data center is not in the child data centers list VeniceControllerClusterConfig controllerConfig = mock(VeniceControllerClusterConfig.class); doReturn(controllerConfig).when(veniceControllerMultiClusterConfig).getCommonConfig(); - doReturn("remote").when(controllerConfig).getChildDatacenters(); + doReturn(Collections.singleton("remote")).when(controllerConfig).getChildDatacenters(); TopicCleanupService blockedTopicCleanupService = new TopicCleanupService( admin, veniceControllerMultiClusterConfig, diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForMultiKafkaClusters.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForMultiKafkaClusters.java index 148f1e94df..60170dcc27 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForMultiKafkaClusters.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForMultiKafkaClusters.java @@ -14,6 +14,7 @@ import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.manager.TopicManager; +import com.linkedin.venice.utils.Utils; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -40,7 +41,7 @@ public void setUp() { VeniceControllerClusterConfig controllerConfig = mock(VeniceControllerClusterConfig.class); doReturn(controllerConfig).when(config).getCommonConfig(); doReturn(new ApacheKafkaAdminAdapterFactory()).when(config).getSourceOfTruthAdminAdapterFactory(); - doReturn("fabric1,fabric2").when(controllerConfig).getChildDatacenters(); + doReturn(Utils.setOf("fabric1", "fabric2")).when(controllerConfig).getChildDatacenters(); String kafkaClusterKey1 = "fabric1"; String kafkaClusterKey2 = "fabric2"; diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForParentController.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForParentController.java index d86395c41f..0c4a987da5 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForParentController.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/TestTopicCleanupServiceForParentController.java @@ -14,6 +14,7 @@ import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapterFactory; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.manager.TopicManager; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -38,7 +39,7 @@ public void setUp() { doReturn(2).when(config).getTopicCleanupDelayFactor(); VeniceControllerClusterConfig controllerConfig = mock(VeniceControllerClusterConfig.class); doReturn(controllerConfig).when(config).getCommonConfig(); - doReturn("dc1").when(controllerConfig).getChildDatacenters(); + doReturn(Collections.singleton("dc1")).when(controllerConfig).getChildDatacenters(); TopicCleanupServiceStats topicCleanupServiceStats = mock(TopicCleanupServiceStats.class); doReturn(new ApacheKafkaAdminAdapterFactory()).when(pubSubClientsFactory).getAdminAdapterFactory(); doReturn(new ApacheKafkaAdminAdapterFactory()).when(config).getSourceOfTruthAdminAdapterFactory(); diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java index 58d8264571..d68d9c2fd6 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java @@ -1,6 +1,5 @@ package com.linkedin.venice.controller.kafka.consumer; -import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; @@ -52,7 +51,6 @@ public void testMultipleAdminConsumerServiceWithSameMetricsRepo() { .put( CLUSTER_TO_SERVER_D2, TestUtils.getClusterToD2String(Collections.singletonMap(someClusterName, "dummy_server_d2"))) - .put(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, true) .put(ADMIN_TOPIC_SOURCE_REGION, adminTopicSourceRegion) .put(NATIVE_REPLICATION_FABRIC_ALLOWLIST, adminTopicSourceRegion) .put(CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + adminTopicSourceRegion, "blah")