diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf2f31f5..4071f6718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Lists all changes with user impact. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [0.20.10] +### Changed +- Implemented locality weighted load balancing + + ## [0.20.9] ### Changed - Configurable path normalization diff --git a/docs/configuration.md b/docs/configuration.md index b6f6d13c6..39acb31e4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -129,14 +129,16 @@ Property **envoy-control.envoy.snapshot.outgoing-permissions.rbac.clients-lists.custom-clients-lists** | Lists of clients which will be applied to each rbac policy, only if key for defined list is present in clients for defined endpoint | empty map ## Load Balancing -Property | Description | Default value -------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- -**envoy-control.envoy.snapshot.load-balancing.weights.enabled** | if set to true, weighted load balancing will be enabled | false -**envoy-control.envoy.snapshot.load-balancing.canary.enabled** | if set to true, routing to canary instances based on *canary header* will be enabled (corresponding Envoy static config is required, see [docs](features/load_balancing.md)) | false -**envoy-control.envoy.snapshot.load-balancing.canary.metadata-key** | metadata that will be set for canary EDS endpoints - key (must match Envoy static `header_to_metadata` filter config, see [docs](features/load_balancing.md)) | canary -**envoy-control.envoy.snapshot.load-balancing.canary.header-value** | only when *canary header* is set to this value request will be routed to canary instances (*canary header* name is set in Envoy static config, see [docs](features/load_balancing.md)) | 1 -**envoy-control.envoy.snapshot.load-balancing.policy** | load balancing policy used for clusters. [Accepted values](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy) | LEAST_REQUEST -**envoy-control.envoy.snapshot.load-balancing.use-keys-subset-fallback-policy** | KEYS_SUBSET fallback policy is used by default when canary and service-tags are enabled. It is not supported in Envoy <= 1.12.x. Set to false for compatibility with Envoy 1.12.x | true +Property | Description | Default value +------------------------------------------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------- +**envoy-control.envoy.snapshot.load-balancing.weights.enabled** | if set to true, weighted load balancing will be enabled | false +**envoy-control.envoy.snapshot.load-balancing.canary.enabled** | if set to true, routing to canary instances based on *canary header* will be enabled (corresponding Envoy static config is required, see [docs](features/load_balancing.md)) | false +**envoy-control.envoy.snapshot.load-balancing.canary.metadata-key** | metadata that will be set for canary EDS endpoints - key (must match Envoy static `header_to_metadata` filter config, see [docs](features/load_balancing.md)) | canary +**envoy-control.envoy.snapshot.load-balancing.canary.header-value** | only when *canary header* is set to this value request will be routed to canary instances (*canary header* name is set in Envoy static config, see [docs](features/load_balancing.md)) | 1 +**envoy-control.envoy.snapshot.load-balancing.policy** | load balancing policy used for clusters. [Accepted values](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy) | LEAST_REQUEST +**envoy-control.envoy.snapshot.load-balancing.use-keys-subset-fallback-policy** | KEYS_SUBSET fallback policy is used by default when canary and service-tags are enabled. It is not supported in Envoy <= 1.12.x. Set to false for compatibility with Envoy 1.12.x | true +**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.zoneName** | a zone to which traffic will be routed if traffic splitting is enabled | "" +**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.weights-by-service-properties.** | a map that maps service name to a map [zoneName: weight] | empty map ## Routing diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index ae160ef02..df4745bc4 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -224,7 +224,7 @@ class EnvoySnapshotFactory( globalSnapshot: GlobalSnapshot, ): RouteSpecification { val trafficSplitting = properties.loadBalancing.trafficSplitting - val weights = trafficSplitting.serviceByWeightsProperties[serviceName] + val weights = trafficSplitting.weightsByService[serviceName] val enabledForDependency = globalSnapshot.endpoints[clusterName]?.endpointsList ?.any { e -> trafficSplitting.zoneName == e.locality.zone } ?: false @@ -268,18 +268,16 @@ class EnvoySnapshotFactory( // endpointsFactory.filterEndpoints() can use this cache to prevent computing the same // ClusterLoadAssignments many times - it may reduce MEM, CPU and latency if some serviceTags are // commonly used - routeSpec.clusterName to endpointsFactory.filterEndpoints(endpoints, routeSpec.settings.routingPolicy) + endpointsFactory.filterEndpoints(endpoints, routeSpec.settings.routingPolicy).let { + endpointsFactory.assignLocalityWeights(routeSpec, it) + } } - }.toMap() + } val rateLimitClusters = if (rateLimitEndpoints.isNotEmpty()) listOf(properties.rateLimit.serviceName) else emptyList() val rateLimitLoadAssignments = rateLimitClusters.mapNotNull { name -> globalSnapshot.endpoints[name] } - val secondaryLoadAssignments = endpointsFactory.getSecondaryClusterEndpoints( - egressLoadAssignments, - egressRouteSpecifications - ) - return egressLoadAssignments.values.toList() + rateLimitLoadAssignments + secondaryLoadAssignments + return egressLoadAssignments + rateLimitLoadAssignments } private fun newSnapshotForGroup( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index 5046aff1a..312df86dc 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -164,15 +164,11 @@ class CanaryProperties { class TrafficSplittingProperties { var zoneName = "" - var headerName = "" - var serviceByWeightsProperties: Map = mapOf() - var secondaryClusterSuffix = "secondary" - var aggregateClusterSuffix = "aggregate" + var weightsByService: Map = mapOf() } class ZoneWeights { - var main = 100 - var secondary = 0 + var weightByZone: Map = mapOf() } class LoadBalancingWeightsProperties { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index b0601c06d..f06ab030d 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -79,16 +79,6 @@ class EnvoyClustersFactory( companion object { private val logger by logger() - - @JvmStatic - fun getSecondaryClusterName(serviceName: String, snapshotProperties: SnapshotProperties): String { - return "$serviceName-${snapshotProperties.loadBalancing.trafficSplitting.secondaryClusterSuffix}" - } - - @JvmStatic - fun getAggregateClusterName(serviceName: String, snapshotProperties: SnapshotProperties): String { - return "$serviceName-${snapshotProperties.loadBalancing.trafficSplitting.aggregateClusterSuffix}" - } } fun getClustersForServices( @@ -207,8 +197,8 @@ class EnvoyClustersFactory( val dependencies = group.proxySettings.outgoing.getServiceDependencies().associateBy { it.service } val clustersForGroup = when (group) { - is ServicesGroup -> dependencies.flatMap { - createClusters( + is ServicesGroup -> dependencies.mapNotNull { + createCluster( group.serviceName, it.value.settings, clusters[it.key], @@ -217,9 +207,9 @@ class EnvoyClustersFactory( } is AllServicesGroup -> { - globalSnapshot.allServicesNames.flatMap { + globalSnapshot.allServicesNames.mapNotNull { val dependency = dependencies[it] - createClusters( + createCluster( group.serviceName, getDependencySettings(dependency, group), clusters[it], @@ -245,7 +235,7 @@ class EnvoyClustersFactory( dependencySettings: DependencySettings, cluster: Cluster, clusterName: String? = cluster.name - ): Cluster { + ): Cluster.Builder { val idleTimeoutPolicy = dependencySettings.timeoutPolicy.connectionIdleTimeout ?: cluster.commonHttpProtocolOptions.idleTimeout return Cluster.newBuilder(cluster) @@ -255,40 +245,36 @@ class EnvoyClustersFactory( Cluster.EdsClusterConfig.newBuilder(cluster.edsClusterConfig) .setServiceName(clusterName) ) - .build() } - private fun createSetOfClustersForGroup( + private fun createClusterWithLocalityWeightedConfigForGroup( dependencySettings: DependencySettings, cluster: Cluster - ): Collection { - val mainCluster = createClusterForGroup(dependencySettings, cluster) - val secondaryCluster = createClusterForGroup( - dependencySettings, - cluster, - getSecondaryClusterName(cluster.name, properties) - ) - val aggregateCluster = - createAggregateCluster(mainCluster.name, linkedSetOf(secondaryCluster.name, mainCluster.name)) - return listOf(mainCluster, secondaryCluster, aggregateCluster) - .onEach { - logger.debug("Created set of cluster configs for traffic splitting: {}", it.toString()) + ): Cluster.Builder { + return createClusterForGroup(dependencySettings, cluster) + .setCommonLbConfig( + Cluster.CommonLbConfig.newBuilder() + .setLocalityWeightedLbConfig(Cluster.CommonLbConfig.LocalityWeightedLbConfig.getDefaultInstance()) + .build() + ) + .also { + logger.debug("Created cluster config for traffic splitting: {}", it.toString()) } } - private fun createClusters( + private fun createCluster( serviceName: String, dependencySettings: DependencySettings, cluster: Cluster?, clusterLoadAssignment: ClusterLoadAssignment? - ): Collection { + ): Cluster? { return cluster?.let { if (enableTrafficSplitting(serviceName, clusterLoadAssignment)) { - createSetOfClustersForGroup(dependencySettings, cluster) + createClusterWithLocalityWeightedConfigForGroup(dependencySettings, cluster) } else { - listOf(createClusterForGroup(dependencySettings, cluster)) - } - } ?: listOf() + createClusterForGroup(dependencySettings, cluster) + }.build() + } } private fun enableTrafficSplitting( @@ -296,7 +282,7 @@ class EnvoyClustersFactory( clusterLoadAssignment: ClusterLoadAssignment? ): Boolean { val trafficSplitting = properties.loadBalancing.trafficSplitting - val trafficSplitEnabled = trafficSplitting.serviceByWeightsProperties.containsKey(serviceName) + val trafficSplitEnabled = trafficSplitting.weightsByService.containsKey(serviceName) return trafficSplitEnabled && hasEndpointsInZone(clusterLoadAssignment, trafficSplitting) } @@ -357,25 +343,6 @@ class EnvoyClustersFactory( } } - private fun createAggregateCluster(clusterName: String, aggregatedClusters: Collection): Cluster { - return Cluster.newBuilder() - .setName(getAggregateClusterName(clusterName, properties)) - .setConnectTimeout(Durations.fromMillis(properties.edsConnectionTimeout.toMillis())) - .setLbPolicy(Cluster.LbPolicy.CLUSTER_PROVIDED) - .setClusterType( - Cluster.CustomClusterType.newBuilder() - .setName("envoy.clusters.aggregate") - .setTypedConfig( - Any.pack( - EnvoyClusterConfig.newBuilder() - .addAllClusters(aggregatedClusters) - .build() - ) - ) - ) - .build() - } - private fun strictDnsCluster( domainDependency: DomainDependency, useTransparentProxy: Boolean diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt index 0472e0718..1d9e5d7db 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt @@ -20,7 +20,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstances import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory.Companion.getSecondaryClusterName +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.ServiceTagMetadataGenerator typealias EnvoyProxyLocality = io.envoyproxy.envoy.config.core.v3.Locality @@ -77,25 +77,30 @@ class EnvoyEndpointsFactory( } } - fun getSecondaryClusterEndpoints( - clusterLoadAssignments: Map, - egressRouteSpecifications: List - ): List { - return egressRouteSpecifications - .filterIsInstance() - .onEach { logger.debug("Traffic splitting is enabled for cluster: ${it.clusterName}") } - .mapNotNull { routeSpec -> - clusterLoadAssignments[routeSpec.clusterName]?.let { assignment -> - ClusterLoadAssignment.newBuilder(assignment) - .clearEndpoints() - .addAllEndpoints(assignment.endpointsList?.filter { e -> - e.locality.zone == properties.loadBalancing.trafficSplitting.zoneName - }) - .setClusterName(getSecondaryClusterName(routeSpec.clusterName, properties)) + fun assignLocalityWeights( + routeSpec: RouteSpecification, + loadAssignment: ClusterLoadAssignment + ): ClusterLoadAssignment { + return if (routeSpec is WeightRouteSpecification) { + ClusterLoadAssignment.newBuilder(loadAssignment) + .clearEndpoints() + .addAllEndpoints(assignWeights(loadAssignment.endpointsList, routeSpec.clusterWeights)) + .setClusterName(routeSpec.clusterName) + .build() + } else loadAssignment + } + + private fun assignWeights( + llbEndpointsList: List, weights: ZoneWeights + ): List { + return llbEndpointsList + .map { + if (weights.weightByZone.containsKey(it.locality.zone)) { + LocalityLbEndpoints.newBuilder(it) + .setLoadBalancingWeight(UInt32Value.of(weights.weightByZone[it.locality.zone] ?: 0)) .build() - } + } else it } - .filter { it.endpointsList.isNotEmpty() } } private fun filterEndpoints(loadAssignment: ClusterLoadAssignment, tag: String): ClusterLoadAssignment? { @@ -148,14 +153,16 @@ class EnvoyEndpointsFactory( serviceInstances: ServiceInstances?, zone: String, locality: Locality - ): LocalityLbEndpoints = - LocalityLbEndpoints.newBuilder() + ): LocalityLbEndpoints { + return LocalityLbEndpoints.newBuilder() .setLocality(EnvoyProxyLocality.newBuilder().setZone(zone).build()) - .addAllLbEndpoints(serviceInstances?.instances?.map { - createLbEndpoint(it, serviceInstances.serviceName, locality) - } ?: emptyList()) + .addAllLbEndpoints(serviceInstances?.instances + ?.map { + createLbEndpoint(it, serviceInstances.serviceName, locality) + } ?: emptyList()) .setPriority(toEnvoyPriority(zone, locality)) .build() + } private fun createLbEndpoint( serviceInstance: ServiceInstance, @@ -258,8 +265,7 @@ class EnvoyEndpointsFactory( false -> toEnvoyPriority(locality) }.also { logger.debug( - "Resolved lb priority to {} with zone={}, currentZone={}, priority props={}", - it, zone, currentZone, zonePriorities + "Resolved lb priority to {} with zone={}, priority props={}", it, zone, zonePriorities ) } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt index 021444709..e204ba0e9 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt @@ -20,7 +20,6 @@ import io.envoyproxy.envoy.config.route.v3.RouteAction import io.envoyproxy.envoy.config.route.v3.RouteConfiguration import io.envoyproxy.envoy.config.route.v3.RouteMatch import io.envoyproxy.envoy.config.route.v3.VirtualHost -import io.envoyproxy.envoy.config.route.v3.WeightedCluster import io.envoyproxy.envoy.extensions.retry.host.omit_canary_hosts.v3.OmitCanaryHostsPredicate import io.envoyproxy.envoy.extensions.retry.host.omit_host_metadata.v3.OmitHostMetadataConfig import io.envoyproxy.envoy.extensions.retry.host.previous_hosts.v3.PreviousHostsPredicate @@ -32,11 +31,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.groups.RetryHostPredicate import pl.allegro.tech.servicemesh.envoycontrol.logger import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.StandardRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory.Companion.getAggregateClusterName import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.ServiceTagFilterFactory -import java.lang.Boolean.TRUE import pl.allegro.tech.servicemesh.envoycontrol.groups.RetryPolicy as EnvoyControlRetryPolicy class EnvoyEgressRoutesFactory( @@ -323,7 +318,7 @@ class EnvoyEgressRoutesFactory( shouldAddRetryPolicy: Boolean = false ): RouteAction.Builder { val routeAction = RouteAction.newBuilder() - .setCluster(routeSpecification) + .setCluster(routeSpecification.clusterName) routeSpecification.settings.timeoutPolicy.let { timeoutPolicy -> timeoutPolicy.idleTimeout?.let { routeAction.setIdleTimeout(it) } @@ -346,65 +341,6 @@ class EnvoyEgressRoutesFactory( return routeAction } - - private fun RouteAction.Builder.setCluster(routeSpec: RouteSpecification): RouteAction.Builder { - return when (routeSpec) { - is WeightRouteSpecification -> { - logger.debug( - "Creating weighted cluster configuration for route spec {}, {}", - routeSpec.clusterName, - routeSpec.clusterWeights - ) - this.setWeightedClusters( - WeightedCluster.newBuilder() - .withClusterWeight(routeSpec.clusterName, routeSpec.clusterWeights.main) - .withClusterWeight( - getAggregateClusterName(routeSpec.clusterName, properties), - routeSpec.clusterWeights.secondary, - true - ) - ) - } - is StandardRouteSpecification -> { - this.setCluster(routeSpec.clusterName) - } - } - } - - private fun WeightedCluster.Builder.withClusterWeight( - clusterName: String, - weight: Int, - withHeader: Boolean = false - ): WeightedCluster.Builder { - val clusters = WeightedCluster.ClusterWeight.newBuilder() - .setName(clusterName) - .setWeight(UInt32Value.of(weight)) - .also { - if (withHeader) { - it.withHeader(properties.loadBalancing.trafficSplitting.headerName) - } - } - return this.addClusters(clusters) - } - - private fun WeightedCluster.ClusterWeight.Builder.withHeader(key: String?): WeightedCluster.ClusterWeight.Builder { - key?.takeIf { it.isNotBlank() } - ?.let { - this.addResponseHeadersToAdd(buildHeader(key)) - } - return this - } - - private fun buildHeader(key: String): HeaderValueOption.Builder { - return HeaderValueOption.newBuilder() - .setHeader( - HeaderValue.newBuilder() - .setKey(key) - .setValue(TRUE.toString()) - ) - .setAppendAction(HeaderValueOption.HeaderAppendAction.OVERWRITE_IF_EXISTS_OR_ADD) - .setKeepEmptyValue(false) - } } class RequestPolicyMapper private constructor() { diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java index 3c5568f71..1a70af568 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java @@ -40,7 +40,6 @@ public class SimpleCacheTest { private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); protected static final String CLUSTER_NAME = "cluster0"; - private static final String SECONDARY_CLUSTER_NAME = "cluster1"; private static final String LISTENER_NAME = "listener0"; private static final String ROUTE_NAME = "route0"; private static final String SECRET_NAME = "secret0"; @@ -65,10 +64,8 @@ public class SimpleCacheTest { VERSION2); protected static final Snapshot MULTIPLE_RESOURCES_SNAPSHOT2 = Snapshot.create( - ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build(), - Cluster.newBuilder().setName(SECONDARY_CLUSTER_NAME).build()), - ImmutableList.of(ClusterLoadAssignment.newBuilder().setClusterName(CLUSTER_NAME).build(), - ClusterLoadAssignment.newBuilder().setClusterName(SECONDARY_CLUSTER_NAME).build()), + ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build()), + ImmutableList.of(ClusterLoadAssignment.newBuilder().setClusterName(CLUSTER_NAME).build()), ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 53fdbed99..da26a343c 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -44,26 +44,18 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_PORT import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_PORT -import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE +import pl.allegro.tech.servicemesh.envoycontrol.utils.SNAPSHOT_PROPERTIES_WITH_WEIGHTS +import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.createCluster import pl.allegro.tech.servicemesh.envoycontrol.utils.createClusterConfigurations import pl.allegro.tech.servicemesh.envoycontrol.utils.createEndpoints +import pl.allegro.tech.servicemesh.envoycontrol.utils.zoneWeights class EnvoySnapshotFactoryTest { companion object { - const val MAIN_CLUSTER_NAME = "service-name-2" - const val SECONDARY_CLUSTER_NAME = "service-name-2-secondary" - const val AGGREGATE_CLUSTER_NAME = "service-name-2-aggregate" const val SERVICE_NAME_2 = "service-name-2" } - private val snapshotPropertiesWithWeights = SnapshotProperties().also { - it.loadBalancing.trafficSplitting.serviceByWeightsProperties = mapOf( - DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS - ) - it.loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE - } - @Test fun shouldGetSnapshotListenersForGroupWhenDynamicListenersEnabled() { // given @@ -230,93 +222,99 @@ class EnvoySnapshotFactoryTest { } @Test - fun `should create weighted snapshot clusters`() { - // given - val envoySnapshotFactory = createSnapshotFactory(snapshotPropertiesWithWeights) - val cluster1 = createCluster(snapshotPropertiesWithWeights, clusterName = DEFAULT_SERVICE_NAME) - val cluster2 = - createCluster(snapshotPropertiesWithWeights, clusterName = SERVICE_NAME_2) + fun `should get regular snapshot cluster when there are no traffic splitting settings for zone`() { + val snapshotProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS.also { + it.loadBalancing.trafficSplitting.zoneName = "not-matching-dc" + } + val envoySnapshotFactory = createSnapshotFactory(snapshotProperties) + val cluster1 = createCluster(snapshotProperties, clusterName = DEFAULT_SERVICE_NAME) + val cluster2 = createCluster(snapshotProperties, clusterName = SERVICE_NAME_2) val group: Group = createServicesGroup( - dependencies = arrayOf(cluster2.name to null), - snapshotProperties = snapshotPropertiesWithWeights + dependencies = arrayOf(SERVICE_NAME_2 to null), + snapshotProperties = snapshotProperties ) val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) + val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) - // when + assertThat(snapshot.clusters().resources().values) + .allSatisfy { !it.hasCommonLbConfig() || !it.commonLbConfig.hasLocalityWeightedLbConfig() } + .hasSize(1) + } + + @Test + fun `should get cluster with locality weighted config when there are traffic splitting settings for zone`() { + val envoySnapshotFactory = createSnapshotFactory(SNAPSHOT_PROPERTIES_WITH_WEIGHTS) + val cluster1 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = DEFAULT_SERVICE_NAME) + val cluster2 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = SERVICE_NAME_2) + val group: Group = createServicesGroup( + dependencies = arrayOf(SERVICE_NAME_2 to null), + snapshotProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS + ) + val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) + val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) + + assertThat(snapshot.clusters().resources().values) + .anySatisfy { it.hasCommonLbConfig() && it.commonLbConfig.hasLocalityWeightedLbConfig() } + .hasSize(1) + } + + @Test + fun `should get weighted locality lb endpoints when there are traffic splitting settings for zone`() { + val envoySnapshotFactory = createSnapshotFactory(SNAPSHOT_PROPERTIES_WITH_WEIGHTS) + val cluster1 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = DEFAULT_SERVICE_NAME) + val cluster2 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = SERVICE_NAME_2) + val group: Group = createServicesGroup( + dependencies = arrayOf(SERVICE_NAME_2 to null), + snapshotProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS + ) + val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) - // then - assertThat(snapshot.clusters().resources()) - .containsKey(MAIN_CLUSTER_NAME) - .containsKey(SECONDARY_CLUSTER_NAME) - .containsKey(AGGREGATE_CLUSTER_NAME) - assertThat(snapshot.endpoints().resources().values) - .anySatisfy { - assertThat(it.clusterName).isEqualTo(MAIN_CLUSTER_NAME) - assertThat(it.endpointsList) - .anyMatch { e -> e.locality.zone == CURRENT_ZONE } - .anyMatch { e -> e.locality.zone == TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE } - } assertThat(snapshot.endpoints().resources().values) .anySatisfy { - assertThat(it.clusterName).isEqualTo(SECONDARY_CLUSTER_NAME) assertThat(it.endpointsList) - .allMatch { e -> e.locality.zone == TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE } + .anySatisfy { e -> + e.locality.zone == TRAFFIC_SPLITTING_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] + } + .anySatisfy { e -> + e.locality.zone == CURRENT_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[CURRENT_ZONE] + } + .hasSize(2) } } @Test - fun `should get regular snapshot clusters when traffic splitting zone condition isn't complied`() { - // given - val defaultProperties = SnapshotProperties().also { - it.dynamicListeners.enabled = false - it.loadBalancing.trafficSplitting.serviceByWeightsProperties = mapOf( - DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS + fun `should not set weight to locality lb endpoints when there are no matching weight settings`() { + val defaultProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS.also { + it.loadBalancing.trafficSplitting.weightsByService = mapOf( + DEFAULT_SERVICE_NAME to zoneWeights(mapOf(CURRENT_ZONE to 60)) ) - it.loadBalancing.trafficSplitting.zoneName = "not-matching-dc" } val envoySnapshotFactory = createSnapshotFactory(defaultProperties) val cluster1 = createCluster(defaultProperties, clusterName = DEFAULT_SERVICE_NAME) val cluster2 = createCluster(defaultProperties, clusterName = SERVICE_NAME_2) val group: Group = createServicesGroup( - dependencies = arrayOf(cluster2.name to null), + dependencies = arrayOf(SERVICE_NAME_2 to null), snapshotProperties = defaultProperties ) val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) - - // when val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) - // then - assertThat(snapshot.clusters().resources()) - .containsKey(MAIN_CLUSTER_NAME) - .doesNotContainKey(SECONDARY_CLUSTER_NAME) - .doesNotContainKey(AGGREGATE_CLUSTER_NAME) - } - - @Test - fun `should create weighted snapshot clusters for wildcard dependencies`() { - // given - val envoySnapshotFactory = createSnapshotFactory(snapshotPropertiesWithWeights) - val cluster1 = createCluster(snapshotPropertiesWithWeights, clusterName = DEFAULT_SERVICE_NAME) - val cluster2 = createCluster(snapshotPropertiesWithWeights, clusterName = SERVICE_NAME_2) - val wildcardTimeoutPolicy = outgoingTimeoutPolicy(connectionIdleTimeout = 12) - - val group: Group = createAllServicesGroup( - dependencies = arrayOf("*" to wildcardTimeoutPolicy), - snapshotProperties = snapshotPropertiesWithWeights, - defaultServiceSettings = DependencySettings(), - ) - val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) - - // when - val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) - - // then - assertThat(snapshot.clusters().resources()) - .containsKey(MAIN_CLUSTER_NAME) - .containsKey(SECONDARY_CLUSTER_NAME) - .containsKey(AGGREGATE_CLUSTER_NAME) + assertThat(snapshot.endpoints().resources().values) + .anySatisfy { + assertThat(it.endpointsList) + .anySatisfy { e -> + e.locality.zone == CURRENT_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[CURRENT_ZONE] + } + .anySatisfy { e -> + e.locality.zone == TRAFFIC_SPLITTING_ZONE && + !e.hasLoadBalancingWeight() + } + .hasSize(2) + } } @Test diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt index 2a1d8917a..9c82af855 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt @@ -8,14 +8,11 @@ import org.junit.jupiter.api.Test import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties -import pl.allegro.tech.servicemesh.envoycontrol.utils.AGGREGATE_CLUSTER_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME1 import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME2 import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_CLUSTER_WEIGHTS import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_SERVICE_NAME -import pl.allegro.tech.servicemesh.envoycontrol.utils.MAIN_CLUSTER_NAME -import pl.allegro.tech.servicemesh.envoycontrol.utils.SECONDARY_CLUSTER_NAME -import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE +import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.createAllServicesGroup import pl.allegro.tech.servicemesh.envoycontrol.utils.createCluster import pl.allegro.tech.servicemesh.envoycontrol.utils.createClusterConfigurations @@ -28,10 +25,10 @@ internal class EnvoyClustersFactoryTest { companion object { private val factory = EnvoyClustersFactory(SnapshotProperties()) private val snapshotPropertiesWithWeights = SnapshotProperties().apply { - loadBalancing.trafficSplitting.serviceByWeightsProperties = mapOf( + loadBalancing.trafficSplitting.weightsByService = mapOf( DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS ) - loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE + loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_ZONE } } @@ -97,7 +94,7 @@ internal class EnvoyClustersFactoryTest { } @Test - fun `should get clusters for group with weighted and aggregate clusters`() { + fun `should get cluster with locality weighted config for group clusters`() { val cluster1 = createCluster(snapshotPropertiesWithWeights, CLUSTER_NAME1) val factory = EnvoyClustersFactory(snapshotPropertiesWithWeights) val result = factory.getClustersForGroup( @@ -110,15 +107,9 @@ internal class EnvoyClustersFactoryTest { ) assertThat(result) .anySatisfy { - assertThat(it.name).isEqualTo(MAIN_CLUSTER_NAME) + assertThat(it.name).isEqualTo(CLUSTER_NAME1) assertThat(it.edsClusterConfig).isEqualTo(cluster1.edsClusterConfig) - } - .anySatisfy { - assertThat(it.name).isEqualTo(SECONDARY_CLUSTER_NAME) - } - .anySatisfy { - assertThat(it.name).isEqualTo(AGGREGATE_CLUSTER_NAME) - assertThat(it.clusterType.typedConfig.isInitialized).isTrue() + assertThat(it.commonLbConfig.localityWeightedLbConfig).isNotNull } } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt index 78fef6122..eb1e693e8 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt @@ -8,7 +8,6 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource -import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.groups.RoutingPolicy import pl.allegro.tech.servicemesh.envoycontrol.services.ClusterState import pl.allegro.tech.servicemesh.envoycontrol.services.Locality @@ -19,12 +18,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceName import pl.allegro.tech.servicemesh.envoycontrol.services.ServicesState import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingPriorityProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.TrafficSplittingProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights -import pl.allegro.tech.servicemesh.envoycontrol.utils.zoneWeights import java.util.concurrent.ConcurrentHashMap import java.util.stream.Stream @@ -57,12 +51,6 @@ internal class EnvoyEndpointsFactoryTest { private val serviceName = "service-one" - private val secondaryClusterName = "service-one-secondary" - - private val serviceName2 = "service-two" - - private val defaultWeights = zoneWeights(50, 50) - private val defaultZone = "DC1" private val endpointsFactory = EnvoyEndpointsFactory( @@ -366,122 +354,6 @@ internal class EnvoyEndpointsFactoryTest { ) } - @Test - fun `should create secondary cluster endpoints`() { - val multiClusterState = MultiClusterState( - listOf( - clusterState(cluster = "DC1"), - clusterState(cluster = "DC2"), - clusterState(cluster = "DC1", serviceName = serviceName2), - clusterState(cluster = "DC2", serviceName = serviceName2), - ) - ) - - val services = setOf(serviceName, serviceName2) - val envoyEndpointsFactory = EnvoyEndpointsFactory( - snapshotPropertiesWithTrafficSplitting( - mapOf(serviceName to defaultWeights) - ), - currentZone = defaultZone - ) - val loadAssignments = envoyEndpointsFactory - .createLoadAssignment(services, multiClusterState) - .associateBy { it.clusterName } - - val result = envoyEndpointsFactory.getSecondaryClusterEndpoints( - loadAssignments, - services.map { it.toRouteSpecification() } - ) - assertThat(result).hasSize(2) - .anySatisfy { x -> assertThat(x.clusterName).isEqualTo(secondaryClusterName) } - .allSatisfy { x -> assertThat(x.endpointsList).allMatch { it.locality.zone == defaultZone } } - } - - @Test - fun `should get empty secondary cluster endpoints for route spec with no weights`() { - val multiClusterState = MultiClusterState( - listOf( - clusterState(cluster = defaultZone), - clusterState(cluster = defaultZone, serviceName = serviceName2), - ) - ) - val services = setOf(serviceName, serviceName2) - val envoyEndpointsFactory = EnvoyEndpointsFactory( - snapshotPropertiesWithTrafficSplitting( - mapOf(serviceName to defaultWeights) - ), - currentZone = defaultZone - ) - val loadAssignments = envoyEndpointsFactory.createLoadAssignment( - services, - multiClusterState - ).associateBy { it.clusterName } - - val result = envoyEndpointsFactory.getSecondaryClusterEndpoints( - loadAssignments, - listOf(serviceName.toRouteSpecification()) - ) - assertThat(result).allSatisfy { x -> - assertThat(x.clusterName) - .isEqualTo(secondaryClusterName) - } - } - - @Test - fun `should get empty secondary cluster endpoints for route spec with no such cluster`() { - val multiClusterState = MultiClusterState( - listOf( - clusterState(cluster = defaultZone), - clusterState(cluster = defaultZone, serviceName = serviceName2), - ) - ) - val services = setOf(serviceName, serviceName2) - val envoyEndpointsFactory = EnvoyEndpointsFactory( - snapshotPropertiesWithTrafficSplitting( - mapOf(serviceName to defaultWeights) - ), - currentZone = defaultZone - ) - val loadAssignments = envoyEndpointsFactory.createLoadAssignment( - services, - multiClusterState - ).associateBy { it.clusterName } - - val result = envoyEndpointsFactory.getSecondaryClusterEndpoints( - loadAssignments, - listOf("some-other-service-name".toRouteSpecification()) - ) - assertThat(result).isEmpty() - } - - @Test - fun `should get empty secondary cluster endpoints when none comply zone condition`() { - val multiClusterState = MultiClusterState( - listOf( - clusterState(cluster = defaultZone), - clusterState(cluster = defaultZone, serviceName = serviceName2), - ) - ) - val services = setOf(serviceName, serviceName2) - val envoyEndpointsFactory = EnvoyEndpointsFactory( - snapshotPropertiesWithTrafficSplitting( - mapOf(serviceName to defaultWeights), - zone = "DC2" - ), - currentZone = defaultZone - ) - val loadAssignments = envoyEndpointsFactory.createLoadAssignment( - services, - multiClusterState - ).associateBy { it.clusterName } - - val result = envoyEndpointsFactory.getSecondaryClusterEndpoints( - loadAssignments, - listOf(serviceName.toRouteSpecification()) - ) - assertThat(result).isEmpty() - } - private fun List.assertHasLoadAssignment(map: Map) { assertThat(this) .isNotEmpty() @@ -539,21 +411,6 @@ internal class EnvoyEndpointsFactoryTest { } } - private fun snapshotPropertiesWithTrafficSplitting( - serviceByWeights: Map, - zone: String = defaultZone - ) = - SnapshotProperties().apply { - loadBalancing.trafficSplitting = TrafficSplittingProperties().apply { - zoneName = zone - serviceByWeightsProperties = serviceByWeights - } - } - - private fun String.toRouteSpecification(weights: ZoneWeights = defaultWeights): RouteSpecification { - return WeightRouteSpecification(this, listOf(), DependencySettings(), weights) - } - private fun String.toClusterLoadAssignment(): ClusterLoadAssignment = ClusterLoadAssignment.newBuilder() .also { builder -> JsonFormat.parser().merge(this, builder) } .build() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactoryTest.kt index 2044699df..5017e983a 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactoryTest.kt @@ -2,13 +2,10 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes import com.google.protobuf.util.Durations import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotNull -import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.groups.Outgoing import pl.allegro.tech.servicemesh.envoycontrol.groups.RetryPolicy -import pl.allegro.tech.servicemesh.envoycontrol.groups.hasClusterThat import pl.allegro.tech.servicemesh.envoycontrol.groups.hasCustomIdleTimeout import pl.allegro.tech.servicemesh.envoycontrol.groups.hasCustomRequestTimeout import pl.allegro.tech.servicemesh.envoycontrol.groups.hasHostRewriteHeader @@ -292,47 +289,4 @@ internal class EnvoyEgressRoutesFactoryTest { defaultRoute.matchingOnPrefix("/") } } - - @Test - fun `should add traffic splitting header for secondary weighted cluster`() { - - val expectedHeaderKey = "test-header" - val routesFactory = EnvoyEgressRoutesFactory(SnapshotProperties().apply { - loadBalancing.trafficSplitting.headerName = expectedHeaderKey - }) - - val routeConfig = routesFactory.createEgressRouteConfig( - "client1", - weightedClusters, - false - ) - - routeConfig - .hasClusterThat("srv1-aggregate") { - assertNotNull(this) - assertNotNull(this!!.responseHeadersToAddList.find { it.header.key == expectedHeaderKey }) - }.hasClusterThat("srv1") { - assertNotNull(this) - assertTrue(this!!.responseHeadersToAddList.isEmpty()) - } - } - - @Test - fun `should not add traffic splitting header if header key is not set`() { - val routesFactory = EnvoyEgressRoutesFactory(SnapshotProperties()) - val routeConfig = routesFactory.createEgressRouteConfig( - "client1", - weightedClusters, - false - ) - - routeConfig - .hasClusterThat("srv1-aggregate") { - assertNotNull(this) - assertTrue(this!!.responseHeadersToAddList.isEmpty()) - }.hasClusterThat("srv1") { - assertNotNull(this) - assertNotNull(this!!.responseHeadersToAddList.isEmpty()) - } - } } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt index a47cf5fb6..38eaf4af5 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt @@ -17,7 +17,7 @@ fun createLoadAssignments(clusters: List): List fun createEndpoints(): List = listOf( createEndpoint(CURRENT_ZONE), - createEndpoint(TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE) + createEndpoint(TRAFFIC_SPLITTING_ZONE) ) fun createEndpoint(zone: String): LocalityLbEndpoints { diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt index fc2ecdaea..4dd10f126 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt @@ -1,5 +1,6 @@ package pl.allegro.tech.servicemesh.envoycontrol.utils +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights const val INGRESS_HOST = "ingress-host" @@ -12,15 +13,19 @@ const val DEFAULT_DISCOVERY_SERVICE_NAME = "discovery-service-name" const val CLUSTER_NAME = "cluster-name" const val CLUSTER_NAME1 = "cluster-1" const val CLUSTER_NAME2 = "cluster-2" -const val MAIN_CLUSTER_NAME = "cluster-1" -const val SECONDARY_CLUSTER_NAME = "cluster-1-secondary" -const val AGGREGATE_CLUSTER_NAME = "cluster-1-aggregate" -const val TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE = "dc2" +const val TRAFFIC_SPLITTING_ZONE = "dc2" const val CURRENT_ZONE = "dc1" -val DEFAULT_CLUSTER_WEIGHTS = zoneWeights(50, 50) +val DEFAULT_CLUSTER_WEIGHTS = zoneWeights(mapOf(CURRENT_ZONE to 60, TRAFFIC_SPLITTING_ZONE to 40)) -fun zoneWeights(main: Int, secondary: Int) = ZoneWeights().also { - it.main = main - it.secondary = secondary +val SNAPSHOT_PROPERTIES_WITH_WEIGHTS = SnapshotProperties().also { + it.dynamicListeners.enabled = false + it.loadBalancing.trafficSplitting.weightsByService = mapOf( + DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS + ) + it.loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_ZONE +} + +fun zoneWeights(weightByZone: Map) = ZoneWeights().also { + it.weightByZone = weightByZone } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt index 564660fce..c5953a0dd 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt @@ -11,7 +11,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension import verifyCallsCountCloseTo -import verifyCallsCountGreaterThan import verifyIsReachable import java.time.Duration @@ -20,19 +19,29 @@ class WeightedClustersRoutingTest { private const val forceTrafficZone = "dc2" private val properties = mapOf( + "pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory" to "DEBUG", + "pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory" to "DEBUG", "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0), "envoy-control.sync.enabled" to true, - "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to forceTrafficZone, - "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties.$serviceName.main" to 90, - "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties.$serviceName.secondary" to 10, - "envoy-control.envoy.snapshot.loadBalancing.priorities.zonePriorities" to mapOf( + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zoneName" to forceTrafficZone, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc1" to 30, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc2" to 10, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc3" to 1, + "envoy-control.envoy.snapshot.load-balancing.priorities.zonePriorities" to mapOf( "dc1" to mapOf( "dc1" to 0, - "dc2" to 1 + "dc2" to 0, + "dc3" to 3, ), "dc2" to mapOf( - "dc1" to 1, + "dc1" to 0, "dc2" to 0, + "dc3" to 3, + ), + "dc3" to mapOf( + "dc1" to 3, + "dc2" to 3, + "dc3" to 0, ), ) ) @@ -62,6 +71,11 @@ class WeightedClustersRoutingTest { val envoyControl2 = EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) + @JvmField + @RegisterExtension + val envoyControl3 = + EnvoyControlClusteredExtension(consul.serverThird, { properties }, listOf(consul)) + @JvmField @RegisterExtension val echoServiceDC1 = EchoServiceExtension() @@ -74,12 +88,20 @@ class WeightedClustersRoutingTest { @RegisterExtension val upstreamServiceDC2 = EchoServiceExtension() + @JvmField + @RegisterExtension + val upstreamServiceDC3 = EchoServiceExtension() + @JvmField @RegisterExtension val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) @JvmField @RegisterExtension val echoEnvoyDC2 = EnvoyExtension(envoyControl2) + + @JvmField + @RegisterExtension + val echoEnvoyDC3 = EnvoyExtension(envoyControl3) } @Test @@ -93,8 +115,9 @@ class WeightedClustersRoutingTest { echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) - .verifyCallsCountCloseTo(upstreamServiceDC1, 90) - .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) + .verifyCallsCountCloseTo(upstreamServiceDC1, 75) + .verifyCallsCountCloseTo(upstreamServiceDC2, 25) + println("snapshot: " + envoyControl.app.getGlobalSnapshot(false).toString()) } @Test @@ -108,7 +131,7 @@ class WeightedClustersRoutingTest { echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") - .verifyCallsCountCloseTo(upstreamServiceDC1, 90) - .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) + .verifyCallsCountCloseTo(upstreamServiceDC1, 75) + .verifyCallsCountCloseTo(upstreamServiceDC2, 25) } }