From ced0295300551d64366e37c99b27bd9149a74805 Mon Sep 17 00:00:00 2001 From: "nastassia.dailidava" Date: Wed, 27 Mar 2024 19:04:27 +0100 Subject: [PATCH] #624 Set flat priority only for services with traffic splitting --- CHANGELOG.md | 6 + .../servicemesh/envoycontrol/ControlPlane.kt | 2 +- .../snapshot/EnvoySnapshotFactory.kt | 1 - .../snapshot/SnapshotProperties.kt | 1 + .../resource/clusters/EnvoyClustersFactory.kt | 17 ++- .../endpoints/EnvoyEndpointsFactory.kt | 44 ++++++-- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 15 ++- .../snapshot/SnapshotUpdaterTest.kt | 2 +- .../clusters/EnvoyClustersFactoryTest.kt | 5 +- .../endpoints/EnvoyEndpointsFactoryTest.kt | 62 ++++++++++- .../envoycontrol/utils/EndpointsOperations.kt | 1 + .../envoycontrol/utils/TestData.kt | 3 + ...t => LocalityWeightedLoadBalancingTest.kt} | 103 ++++++++++-------- ...eightedLoadBalancingUnlistedServiceTest.kt | 92 ++++++++++++++++ .../trafficsplitting/TrafficSplitting.kt | 40 +++++-- 15 files changed, 305 insertions(+), 89 deletions(-) rename envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/{WeightedClustersRoutingTest.kt => LocalityWeightedLoadBalancingTest.kt} (54%) create mode 100644 envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index f60061aa0..fef02f9a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Lists all changes with user impact. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## [0.20.13] +### Changed +- Added setting: "zonesAllowingTrafficSplitting", so changes in a config would be made only for envoys in that zone +- Fixed setting priority for traffic splitting endpoints, they will be duplicated with higher priorities + ## [0.20.12] ### Changed - Added "trackRemaining" flag to enable possibility of tracking additional circuit breaker metrics diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt index 6905f9f3a..d03592962 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt @@ -169,7 +169,7 @@ class ControlPlane private constructor( val envoySnapshotFactory = EnvoySnapshotFactory( ingressRoutesFactory = EnvoyIngressRoutesFactory(snapshotProperties, envoyHttpFilters, currentZone), egressRoutesFactory = EnvoyEgressRoutesFactory(snapshotProperties), - clustersFactory = EnvoyClustersFactory(snapshotProperties), + clustersFactory = EnvoyClustersFactory(snapshotProperties, currentZone), endpointsFactory = EnvoyEndpointsFactory( snapshotProperties, ServiceTagMetadataGenerator(snapshotProperties.routing.serviceTags), diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index df4745bc4..c56165dad 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -273,7 +273,6 @@ class EnvoySnapshotFactory( } } } - val rateLimitClusters = if (rateLimitEndpoints.isNotEmpty()) listOf(properties.rateLimit.serviceName) else emptyList() val rateLimitLoadAssignments = rateLimitClusters.mapNotNull { name -> globalSnapshot.endpoints[name] } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index b85ffe283..9eae0d922 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -166,6 +166,7 @@ class TrafficSplittingProperties { var zoneName = "" var headerName = "" var weightsByService: Map = mapOf() + var zonesAllowingTrafficSplitting = listOf() } class ZoneWeights { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index 846b6381c..68776dcf3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -52,13 +52,13 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.OAuthProvider import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.Threshold -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.TrafficSplittingProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.SanUriMatcherFactory typealias EnvoyClusterConfig = io.envoyproxy.envoy.extensions.clusters.aggregate.v3.ClusterConfig class EnvoyClustersFactory( - private val properties: SnapshotProperties + private val properties: SnapshotProperties, + private val currentZone: String ) { private val httpProtocolOptions: HttpProtocolOptions = HttpProtocolOptions.newBuilder().setIdleTimeout( Durations.fromMillis(properties.egress.commonHttp.connectionIdleTimeout.toMillis()) @@ -283,15 +283,14 @@ class EnvoyClustersFactory( ): Boolean { val trafficSplitting = properties.loadBalancing.trafficSplitting val trafficSplitEnabled = trafficSplitting.weightsByService.containsKey(serviceName) - return trafficSplitEnabled && hasEndpointsInZone(clusterLoadAssignment, trafficSplitting) + val allowed = clusterLoadAssignment != null && + properties.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting.contains(currentZone) + if (serviceName == "varnish" || serviceName == "service-mesh-service-second") { + logger.info("trafficSplitEnabled $trafficSplitEnabled allowed $allowed, $currentZone") + } + return trafficSplitEnabled && allowed } - private fun hasEndpointsInZone( - clusterLoadAssignment: ClusterLoadAssignment?, - trafficSplitting: TrafficSplittingProperties - ) = clusterLoadAssignment?.endpointsList - ?.any { e -> trafficSplitting.zoneName == e.locality.zone && e.lbEndpointsCount > 0 } ?: false - private fun shouldAddDynamicForwardProxyCluster(group: Group) = group.proxySettings.outgoing.getDomainPatternDependencies().isNotEmpty() diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt index 1d9e5d7db..b6ab798e3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt @@ -34,6 +34,8 @@ class EnvoyEndpointsFactory( ) { companion object { private val logger by logger() + private const val HIGHEST_PRIORITY = 0 + private const val DEFAULT_WEIGHT = 1 } fun createLoadAssignment( @@ -84,22 +86,48 @@ class EnvoyEndpointsFactory( return if (routeSpec is WeightRouteSpecification) { ClusterLoadAssignment.newBuilder(loadAssignment) .clearEndpoints() - .addAllEndpoints(assignWeights(loadAssignment.endpointsList, routeSpec.clusterWeights)) + .addAllEndpoints( + assignWeightsAndDuplicateEndpoints( + loadAssignment.endpointsList, + routeSpec.clusterWeights + ) + ) .setClusterName(routeSpec.clusterName) .build() } else loadAssignment } - private fun assignWeights( - llbEndpointsList: List, weights: ZoneWeights + private fun assignWeightsAndDuplicateEndpoints( + llbEndpointsList: List, + weights: ZoneWeights ): List { + if (properties.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting.contains(currentZone)) { + val endpoints = llbEndpointsList + .map { + if (weights.weightByZone.containsKey(it.locality.zone)) { + LocalityLbEndpoints.newBuilder(it) + .setLoadBalancingWeight( + UInt32Value.of( + weights.weightByZone[it.locality.zone] ?: DEFAULT_WEIGHT + ) + ) + .build() + } else it + } + return overrideTrafficSplittingZoneEndpointsPriority(endpoints) + endpoints + } return llbEndpointsList + } + + private fun overrideTrafficSplittingZoneEndpointsPriority( + endpoints: List + ): List { + return endpoints + .filter { properties.loadBalancing.trafficSplitting.zoneName == it.locality.zone } .map { - if (weights.weightByZone.containsKey(it.locality.zone)) { - LocalityLbEndpoints.newBuilder(it) - .setLoadBalancingWeight(UInt32Value.of(weights.weightByZone[it.locality.zone] ?: 0)) - .build() - } else it + LocalityLbEndpoints.newBuilder(it) + .setPriority(HIGHEST_PRIORITY) + .build() } } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 5ba51e613..4dfc366be 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -38,9 +38,11 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_CLUSTER_WEIGHTS import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_DISCOVERY_SERVICE_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_IDLE_TIMEOUT +import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_PRIORITY import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_SERVICE_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_PORT +import pl.allegro.tech.servicemesh.envoycontrol.utils.HIGHEST_PRIORITY import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_PORT import pl.allegro.tech.servicemesh.envoycontrol.utils.SNAPSHOT_PROPERTIES_WITH_WEIGHTS @@ -275,13 +277,18 @@ class EnvoySnapshotFactoryTest { assertThat(it.endpointsList) .anySatisfy { e -> e.locality.zone == TRAFFIC_SPLITTING_ZONE && - e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] && + e.priority == DEFAULT_PRIORITY + }.anySatisfy { e -> + e.locality.zone == TRAFFIC_SPLITTING_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] && + e.priority == HIGHEST_PRIORITY } .anySatisfy { e -> e.locality.zone == CURRENT_ZONE && e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[CURRENT_ZONE] } - .hasSize(2) + .hasSize(3) } } @@ -313,7 +320,7 @@ class EnvoySnapshotFactoryTest { e.locality.zone == TRAFFIC_SPLITTING_ZONE && !e.hasLoadBalancingWeight() } - .hasSize(2) + .hasSize(3) } } @@ -457,7 +464,7 @@ class EnvoySnapshotFactoryTest { CURRENT_ZONE ) val egressRoutesFactory = EnvoyEgressRoutesFactory(properties) - val clustersFactory = EnvoyClustersFactory(properties) + val clustersFactory = EnvoyClustersFactory(properties, CURRENT_ZONE) val endpointsFactory = EnvoyEndpointsFactory(properties, ServiceTagMetadataGenerator(), CURRENT_ZONE) val envoyHttpFilters = EnvoyHttpFilters.defaultFilters(properties) val listenersFactory = EnvoyListenersFactory(properties, envoyHttpFilters) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt index 916357f72..247f879e3 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt @@ -1312,7 +1312,7 @@ class SnapshotUpdaterTest { EnvoySnapshotFactory( ingressRoutesFactory = EnvoyIngressRoutesFactory(snapshotProperties, currentZone = CURRENT_ZONE), egressRoutesFactory = EnvoyEgressRoutesFactory(snapshotProperties), - clustersFactory = EnvoyClustersFactory(snapshotProperties), + clustersFactory = EnvoyClustersFactory(snapshotProperties, CURRENT_ZONE), endpointsFactory = EnvoyEndpointsFactory( snapshotProperties, ServiceTagMetadataGenerator(snapshotProperties.routing.serviceTags), CURRENT_ZONE ), diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt index 9c82af855..d91520b18 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt @@ -10,6 +10,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME1 import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME2 +import pl.allegro.tech.servicemesh.envoycontrol.utils.CURRENT_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_CLUSTER_WEIGHTS import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_SERVICE_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_ZONE @@ -23,7 +24,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.createServicesGroup internal class EnvoyClustersFactoryTest { companion object { - private val factory = EnvoyClustersFactory(SnapshotProperties()) + private val factory = EnvoyClustersFactory(SnapshotProperties(), CURRENT_ZONE) private val snapshotPropertiesWithWeights = SnapshotProperties().apply { loadBalancing.trafficSplitting.weightsByService = mapOf( DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS @@ -96,7 +97,7 @@ internal class EnvoyClustersFactoryTest { @Test fun `should get cluster with locality weighted config for group clusters`() { val cluster1 = createCluster(snapshotPropertiesWithWeights, CLUSTER_NAME1) - val factory = EnvoyClustersFactory(snapshotPropertiesWithWeights) + val factory = EnvoyClustersFactory(snapshotPropertiesWithWeights, CURRENT_ZONE) val result = factory.getClustersForGroup( createServicesGroup( snapshotProperties = snapshotPropertiesWithWeights, diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt index eb1e693e8..de151d627 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt @@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource +import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.groups.RoutingPolicy import pl.allegro.tech.servicemesh.envoycontrol.services.ClusterState import pl.allegro.tech.servicemesh.envoycontrol.services.Locality @@ -19,6 +20,9 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServicesState import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingPriorityProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.TrafficSplittingProperties +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights import java.util.concurrent.ConcurrentHashMap import java.util.stream.Stream @@ -33,13 +37,13 @@ internal class EnvoyEndpointsFactoryTest { ), "DC2" to mapOf( "DC1" to 1, - "DC2" to 2, - "DC3" to 3 + "DC2" to 0, + "DC3" to 2 ), "DC3" to mapOf( - "DC1" to 2, - "DC2" to 3, - "DC3" to 4 + "DC1" to 1, + "DC2" to 2, + "DC3" to 0 ) ) @@ -52,6 +56,7 @@ internal class EnvoyEndpointsFactoryTest { private val serviceName = "service-one" private val defaultZone = "DC1" + private val trafficSplittingZone = "DC2" private val endpointsFactory = EnvoyEndpointsFactory( SnapshotProperties().apply { @@ -77,6 +82,31 @@ internal class EnvoyEndpointsFactoryTest { ) ) + private val defaultZoneWeights = mapOf( + "DC1" to 100, + "DC2" to 10, + "DC3" to 2 + ) + + private val snapshotPropertiesWithWeights = SnapshotProperties().apply { + loadBalancing = LoadBalancingProperties() + .apply { + priorities = LoadBalancingPriorityProperties() + .apply { zonePriorities = dcPriorityProperties } + trafficSplitting = TrafficSplittingProperties() + .apply { + zoneName = trafficSplittingZone + zonesAllowingTrafficSplitting = listOf("DC1") + weightsByService = mapOf( + serviceName to ZoneWeights() + .apply { + weightByZone = defaultZoneWeights + } + ) + } + } + } + // language=json private val globalLoadAssignmentJson = """{ "cluster_name": "lorem-service", @@ -354,6 +384,28 @@ internal class EnvoyEndpointsFactoryTest { ) } + @Test + fun `should override priority and duplicate endpoints for traffic splitting zone`() { + val envoyEndpointsFactory = EnvoyEndpointsFactory( + snapshotPropertiesWithWeights, + currentZone = "DC1" + ) + val loadAssignments = envoyEndpointsFactory.createLoadAssignment(setOf(serviceName), multiClusterStateDC1Local) + var resultLoadAssignment = envoyEndpointsFactory.assignLocalityWeights( + WeightRouteSpecification( + serviceName, listOf(), DependencySettings(), ZoneWeights().apply { weightByZone = defaultZoneWeights } + ), + loadAssignments[0] + ) + + assertThat(resultLoadAssignment.endpointsList).hasSize(dcPriorityProperties.size + 1) + assertThat(resultLoadAssignment.endpointsList) + .anySatisfy { it.hasZoneWithPriority("DC2", 1) } + .anySatisfy { it.hasZoneWithPriority("DC2", 0) } + .anySatisfy { it.hasZoneWithPriority("DC1", 0) } + .anySatisfy { it.hasZoneWithPriority("DC3", 2) } + } + private fun List.assertHasLoadAssignment(map: Map) { assertThat(this) .isNotEmpty() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt index 38eaf4af5..86e222692 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt @@ -29,5 +29,6 @@ fun createEndpoint(zone: String): LocalityLbEndpoints { .build() ) .addAllLbEndpoints(listOf(LbEndpoint.getDefaultInstance())) + .setPriority(DEFAULT_PRIORITY) .build() } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt index 4dd10f126..2ac3e9b9e 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt @@ -15,6 +15,8 @@ const val CLUSTER_NAME1 = "cluster-1" const val CLUSTER_NAME2 = "cluster-2" const val TRAFFIC_SPLITTING_ZONE = "dc2" const val CURRENT_ZONE = "dc1" +const val DEFAULT_PRIORITY = 1 +const val HIGHEST_PRIORITY = 0 val DEFAULT_CLUSTER_WEIGHTS = zoneWeights(mapOf(CURRENT_ZONE to 60, TRAFFIC_SPLITTING_ZONE to 40)) @@ -24,6 +26,7 @@ val SNAPSHOT_PROPERTIES_WITH_WEIGHTS = SnapshotProperties().also { DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS ) it.loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_ZONE + it.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting = listOf(CURRENT_ZONE) } fun zoneWeights(weightByZone: Map) = ZoneWeights().also { diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt similarity index 54% rename from envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt rename to envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt index c5953a0dd..0ddcc9b39 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt @@ -1,7 +1,9 @@ package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting -import TrafficSplitting.serviceName -import TrafficSplitting.upstreamServiceName +import TrafficSplitting.DEFAULT_PRIORITIES +import TrafficSplitting.FORCE_TRAFFIC_ZONE +import TrafficSplitting.SERVICE_NAME +import TrafficSplitting.UPSTREAM_SERVICE_NAME import callUpstreamServiceRepeatedly import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension @@ -11,39 +13,21 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension import verifyCallsCountCloseTo +import verifyCallsCountEq import verifyIsReachable import java.time.Duration -class WeightedClustersRoutingTest { +class LocalityWeightedLoadBalancingTest { companion object { - private const val forceTrafficZone = "dc2" - private val properties = mapOf( - "pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory" to "DEBUG", - "pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory" to "DEBUG", - "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0), + "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ZERO, "envoy-control.sync.enabled" to true, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zoneName" to forceTrafficZone, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc1" to 30, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc2" to 10, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc3" to 1, - "envoy-control.envoy.snapshot.load-balancing.priorities.zonePriorities" to mapOf( - "dc1" to mapOf( - "dc1" to 0, - "dc2" to 0, - "dc3" to 3, - ), - "dc2" to mapOf( - "dc1" to 0, - "dc2" to 0, - "dc3" to 3, - ), - "dc3" to mapOf( - "dc1" to 3, - "dc2" to 3, - "dc3" to 0, - ), - ) + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zoneName" to FORCE_TRAFFIC_ZONE, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zonesAllowingTrafficSplitting" to listOf("dc1"), + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$SERVICE_NAME.weightByZone.dc1" to 30, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$SERVICE_NAME.weightByZone.dc2" to 10, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$SERVICE_NAME.weightByZone.dc3" to 1, + "envoy-control.envoy.snapshot.load-balancing.priorities.zonePriorities" to DEFAULT_PRIORITIES ) private val echo2Config = """ @@ -53,6 +37,7 @@ class WeightedClustersRoutingTest { outgoing: dependencies: - service: "service-1" + - service: "service-2" """.trimIndent() private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") @@ -94,44 +79,68 @@ class WeightedClustersRoutingTest { @JvmField @RegisterExtension - val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + val downstreamServiceEnvoy = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + @JvmField @RegisterExtension - val echoEnvoyDC2 = EnvoyExtension(envoyControl2) + val envoyDC2 = EnvoyExtension(envoyControl2) @JvmField @RegisterExtension - val echoEnvoyDC3 = EnvoyExtension(envoyControl3) + val envoyDC3 = EnvoyExtension(envoyControl3) } @Test fun `should route traffic according to weights`() { - consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(downstreamServiceEnvoy, name = SERVICE_NAME) - consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = UPSTREAM_SERVICE_NAME) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC1, UPSTREAM_SERVICE_NAME) - consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = UPSTREAM_SERVICE_NAME) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) - echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) + downstreamServiceEnvoy.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) .verifyCallsCountCloseTo(upstreamServiceDC1, 75) .verifyCallsCountCloseTo(upstreamServiceDC2, 25) - println("snapshot: " + envoyControl.app.getGlobalSnapshot(false).toString()) + .verifyCallsCountEq(upstreamServiceDC3, 0) } @Test fun `should route traffic according to weights with service tag`() { - consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) - - consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName, tags = listOf("tag")) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(downstreamServiceEnvoy, name = SERVICE_NAME) - consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName, tags = listOf("tag")) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) - - echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") + consul.serverFirst.operations.registerService( + upstreamServiceDC1, + name = UPSTREAM_SERVICE_NAME, + tags = listOf("tag") + ) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC1, UPSTREAM_SERVICE_NAME) + consul.serverSecond.operations.registerService( + upstreamServiceDC2, + name = UPSTREAM_SERVICE_NAME, + tags = listOf("tag") + ) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + downstreamServiceEnvoy.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") .verifyCallsCountCloseTo(upstreamServiceDC1, 75) .verifyCallsCountCloseTo(upstreamServiceDC2, 25) + .verifyCallsCountEq(upstreamServiceDC3, 0) + } + + @Test + fun `should not split traffic from unlisted zone`() { + consul.serverThird.operations.registerServiceWithEnvoyOnEgress(envoyDC3, name = "echo") + + consul.serverThird.operations.registerService(upstreamServiceDC3, name = UPSTREAM_SERVICE_NAME) + envoyDC3.verifyIsReachable(upstreamServiceDC3, UPSTREAM_SERVICE_NAME) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = UPSTREAM_SERVICE_NAME) + envoyDC2.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + + envoyDC3.callUpstreamServiceRepeatedly(upstreamServiceDC3, upstreamServiceDC2) + .verifyCallsCountEq(upstreamServiceDC3, 100) + .verifyCallsCountEq(upstreamServiceDC2, 0) + .verifyCallsCountEq(upstreamServiceDC1, 0) } } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt new file mode 100644 index 000000000..5dada8a02 --- /dev/null +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt @@ -0,0 +1,92 @@ +package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting + +import TrafficSplitting.DEFAULT_PRIORITIES +import TrafficSplitting.FORCE_TRAFFIC_ZONE +import TrafficSplitting.SERVICE_NAME +import TrafficSplitting.UPSTREAM_SERVICE_NAME +import callUpstreamServiceRepeatedly +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.Xds +import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting.LocalityWeightedLoadBalancingTest.Companion.upstreamServiceDC3 +import verifyCallsCountEq +import verifyIsReachable +import java.time.Duration + +class LocalityWeightedLoadBalancingUnlistedServiceTest { + companion object { + private val properties = mapOf( + "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ZERO, + "envoy-control.sync.enabled" to true, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zoneName" to FORCE_TRAFFIC_ZONE, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zonesAllowingTrafficSplitting" to listOf("dc1"), + "envoy-control.envoy.snapshot.load-balancing.priorities.zonePriorities" to DEFAULT_PRIORITIES + ) + + private val echo2Config = """ + node: + metadata: + proxy_settings: + outgoing: + dependencies: + - service: "service-1" + - service: "service-2" + """.trimIndent() + + private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") + + @JvmField + @RegisterExtension + val consul = ConsulMultiClusterExtension() + + @JvmField + @RegisterExtension + val envoyControl = + EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val envoyControl2 = + EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val echoServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC2 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + + @JvmField + @RegisterExtension + val echoEnvoyDC2 = EnvoyExtension(envoyControl2) + } + + @Test + fun `should not split traffic for not listed service`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = SERVICE_NAME) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = UPSTREAM_SERVICE_NAME) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, UPSTREAM_SERVICE_NAME) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = UPSTREAM_SERVICE_NAME) + echoEnvoyDC2.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + + echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) + .verifyCallsCountEq(upstreamServiceDC1, 100) + .verifyCallsCountEq(upstreamServiceDC2, 0) + .verifyCallsCountEq(upstreamServiceDC3, 0) + } +} diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt index a49c7f996..cbd7da660 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt @@ -1,5 +1,5 @@ -import TrafficSplitting.deltaPercentage -import TrafficSplitting.upstreamServiceName +import TrafficSplitting.DELTA_PERCENTAGE +import TrafficSplitting.UPSTREAM_SERVICE_NAME import org.assertj.core.api.Assertions import org.assertj.core.data.Percentage import pl.allegro.tech.servicemesh.envoycontrol.assertions.isFrom @@ -10,9 +10,27 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension internal object TrafficSplitting { - const val upstreamServiceName = "service-1" - const val serviceName = "echo2" - const val deltaPercentage = 20.0 + const val UPSTREAM_SERVICE_NAME = "service-1" + const val SERVICE_NAME = "echo2" + const val DELTA_PERCENTAGE = 20.0 + const val FORCE_TRAFFIC_ZONE = "dc2" + val DEFAULT_PRIORITIES = mapOf( + "dc1" to mapOf( + "dc1" to 0, + "dc2" to 1, + "dc3" to 2, + ), + "dc2" to mapOf( + "dc1" to 1, + "dc2" to 0, + "dc3" to 2, + ), + "dc3" to mapOf( + "dc1" to 2, + "dc2" to 1, + "dc3" to 0, + ) + ) } fun EnvoyExtension.verifyIsReachable(echoServiceExtension: EchoServiceExtension, service: String) { @@ -24,12 +42,12 @@ fun EnvoyExtension.verifyIsReachable(echoServiceExtension: EchoServiceExtension, } fun CallStats.verifyCallsCountCloseTo(service: EchoServiceExtension, expectedCount: Int): CallStats { - Assertions.assertThat(this.hits(service)).isCloseTo(expectedCount, Percentage.withPercentage(deltaPercentage)) + Assertions.assertThat(this.hits(service)).isCloseTo(expectedCount, Percentage.withPercentage(DELTA_PERCENTAGE)) return this } -fun CallStats.verifyCallsCountGreaterThan(service: EchoServiceExtension, hits: Int): CallStats { - Assertions.assertThat(this.hits(service)).isGreaterThan(hits) +fun CallStats.verifyCallsCountEq(service: EchoServiceExtension, expectedCount: Int): CallStats { + Assertions.assertThat(this.hits(service)).isEqualTo(expectedCount) return this } @@ -39,7 +57,7 @@ fun EnvoyExtension.callUpstreamServiceRepeatedly( ): CallStats { val stats = CallStats(services.asList()) this.egressOperations.callServiceRepeatedly( - service = upstreamServiceName, + service = UPSTREAM_SERVICE_NAME, stats = stats, minRepeat = numberOfCalls, maxRepeat = numberOfCalls, @@ -56,12 +74,12 @@ fun EnvoyExtension.callUpstreamServiceRepeatedly( ): CallStats { val stats = CallStats(services.asList()) this.egressOperations.callServiceRepeatedly( - service = upstreamServiceName, + service = UPSTREAM_SERVICE_NAME, stats = stats, minRepeat = numberOfCalls, maxRepeat = numberOfCalls, repeatUntil = { true }, headers = tag?.let { mapOf("x-service-tag" to it) } ?: emptyMap(), - ) + ) return stats }