From f67dd0a928228728ee3723e816b272d091b8f9ae Mon Sep 17 00:00:00 2001 From: Karel Cemus Date: Tue, 7 May 2024 11:14:19 +0200 Subject: [PATCH] Fixed redis sentinel (#302) --- .../master-slave}/docker-compose.yml | 8 +- docker/sentinel/docker-compose.yml | 79 +++++++++++++++++++ .../cache/redis/connector/RedisCommands.scala | 7 +- .../redis/connector/RedisSentinelSpec.scala | 8 +- .../test/RedisMasterSlaveContainer.scala | 2 +- .../redis/test/RedisSentinelContainer.scala | 62 +++++++++------ 6 files changed, 127 insertions(+), 39 deletions(-) rename {src/test/resources => docker/master-slave}/docker-compose.yml (79%) create mode 100644 docker/sentinel/docker-compose.yml diff --git a/src/test/resources/docker-compose.yml b/docker/master-slave/docker-compose.yml similarity index 79% rename from src/test/resources/docker-compose.yml rename to docker/master-slave/docker-compose.yml index ddd3454..e297a98 100644 --- a/src/test/resources/docker-compose.yml +++ b/docker/master-slave/docker-compose.yml @@ -1,13 +1,15 @@ -version: '3.7' +version: '3.9' + services: + redis-master: - image: redis:latest + image: redis:7.2 hostname: redis-master ports: - '${REDIS_MASTER_PORT}:6379' redis-slave: - image: redis:latest + image: redis:7.2 hostname: redis-slave ports: - '${REDIS_SLAVE_PORT}:6379' diff --git a/docker/sentinel/docker-compose.yml b/docker/sentinel/docker-compose.yml new file mode 100644 index 0000000..80926a3 --- /dev/null +++ b/docker/sentinel/docker-compose.yml @@ -0,0 +1,79 @@ +version: '3.9' + +services: + + redis-master: + image: "bitnami/redis:7.2" + hostname: redis-master + ports: + - "${REDIS_MASTER_PORT}:6379" + networks: + - redis-sentinel + environment: + - REDIS_REPLICATION_MODE=master + - ALLOW_EMPTY_PASSWORD=yes + + redis-slave: + image: "bitnami/redis:7.2" + hostname: redis-slave + ports: + - "${REDIS_SLAVE_PORT}:6379" + networks: + - redis-sentinel + environment: + - REDIS_REPLICATION_MODE=slave + - REDIS_MASTER_HOST=redis-master + - ALLOW_EMPTY_PASSWORD=yes + depends_on: + - redis-master + + redis-sentinel-1: + image: "bitnami/redis-sentinel:7.2" + hostname: redis-sentinel-1 + ports: + - "${REDIS_SENTINEL_1_PORT}:26379" + networks: + - redis-sentinel + environment: + - REDIS_MASTER_SET=mymaster + - REDIS_MASTER_HOST=redis-master + - REDIS_SENTINEL_DOWN_AFTER_MILLISECONDS=10000 + - ALLOW_EMPTY_PASSWORD=yes + depends_on: + - redis-master + - redis-slave + + redis-sentinel-2: + image: "bitnami/redis-sentinel:7.2" + hostname: redis-sentinel-2 + ports: + - "${REDIS_SENTINEL_2_PORT}:26379" + networks: + - redis-sentinel + environment: + - REDIS_MASTER_SET=mymaster + - REDIS_MASTER_HOST=redis-master + - REDIS_SENTINEL_DOWN_AFTER_MILLISECONDS=10000 + - ALLOW_EMPTY_PASSWORD=yes + depends_on: + - redis-master + - redis-slave + + redis-sentinel-3: + image: "bitnami/redis-sentinel:7.2" + hostname: redis-sentinel-3 + ports: + - "${REDIS_SENTINEL_3_PORT}:26379" + networks: + - redis-sentinel + environment: + - REDIS_MASTER_SET=mymaster + - REDIS_MASTER_HOST=redis-master + - REDIS_SENTINEL_DOWN_AFTER_MILLISECONDS=10000 + - ALLOW_EMPTY_PASSWORD=yes + depends_on: + - redis-master + - redis-slave + +networks: + redis-sentinel: diff --git a/src/main/scala/play/api/cache/redis/connector/RedisCommands.scala b/src/main/scala/play/api/cache/redis/connector/RedisCommands.scala index f615dcf..f91c815 100644 --- a/src/main/scala/play/api/cache/redis/connector/RedisCommands.scala +++ b/src/main/scala/play/api/cache/redis/connector/RedisCommands.scala @@ -213,10 +213,10 @@ private[connector] class RedisCommandsSentinel( private val redisUri: RedisURI = RedisURI.Builder - .sentinel(sentinel.host, sentinel.port) + .sentinel(sentinel.host, sentinel.port, configuration.masterGroup) .withDatabase(configuration.database) .withCredentials(configuration.username, configuration.password) - .withSentinels(configuration.sentinels) + .withSentinels(configuration.sentinels.tail) .build() override protected def connectionString: String = redisUri.toString @@ -232,7 +232,8 @@ private[connector] class RedisCommandsSentinel( val newConnection: RedisConnection = RedisConnection.fromStandalone( - client.connect().withTimeout(configuration.timeout.connection), + MasterReplica.connect(client, StringCodec.UTF8, redisUri) + .withReadFrom(ReadFrom.MASTER_PREFERRED), ) } diff --git a/src/test/scala/play/api/cache/redis/connector/RedisSentinelSpec.scala b/src/test/scala/play/api/cache/redis/connector/RedisSentinelSpec.scala index 6f22198..b2cd8d9 100644 --- a/src/test/scala/play/api/cache/redis/connector/RedisSentinelSpec.scala +++ b/src/test/scala/play/api/cache/redis/connector/RedisSentinelSpec.scala @@ -1,7 +1,6 @@ package play.api.cache.redis.connector import org.apache.pekko.actor.ActorSystem -import org.scalatest.Ignore import play.api.cache.redis._ import play.api.cache.redis.configuration._ import play.api.cache.redis.impl._ @@ -11,7 +10,6 @@ import play.api.inject.{ApplicationLifecycle, Injector} import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} -@Ignore class RedisSentinelSpec extends IntegrationSpec with RedisSentinelContainer with DefaultInjector { test("pong on ping") { connector => @@ -74,10 +72,8 @@ class RedisSentinelSpec extends IntegrationSpec with RedisSentinelContainer with name = "sentinel", masterGroup = master, sentinels = 0 - .until(nodes) - .map { i => - RedisHost(container.containerIpAddress, container.mappedPort(sentinelPort + i)) - } + .until(sentinels) + .map(i => RedisHost(host, sentinelPort + i)) .toList, settings = RedisSettings.load( config = Helpers.configuration.default.underlying, diff --git a/src/test/scala/play/api/cache/redis/test/RedisMasterSlaveContainer.scala b/src/test/scala/play/api/cache/redis/test/RedisMasterSlaveContainer.scala index cd7108b..bc20b69 100644 --- a/src/test/scala/play/api/cache/redis/test/RedisMasterSlaveContainer.scala +++ b/src/test/scala/play/api/cache/redis/test/RedisMasterSlaveContainer.scala @@ -17,7 +17,7 @@ trait RedisMasterSlaveContainer extends ForAllTestContainer { protected def newContainer: TestContainerDef[TestContainer] = DockerComposeContainer.Def( - new File("src/test/resources/docker-compose.yml"), + new File("docker/master-slave/docker-compose.yml"), tailChildContainers = true, env = Map( "REDIS_MASTER_PORT" -> s"$masterPort", diff --git a/src/test/scala/play/api/cache/redis/test/RedisSentinelContainer.scala b/src/test/scala/play/api/cache/redis/test/RedisSentinelContainer.scala index fd65e74..75e4dcc 100644 --- a/src/test/scala/play/api/cache/redis/test/RedisSentinelContainer.scala +++ b/src/test/scala/play/api/cache/redis/test/RedisSentinelContainer.scala @@ -1,43 +1,53 @@ package play.api.cache.redis.test +import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService} import org.scalatest.Suite -import play.api.Logger +import org.testcontainers.containers.wait.strategy.Wait -import scala.concurrent.duration.DurationInt +import java.io.File -trait RedisSentinelContainer extends RedisContainer { +trait RedisSentinelContainer extends ForAllTestContainer { this: Suite => - private val log = Logger("play.api.cache.redis.test") + override protected type TestContainer = DockerComposeContainer - protected def nodes: Int = 3 + protected def master: String = "mymaster" - final protected def initialPort: Int = 7000 + final protected val host = "localhost" - final protected def sentinelPort: Int = initialPort - 2000 + final private val initialPort = 7000 + + final protected val masterPort = initialPort + final protected val slavePort = masterPort + 1 - protected def master: String = s"sentinel$initialPort" + protected def sentinels: Int = 3 - private val waitForStart = 7.seconds + final protected def sentinelPort: Int = initialPort - 2000 - override protected lazy val redisConfig: RedisContainerConfig = - RedisContainerConfig( - redisDockerImage = "grokzen/redis-cluster:7.0.10", - redisMappedPorts = Seq.empty, - redisFixedPorts = 0.until(nodes).flatMap(i => Seq(initialPort + i, sentinelPort + i)), - redisEnvironment = Map( - "IP" -> "0.0.0.0", - "INITIAL_PORT" -> initialPort.toString, - "SENTINEL" -> "true", + // note: waiting for sentinels is not working + // private def sentinelWaitStrategy = new WaitAllStrategy() + // .withStrategy(Wait.forLogMessage(".*\\+monitor master.*\\n", 1)) + // .withStrategy(Wait.forLogMessage(".*\\+slave slave.*\\n", 1)) + + protected def newContainer: TestContainerDef[TestContainer] = + DockerComposeContainer.Def( + new File("docker/sentinel/docker-compose.yml"), + tailChildContainers = true, + env = Map( + "REDIS_MASTER_PORT" -> s"$masterPort", + "REDIS_SLAVE_PORT" -> s"$slavePort", + "REDIS_SENTINEL_1_PORT" -> s"${sentinelPort + 0}", + "REDIS_SENTINEL_2_PORT" -> s"${sentinelPort + 1}", + "REDIS_SENTINEL_3_PORT" -> s"${sentinelPort + 2}", + ), + exposedServices = Seq( + ExposedService("redis-master", masterPort, Wait.forLogMessage(".*Ready to accept connections tcp.*\\n", 1)), + ExposedService("redis-slave", slavePort, Wait.forLogMessage(".*MASTER <-> REPLICA sync: Finished with success.*\\n", 1)), + // note: waiting for sentinels doesn't work, it says "service is not running" + // ExposedService("redis-sentinel-1", sentinelPort + 0, sentinelWaitStrategy), + // ExposedService("redis-sentinel-2", sentinelPort + 1, sentinelWaitStrategy), + // ExposedService("redis-sentinel-3", sentinelPort + 2, sentinelWaitStrategy), ), ) - @SuppressWarnings(Array("org.wartremover.warts.ThreadSleep")) - override def beforeAll(): Unit = { - super.beforeAll() - log.info(s"Waiting for Redis Sentinel to start on ${container.containerIpAddress}, will wait for $waitForStart") - Thread.sleep(waitForStart.toMillis) - log.info(s"Finished waiting for Redis Sentinel to start on ${container.containerIpAddress}") - } - }