NOISSUE - Refactor Redis URI (#1898)

* Replace Nats with Nats Jestream For PubSub Signed-off-by: rodneyosodo <[email protected]> Signed-off-by: Rodney Osodo <[email protected]> * Refactor Redis client connection Use a single Redis URI to connect to Redis where the URI contains host, port, database and possibly username and password. This is to simplify environment variable configuration. Signed-off-by: Rodney Osodo <[email protected]> Signed-off-by: Rodney Osodo <[email protected]> --------- Signed-off-by: rodneyosodo <[email protected]> Signed-off-by: Rodney Osodo <[email protected]> Signed-off-by: Rodney Osodo <[email protected]>
dborovcanin · Oct 23, 2023 · 059b954 · 059b954
1 parent 9b4c2b2
commit 059b954
Show file tree

Hide file tree

Showing 14 changed files with 99 additions and 171 deletions.
diff --git a/cmd/lora/main.go b/cmd/lora/main.go
@@ -35,10 +35,9 @@ import (
 )
 
 const (
-	svcName           = "lora-adapter"
-	envPrefixHTTP     = "MF_LORA_ADAPTER_HTTP_"
-	envPrefixRouteMap = "MF_LORA_ADAPTER_ROUTE_MAP_"
-	defSvcHTTPPort    = "9017"
+	svcName        = "lora-adapter"
+	envPrefixHTTP  = "MF_LORA_ADAPTER_HTTP_"
+	defSvcHTTPPort = "9017"
 
 	thingsRMPrefix   = "thing"
 	channelsRMPrefix = "channel"
@@ -59,6 +58,7 @@ type config struct {
 	SendTelemetry  bool          `env:"MF_SEND_TELEMETRY"                   envDefault:"true"`
 	InstanceID     string        `env:"MF_LORA_ADAPTER_INSTANCE_ID"         envDefault:""`
 	ESURL          string        `env:"MF_LORA_ADAPTER_ES_URL"              envDefault:"redis://localhost:6379/0"`
+	RouteMapURL    string        `env:"MF_LORA_ADAPTER_ROUTE_MAP_URL"       envDefault:"redis://localhost:6379/0"`
 }
 
 func main() {
@@ -93,7 +93,7 @@ func main() {
 		return
 	}
 
-	rmConn, err := redisclient.Setup(envPrefixRouteMap)
+	rmConn, err := redisclient.Connect(cfg.RouteMapURL)
 	if err != nil {
 		logger.Error(fmt.Sprintf("failed to setup route map redis client : %s", err))
 		exitCode = 1

diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go
@@ -33,10 +33,9 @@ import (
 )
 
 const (
-	svcName           = "opc-ua-adapter"
-	envPrefixHTTP     = "MF_OPCUA_ADAPTER_HTTP_"
-	envPrefixRouteMap = "MF_OPCUA_ADAPTER_ROUTE_MAP_"
-	defSvcHTTPPort    = "8180"
+	svcName        = "opc-ua-adapter"
+	envPrefixHTTP  = "MF_OPCUA_ADAPTER_HTTP_"
+	defSvcHTTPPort = "8180"
 
 	thingsRMPrefix     = "thing"
 	channelsRMPrefix   = "channel"
@@ -53,6 +52,7 @@ type config struct {
 	SendTelemetry  bool   `env:"MF_SEND_TELEMETRY"                   envDefault:"true"`
 	InstanceID     string `env:"MF_OPCUA_ADAPTER_INSTANCE_ID"        envDefault:""`
 	ESURL          string `env:"MF_OPCUA_ADAPTER_ES_URL"             envDefault:"redis://localhost:6379/0"`
+	RouteMapURL    string `env:"MF_OPCUA_ADAPTER_ROUTE_MAP_URL"      envDefault:"redis://localhost:6379/0"`
 }
 
 func main() {
@@ -92,7 +92,7 @@ func main() {
 		return
 	}
 
-	rmConn, err := redisclient.Setup(envPrefixRouteMap)
+	rmConn, err := redisclient.Connect(cfg.RouteMapURL)
 	if err != nil {
 		logger.Error(fmt.Sprintf("failed to setup %s bootstrap event store redis client : %s", svcName, err))
 		exitCode = 1

diff --git a/cmd/things/main.go b/cmd/things/main.go
@@ -51,8 +51,6 @@ import (
 const (
 	svcName            = "things"
 	envPrefixDB        = "MF_THINGS_DB_"
-	envPrefixCache     = "MF_THINGS_CACHE_"
-	envPrefixES        = "MF_THINGS_ES_"
 	envPrefixHTTP      = "MF_THINGS_HTTP_"
 	envPrefixGRPC      = "MF_THINGS_AUTH_GRPC_"
 	defDB              = "things"
@@ -67,8 +65,9 @@ type config struct {
 	JaegerURL        string `env:"MF_JAEGER_URL"                 envDefault:"http://jaeger:14268/api/traces"`
 	CacheKeyDuration string `env:"MF_THINGS_CACHE_KEY_DURATION"  envDefault:"10m"`
 	SendTelemetry    bool   `env:"MF_SEND_TELEMETRY"             envDefault:"true"`
-	InstanceID       string `env:"MF_THINGS_INSTANCE_ID"        envDefault:""`
+	InstanceID       string `env:"MF_THINGS_INSTANCE_ID"         envDefault:""`
 	ESURL            string `env:"MF_THINGS_ES_URL"              envDefault:"redis://localhost:6379/0"`
+	CacheURL         string `env:"MF_THINGS_CACHE_URL"           envDefault:"redis://localhost:6379/0"`
 }
 
 func main() {
@@ -128,23 +127,14 @@ func main() {
 	tracer := tp.Tracer(svcName)
 
 	// Setup new redis cache client
-	cacheclient, err := redisclient.Setup(envPrefixCache)
+	cacheclient, err := redisclient.Connect(cfg.CacheURL)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1
 		return
 	}
 	defer cacheclient.Close()
 
-	// Setup new redis event store client
-	esclient, err := redisclient.Setup(envPrefixES)
-	if err != nil {
-		logger.Error(err.Error())
-		exitCode = 1
-		return
-	}
-	defer esclient.Close()
-
 	var auth mainflux.AuthServiceClient
 
 	switch cfg.StandaloneID != "" && cfg.StandaloneToken != "" {
@@ -163,7 +153,7 @@ func main() {
 		logger.Info("Successfully connected to auth grpc server " + authHandler.Secure())
 	}
 
-	csvc, gsvc, err := newService(ctx, db, dbConfig, auth, cacheclient, esclient, cfg.CacheKeyDuration, cfg.ESURL, tracer, logger)
+	csvc, gsvc, err := newService(ctx, db, dbConfig, auth, cacheclient, cfg.CacheKeyDuration, cfg.ESURL, tracer, logger)
 	if err != nil {
 		logger.Error(fmt.Sprintf("failed to create services: %s", err))
 		exitCode = 1
@@ -214,7 +204,7 @@ func main() {
 	}
 }
 
-func newService(ctx context.Context, db *sqlx.DB, dbConfig pgclient.Config, auth mainflux.AuthServiceClient, cacheClient *redis.Client, esClient *redis.Client, keyDuration, esURL string, tracer trace.Tracer, logger mflog.Logger) (things.Service, groups.Service, error) {
+func newService(ctx context.Context, db *sqlx.DB, dbConfig pgclient.Config, auth mainflux.AuthServiceClient, cacheClient *redis.Client, keyDuration, esURL string, tracer trace.Tracer, logger mflog.Logger) (things.Service, groups.Service, error) {
 	database := postgres.NewDatabase(db, dbConfig, tracer)
 	cRepo := thingspg.NewRepository(database)
 	gRepo := gpostgres.New(database)

diff --git a/cmd/twins/main.go b/cmd/twins/main.go
@@ -42,7 +42,6 @@ const (
 	svcName        = "twins"
 	envPrefixDB    = "MF_TWINS_DB_"
 	envPrefixHTTP  = "MF_TWINS_HTTP_"
-	envPrefixCache = "MF_TWINS_CACHE_"
 	defSvcHTTPPort = "9018"
 )
 
@@ -56,6 +55,7 @@ type config struct {
 	SendTelemetry   bool   `env:"MF_SEND_TELEMETRY"           envDefault:"true"`
 	InstanceID      string `env:"MF_TWINS_INSTANCE_ID"        envDefault:""`
 	ESURL           string `env:"MF_TWINS_ES_URL"             envDefault:"redis://localhost:6379/0"`
+	CacheURL        string `env:"MF_TWINS_CACHE_URL"          envDefault:"redis://localhost:6379/0"`
 }
 
 func main() {
@@ -90,7 +90,7 @@ func main() {
 		return
 	}
 
-	cacheClient, err := redisclient.Setup(envPrefixCache)
+	cacheClient, err := redisclient.Connect(cfg.CacheURL)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/docker/.env b/docker/.env
@@ -157,9 +157,7 @@ MF_THINGS_AUTH_GRPC_SERVER_CA_CERTS=${GRPC_MTLS:+./ssl/certs/ca.crt}${GRPC_TLS:+
 MF_THINGS_ES_URL=es-redis:${MF_REDIS_TCP_PORT}
 MF_THINGS_ES_PASS=
 MF_THINGS_ES_DB=0
-MF_THINGS_CACHE_URL=things-redis:${MF_REDIS_TCP_PORT}
-MF_THINGS_CACHE_PASS=
-MF_THINGS_CACHE_DB=0
+MF_THINGS_CACHE_URL=redis://things-redis:${MF_REDIS_TCP_PORT}/0
 MF_THINGS_DB_HOST=things-db
 MF_THINGS_DB_PORT=5432
 MF_THINGS_DB_USER=mainflux
@@ -316,9 +314,7 @@ MF_LORA_ADAPTER_HTTP_HOST=lora-adapter
 MF_LORA_ADAPTER_HTTP_PORT=9017
 MF_LORA_ADAPTER_HTTP_SERVER_CERT=
 MF_LORA_ADAPTER_HTTP_SERVER_KEY=
-MF_LORA_ADAPTER_ROUTE_MAP_URL=lora-redis:${MF_REDIS_TCP_PORT}
-MF_LORA_ADAPTER_ROUTE_MAP_PASS=
-MF_LORA_ADAPTER_ROUTE_MAP_DB=0
+MF_LORA_ADAPTER_ROUTE_MAP_URL=redis://lora-redis:${MF_REDIS_TCP_PORT}/0
 MF_LORA_ADAPTER_INSTANCE_ID=
 
 ### OPC-UA
@@ -331,9 +327,7 @@ MF_OPCUA_ADAPTER_HTTP_SERVER_KEY=
 MF_OPCUA_ADAPTER_ES_URL=es-redis:${MF_REDIS_TCP_PORT}
 MF_OPCUA_ADAPTER_ES_PASS=
 MF_OPCUA_ADAPTER_ES_DB=0
-MF_OPCUA_ADAPTER_ROUTE_MAP_URL=opcua-redis:${MF_REDIS_TCP_PORT}
-MF_OPCUA_ADAPTER_ROUTE_MAP_PASS=
-MF_OPCUA_ADAPTER_ROUTE_MAP_DB=0
+MF_OPCUA_ADAPTER_ROUTE_MAP_URL=redis://opcua-redis:${MF_REDIS_TCP_PORT}/0
 MF_OPCUA_ADAPTER_INSTANCE_ID=
 
 ### Cassandra
@@ -482,9 +476,7 @@ MF_TWINS_HTTP_HOST=twins
 MF_TWINS_HTTP_PORT=9018
 MF_TWINS_HTTP_SERVER_CERT=
 MF_TWINS_HTTP_SERVER_KEY=
-MF_TWINS_CACHE_URL=twins-redis:${MF_REDIS_TCP_PORT}
-MF_TWINS_CACHE_PASS=
-MF_TWINS_CACHE_DB=0
+MF_TWINS_CACHE_URL=redis://twins-redis:${MF_REDIS_TCP_PORT}/0
 MF_THINGS_ES_URL=es-redis:${MF_REDIS_TCP_PORT}
 MF_THINGS_ES_PASS=
 MF_THINGS_ES_DB=0

diff --git a/docker/addons/lora-adapter/docker-compose.yml b/docker/addons/lora-adapter/docker-compose.yml
@@ -35,9 +35,7 @@ services:
       MF_LORA_ADAPTER_HTTP_PORT: ${MF_LORA_ADAPTER_HTTP_PORT}
       MF_LORA_ADAPTER_HTTP_SERVER_CERT: ${MF_LORA_ADAPTER_HTTP_SERVER_CERT}
       MF_LORA_ADAPTER_HTTP_SERVER_KEY: ${MF_LORA_ADAPTER_HTTP_SERVER_KEY}
-      MF_LORA_ADAPTER_ROUTE_MAP_URL: lora-redis:${MF_REDIS_TCP_PORT}
-      MF_LORA_ADAPTER_ROUTE_MAP_PASS: ${MF_LORA_ADAPTER_ROUTE_MAP_PASS}
-      MF_LORA_ADAPTER_ROUTE_MAP_DB: ${MF_LORA_ADAPTER_ROUTE_MAP_DB}
+      MF_LORA_ADAPTER_ROUTE_MAP_URL: ${MF_LORA_ADAPTER_ROUTE_MAP_URL}
       MF_LORA_ADAPTER_ES_URL: ${MF_ES_URL}
       MF_MESSAGE_BROKER_URL: ${MF_MESSAGE_BROKER_URL}
       MF_JAEGER_URL: ${MF_JAEGER_URL}

diff --git a/docker/addons/opcua-adapter/docker-compose.yml b/docker/addons/opcua-adapter/docker-compose.yml
@@ -38,8 +38,6 @@ services:
       MF_OPCUA_ADAPTER_HTTP_SERVER_KEY: ${MF_OPCUA_ADAPTER_HTTP_SERVER_KEY}
       MF_OPCUA_ADAPTER_ES_URL: ${MF_ES_URL}
       MF_OPCUA_ADAPTER_ROUTE_MAP_URL: ${MF_OPCUA_ADAPTER_ROUTE_MAP_URL}
-      MF_OPCUA_ADAPTER_ROUTE_MAP_PASS: ${MF_OPCUA_ADAPTER_ROUTE_MAP_PASS}
-      MF_OPCUA_ADAPTER_ROUTE_MAP_DB: ${MF_OPCUA_ADAPTER_ROUTE_MAP_DB}
       MF_MESSAGE_BROKER_URL: ${MF_MESSAGE_BROKER_URL}
       MF_JAEGER_URL: ${MF_JAEGER_URL}
       MF_SEND_TELEMETRY: ${MF_SEND_TELEMETRY}

diff --git a/docker/addons/twins/docker-compose.yml b/docker/addons/twins/docker-compose.yml
@@ -52,8 +52,6 @@ services:
       MF_TWINS_HTTP_SERVER_CERT: ${MF_TWINS_HTTP_SERVER_CERT}
       MF_TWINS_HTTP_SERVER_KEY: ${MF_TWINS_HTTP_SERVER_KEY}
       MF_TWINS_CACHE_URL: ${MF_TWINS_CACHE_URL}
-      MF_TWINS_CACHE_PASS: ${MF_TWINS_CACHE_PASS}
-      MF_TWINS_CACHE_DB: ${MF_TWINS_CACHE_DB}
       MF_TWINS_ES_URL: ${MF_ES_URL}
       MF_THINGS_STANDALONE_ID: ${MF_THINGS_STANDALONE_ID}
       MF_THINGS_STANDALONE_TOKEN: ${MF_THINGS_STANDALONE_TOKEN}

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -192,8 +192,6 @@ services:
       MF_THINGS_AUTH_GRPC_CLIENT_CA_CERTS: ${MF_THINGS_AUTH_GRPC_CLIENT_CA_CERTS:+/things-grpc-client-ca.crt}
       MF_THINGS_ES_URL: ${MF_ES_URL}
       MF_THINGS_CACHE_URL: ${MF_THINGS_CACHE_URL}
-      MF_THINGS_CACHE_PASS: ${MF_THINGS_CACHE_PASS}
-      MF_THINGS_CACHE_DB: ${MF_THINGS_CACHE_DB}
       MF_THINGS_DB_HOST: ${MF_THINGS_DB_HOST}
       MF_THINGS_DB_PORT: ${MF_THINGS_DB_PORT}
       MF_THINGS_DB_USER: ${MF_THINGS_DB_USER}

diff --git a/internal/clients/redis/redis.go b/internal/clients/redis/redis.go
@@ -3,49 +3,14 @@
 
 package redis
 
-import (
-	"strconv"
-
-	"github.com/go-redis/redis/v8"
-	"github.com/mainflux/mainflux/internal/env"
-	"github.com/mainflux/mainflux/pkg/errors"
-)
-
-var (
-	errConfig  = errors.New("failed to load redis client configuration")
-	errConnect = errors.New("failed to connect to redis server")
-)
-
-// Config of RedisDB.
-type Config struct {
-	URL  string `env:"URL"    envDefault:"localhost:6379"`
-	Pass string `env:"PASS"   envDefault:""`
-	DB   string `env:"DB"     envDefault:"0"`
-}
-
-// Setup load configuration from environment, creates new RedisDB client and connect to RedisDB Server.
-func Setup(prefix string) (*redis.Client, error) {
-	cfg := Config{}
-	if err := env.Parse(&cfg, env.Options{Prefix: prefix}); err != nil {
-		return nil, errors.Wrap(errConfig, err)
-	}
-	client, err := Connect(cfg)
-	if err != nil {
-		return nil, errors.Wrap(errConnect, err)
-	}
-	return client, nil
-}
+import "github.com/go-redis/redis/v8"
 
 // Connect create new RedisDB client and connect to RedisDB server.
-func Connect(cfg Config) (*redis.Client, error) {
-	db, err := strconv.Atoi(cfg.DB)
+func Connect(url string) (*redis.Client, error) {
+	opts, err := redis.ParseURL(url)
 	if err != nil {
 		return nil, err
 	}
 
-	return redis.NewClient(&redis.Options{
-		Addr:     cfg.URL,
-		Password: cfg.Pass,
-		DB:       db,
-	}), nil
+	return redis.NewClient(opts), nil
 }
diff --git a/lora/README.md b/lora/README.md
@@ -1,4 +1,5 @@
 # LoRa Adapter
+
 Adapter between Mainflux IoT system and [LoRa Server](https://github.com/brocaar/chirpstack-network-server).
 
 This adapter sits between Mainflux and LoRa Server and just forwards the messages from one system to another via MQTT protocol, using the adequate MQTT topics and in the good message format (JSON and SenML), i.e. respecting the APIs of both systems.
@@ -11,25 +12,23 @@ The service is configured using the environment variables presented in the
 following table. Note that any unset variables will be replaced with their
 default values.
 
-| Variable                         | Description                                  | Default                         |
-|----------------------------------|----------------------------------------------|---------------------------------|
-| MF_LORA_ADAPTER_HTTP_PORT        | Service HTTP port                            | 9017                            |
-| MF_LORA_ADAPTER_LOG_LEVEL        | Service Log level                            | info                            |
-| MF_MESSAGE_BROKER_URL            | Message broker instance URL                  | nats://localhost:4222           |
-| MF_LORA_ADAPTER_MESSAGES_URL     | LoRa adapter MQTT broker URL                 | tcp://localhost:1883            |
-| MF_LORA_ADAPTER_MESSAGES_TOPIC   | LoRa adapter MQTT subscriber Topic           | application/+/device/+/event/up |
-| MF_LORA_ADAPTER_MESSAGES_USER    | LoRa adapter MQTT subscriber Username        |                                 |
-| MF_LORA_ADAPTER_MESSAGES_PASS    | LoRa adapter MQTT subscriber Password        |                                 |
-| MF_LORA_ADAPTER_MESSAGES_TIMEOUT | LoRa adapter MQTT subscriber Timeout         | 30s                             |
-| MF_LORA_ADAPTER_ROUTE_MAP_URL    | Route-map database URL                       | localhost:6379                  |
-| MF_LORA_ADAPTER_ROUTE_MAP_PASS   | Route-map database password                  |                                 |
-| MF_LORA_ADAPTER_ROUTE_MAP_DB     | Route-map instance                           | 0                               |
-| MF_THINGS_ES_URL                 | Things service event source URL              | localhost:6379                  |
-| MF_THINGS_ES_PASS                | Things service event source password         |                                 |
-| MF_THINGS_ES_DB                  | Things service event source DB               | 0                               |
-| MF_LORA_ADAPTER_EVENT_CONSUMER   | Service event consumer name                  | lora                            |
-| MF_JAEGER_URL                    | Jaeger server URL                            | http://jaeger:14268/api/traces                  |
-| MF_SEND_TELEMETRY                | Send telemetry to mainflux call home server  | true                            |
+| Variable                         | Description                                 | Default                         |
+| -------------------------------- | ------------------------------------------- | ------------------------------- |
+| MF_LORA_ADAPTER_HTTP_PORT        | Service HTTP port                           | 9017                            |
+| MF_LORA_ADAPTER_LOG_LEVEL        | Service Log level                           | info                            |
+| MF_MESSAGE_BROKER_URL            | Message broker instance URL                 | nats://localhost:4222           |
+| MF_LORA_ADAPTER_MESSAGES_URL     | LoRa adapter MQTT broker URL                | tcp://localhost:1883            |
+| MF_LORA_ADAPTER_MESSAGES_TOPIC   | LoRa adapter MQTT subscriber Topic          | application/+/device/+/event/up |
+| MF_LORA_ADAPTER_MESSAGES_USER    | LoRa adapter MQTT subscriber Username       |                                 |
+| MF_LORA_ADAPTER_MESSAGES_PASS    | LoRa adapter MQTT subscriber Password       |                                 |
+| MF_LORA_ADAPTER_MESSAGES_TIMEOUT | LoRa adapter MQTT subscriber Timeout        | 30s                             |
+| MF_LORA_ADAPTER_ROUTE_MAP_URL    | Route-map database URL                      | redis://localhost:6379          |
+| MF_THINGS_ES_URL                 | Things service event source URL             | localhost:6379                  |
+| MF_THINGS_ES_PASS                | Things service event source password        |                                 |
+| MF_THINGS_ES_DB                  | Things service event source DB              | 0                               |
+| MF_LORA_ADAPTER_EVENT_CONSUMER   | Service event consumer name                 | lora                            |
+| MF_JAEGER_URL                    | Jaeger server URL                           | http://jaeger:14268/api/traces  |
+| MF_SEND_TELEMETRY                | Send telemetry to mainflux call home server | true                            |
 
 ## Deployment
 
@@ -59,8 +58,6 @@ MF_LORA_ADAPTER_MESSAGES_USER=[LoRa adapter MQTT subscriber Username] \
 MF_LORA_ADAPTER_MESSAGES_PASS=[LoRa adapter MQTT subscriber Password] \
 MF_LORA_ADAPTER_MESSAGES_TIMEOUT=[LoRa adapter MQTT subscriber Timeout]
 MF_LORA_ADAPTER_ROUTE_MAP_URL=[Lora adapter routemap URL] \
-MF_LORA_ADAPTER_ROUTE_MAP_PASS=[Lora adapter routemap password] \
-MF_LORA_ADAPTER_ROUTE_MAP_DB=[Lora adapter routemap instance] \
 MF_THINGS_ES_URL=[Things service event source URL] \
 MF_THINGS_ES_PASS=[Things service event source password] \
 MF_THINGS_ES_DB=[Things service event source password] \