Merge pull request #1668 from scpwiki/WJ-1145-redis

[WJ-1145] Add initial redis support

.github/workflows/docker-build-minio.local.yaml

@@ -0,0 +1,20 @@
+name: '[backend] Docker build Minio (local)'
+
+on:
+  pull_request:
+    paths:
+      - 'install/local/dev/minio/*'
+      - '.github/workflows/docker-build-minio.local.yaml'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build image
+        run: cd install/local/dev/minio && docker build .
+        env:
+          DOCKER_BUILDKIT: 1
+

.github/workflows/docker-build-redis.local.yaml

@@ -0,0 +1,20 @@
+name: '[backend] Docker build Redis (local)'
+
+on:
+  pull_request:
+    paths:
+      - 'install/local/dev/redis/*'
+      - '.github/workflows/docker-build-redis.local.yaml'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build image
+        run: cd install/local/dev/redis && docker build .
+        env:
+          DOCKER_BUILDKIT: 1
+

deepwell/.env.example

@@ -4,9 +4,13 @@
 # If you're using docker-compose, these are already set in the container as appropriate.
 
 # Postgres URL
-# Includes password (if needed) to connect to the database.
+# Includes password (if needed) to connect.
 DATABASE_URL=postgres://localhost
 
+# Redis URL
+# Includes password (if needed) to connect.
+REDIS_URL=redis://localhost
+
 # S3 configuration settings
 S3_BUCKET=deepwell-files
 

deepwell/Cargo.toml

@@ -43,9 +43,11 @@ once_cell = "1"
 otp = { git = "https://github.com/TimDumol/rust-otp" }
 paste = "1"
 rand = "0.8"
+redis = { version = "0.23", features = ["aio", "connection-manager", "keep-alive", "tokio-comp"] }
 ref-map = "0.1"
 regex = "1"
 reqwest = { version = "0.11", features = ["json", "rustls-tls"], default-features = false }
+rsmq_async = "6"
 rust-s3 = { version = "0.32", features = ["with-async-std"], default-features = false }
 sea-orm = { version = "0.12", features = ["sqlx-postgres", "runtime-async-std-rustls", "postgres-array", "macros", "with-json", "with-time"], default-features = false }
 sea-query = "0.30"

deepwell/src/api.rs

@@ -27,7 +27,6 @@
 //! not any of the implementations themselves. Those should be in the `methods` module.
 
 use crate::config::{Config, Secrets};
-use crate::database;
 use crate::endpoints::{
     auth::*, category::*, domain::*, email::*, file::*, file_revision::*, link::*,
     locale::*, message::*, misc::*, page::*, page_revision::*, parent::*, site::*,
@@ -37,33 +36,57 @@ use crate::locales::Localizations;
 use crate::services::blob::MimeAnalyzer;
 use crate::services::job::JobQueue;
 use crate::services::{into_rpc_error, ServiceContext};
+use crate::utils::debug_pointer;
+use crate::{database, redis as redis_db};
 use jsonrpsee::server::{RpcModule, Server, ServerHandle};
 use jsonrpsee::types::error::ErrorObjectOwned;
+use redis::aio::ConnectionManager;
+use rsmq_async::MultiplexedRsmq;
 use s3::bucket::Bucket;
 use sea_orm::{DatabaseConnection, TransactionTrait};
+use std::fmt::{self, Debug};
 use std::sync::Arc;
 use std::time::Duration;
 
 pub type ServerState = Arc<ServerStateInner>;
 
-#[derive(Debug)]
 pub struct ServerStateInner {
     pub config: Config,
     pub database: DatabaseConnection,
+    pub redis: ConnectionManager,
+    pub rsmq: MultiplexedRsmq,
     pub localizations: Localizations,
     pub mime_analyzer: MimeAnalyzer,
     pub job_queue: JobQueue,
     pub s3_bucket: Bucket,
 }
 
+impl Debug for ServerStateInner {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("ServerStateInner")
+            .field("config", &self.config)
+            .field("database", &self.database)
+            .field("redis", &debug_pointer(&self.redis))
+            .field("rsmq", &self.rsmq)
+            .field("localizations", &self.localizations)
+            .field("mime_analyzer", &self.mime_analyzer)
+            .field("job_queue", &self.job_queue)
+            .field("s3_bucket", &self.s3_bucket)
+            .finish()
+    }
+}
+
 pub async fn build_server_state(
     config: Config,
     secrets: Secrets,
 ) -> anyhow::Result<ServerState> {
-    // Connect to database
+    // Connect to databases
     info!("Connecting to PostgreSQL database");
     let database = database::connect(&secrets.database_url).await?;
 
+    info!("Connecting to Redis");
+    let (redis, rsmq) = redis_db::connect(&secrets.redis_url).await?;
+
     // Load localization data
     info!("Loading localization data");
     let localizations = Localizations::open(&config.localization_path).await?;
@@ -96,6 +119,8 @@ pub async fn build_server_state(
     let state = Arc::new(ServerStateInner {
         config,
         database,
+        redis,
+        rsmq,
         localizations,
         mime_analyzer,
         job_queue,

deepwell/src/cache.rs

@@ -0,0 +1,39 @@
+/*
+ * cache.rs
+ *
+ * DEEPWELL - Wikijump API provider and database manager
+ * Copyright (C) 2019-2023 Wikijump Team
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::services::Result;
+use rsmq_async::Rsmq;
+
+/// Creates primary `redis::Client` instance.
+///
+/// Used to spawn off other connections to Redis as needed.
+pub async fn connect_redis(redis_uri: &str) -> Result<redis::Client> {
+    let client = redis::Client::open(redis_uri)?;
+    Ok(client)
+}
+
+/// Creates an RSMQ instance for temporary use.
+///
+/// A connection is fetched from the `Client`, which is then used to create
+/// an owned `Rsmq` available for use by the caller.
+pub async fn connect_rsmq(client: &redis::Client) -> Result<Rsmq> {
+    let connection = client.get_tokio_connection().await?;
+    Ok(Rsmq::new_with_connection(connection, true, None))
+}

deepwell/src/config/secrets.rs

@@ -30,6 +30,11 @@ pub struct Secrets {
     /// Set using environment variable `DATABASE_URL`.
     pub database_url: String,
 
+    /// The URL of the Redis database to connect to.
+    ///
+    /// Set using environment variable `REDIS_URL`.
+    pub redis_url: String,
+
     /// The name of the S3 bucket that file blobs are kept in.
     /// The bucket must already exist prior to program invocation.
     ///
@@ -77,6 +82,7 @@ impl Secrets {
         }
 
         let database_url = get_env!("DATABASE_URL");
+        let redis_url = get_env!("REDIS_URL");
 
         let s3_bucket = get_env!("S3_BUCKET");
         let s3_region = match env::var("S3_AWS_REGION") {
@@ -139,6 +145,7 @@ impl Secrets {
         // Build and return
         Secrets {
             database_url,
+            redis_url,
             s3_bucket,
             s3_region,
             s3_path_style,