Merge pull request #2193 from scpwiki/WJ-1032-file-ops

[WJ-1032] [WJ-1287] Add hard deletion blob operation, add files to seeder

deepwell/Cargo.toml

@@ -8,7 +8,7 @@ keywords = ["wikijump", "api", "backend", "wiki"]
 categories = ["asynchronous", "database", "web-programming::http-server"]
 exclude = [".gitignore", ".editorconfig"]
 
-version = "2024.10.23"
+version = "2024.11.29"
 authors = ["Emmie Smith <[email protected]>"]
 edition = "2021"
 

deepwell/migrations/20220906103252_deepwell.sql

@@ -437,6 +437,13 @@ CREATE TABLE blob_pending (
     CHECK (s3_hash IS NULL OR length(s3_hash) = 64)  -- SHA-512 hash size, if present
 );
 
+-- Manages blobs which are prohibited from being uploaded
+CREATE TABLE blob_blacklist (
+    s3_hash BYTEA PRIMARY KEY CHECK (length(s3_hash) = 64),  -- SHA-512 hash size
+    created_at TIMESTAMP WITH TIME ZONE NULL DEFAULT now(),
+    created_by BIGINT NOT NULL REFERENCES "user"(user_id)
+);
+
 --
 -- Files
 --

deepwell/scripts/request.py

@@ -19,6 +19,14 @@ def color_settings(value):
             return False
 
 
+def parse_data(value):
+    try:
+        return json.loads(value)
+    except json.decoder.JSONDecodeError:
+        # Just interpret as a string
+        return value
+
+
 def print_data(data):
     if isinstance(data, str):
         print(data)
@@ -32,15 +40,22 @@ def print_data(data):
 
 
 def deepwell_request(endpoint, method, data, id=0, color=False):
-    r = requests.post(
-        endpoint,
-        json={
-            "jsonrpc": "2.0",
-            "method": method,
-            "params": data,
-            "id": id,
-        },
-    )
+    try:
+        r = requests.post(
+            endpoint,
+            json={
+                "jsonrpc": "2.0",
+                "method": method,
+                "params": data,
+                "id": id,
+            },
+        )
+    except KeyboardInterrupt:
+        print("Interrupted!")
+        return -1
+    except Exception as exc:
+        print(f"Error: {exc}")
+        return 1
 
     if color:
         green_start = "\x1b[32m"
@@ -98,7 +113,7 @@ def deepwell_request(endpoint, method, data, id=0, color=False):
         default="auto",
     )
     argparser.add_argument("method")
-    argparser.add_argument("data", nargs="?", type=json.loads, default="{}")
+    argparser.add_argument("data", nargs="?", type=parse_data, default="{}")
     args = argparser.parse_args()
     enable_color = color_settings(args.color)
 

deepwell/seeder/.gitattributes

@@ -0,0 +1 @@
+*.svg                   binary

deepwell/seeder/files.json

@@ -0,0 +1,65 @@
+{
+    "www": {
+        "start": [
+            {
+                "name": "default_avatar.png",
+                "path": "default_avatar.png"
+            },
+            {
+                "name": "lock.svg",
+                "path": "lock.svg"
+            },
+            {
+                "name": "null",
+                "path": "empty"
+            }
+        ]
+    },
+    "test": {
+        "system:join": [
+            {
+                "name": "restricted.svg",
+                "path": "lock.svg"
+            },
+            {
+                "name": "restricted2.svg",
+                "path": "lock.svg",
+                "overwrite": "empty"
+            },
+            {
+                "name": "restricted3.svg",
+                "path": "lock.svg",
+                "deleted": true
+            },
+            {
+                "name": "restricted4.svg",
+                "path": "lock.svg",
+                "overwrite": "empty",
+                "deleted": true
+            }
+        ],
+        "system:page-tags": [
+            {
+                "name": "seed_pages.JSON",
+                "path": "pages.json"
+            },
+            {
+                "name": "seed_users.JSON",
+                "path": "users.json"
+            },
+            {
+                "name": "seed_sites.JSON",
+                "path": "sites.json"
+            }
+        ]
+    },
+    "scp-wiki": {
+        "scp-002": [
+            {
+                "name": "SCP.txt",
+                "path": "scp-main.ftml",
+                "overwrite": "scp-002.ftml"
+            }
+        ]
+    }
+}

deepwell/src/api.rs

@@ -263,6 +263,15 @@ async fn build_module(app_state: ServerState) -> anyhow::Result<RpcModule<Server
     register!("blob_upload", blob_upload);
     register!("blob_cancel", blob_cancel);
 
+    // Blob hard deletion
+    register!("blob_hard_delete_preview", blob_hard_delete_preview);
+    register!("blob_hard_delete_confirm", blob_hard_delete_confirm);
+
+    // Blob blacklist
+    register!("blob_blacklist_add", blob_blacklist_add);
+    register!("blob_blacklist_remove", blob_blacklist_remove);
+    register!("blob_blacklist_check", blob_blacklist_check);
+
     // Files
     register!("file_create", file_create);
     register!("file_edit", file_edit);
@@ -271,7 +280,6 @@ async fn build_module(app_state: ServerState) -> anyhow::Result<RpcModule<Server
     register!("file_move", file_move);
     register!("file_restore", file_restore);
     register!("file_rollback", file_rollback);
-    register!("file_hard_delete", file_hard_delete);
 
     // File revisions
     register!("file_revision_get", file_revision_get);

deepwell/src/database/seeder/data.rs

@@ -23,7 +23,7 @@ use anyhow::Result;
 use ftml::layout::Layout;
 use serde::Deserialize;
 use std::collections::HashMap;
-use std::fs::{self, File};
+use std::fs;
 use std::path::{Path, PathBuf};
 use time::Date;
 
@@ -32,6 +32,7 @@ pub struct SeedData {
     pub users: Vec<User>,
     pub sites: Vec<Site>,
     pub pages: HashMap<String, Vec<Page>>,
+    pub files: HashMap<String, HashMap<String, Vec<File>>>,
     pub filters: Vec<Filter>,
 }
 
@@ -63,6 +64,10 @@ impl SeedData {
             }
         }
 
+        // Load file data
+        let files: HashMap<String, HashMap<String, Vec<File>>> =
+            Self::load_json(&mut path, "files")?;
+
         // Load filter data
         let filters: Vec<Filter> = Self::load_json(&mut path, "filters")?;
 
@@ -71,6 +76,7 @@ impl SeedData {
             users,
             sites,
             pages: site_pages,
+            files,
             filters,
         })
     }
@@ -83,7 +89,7 @@ impl SeedData {
         path.set_extension("json");
         debug!("Loading JSON from {}", path.display());
 
-        let mut file = File::open(path)?;
+        let mut file = fs::File::open(path)?;
         let data = serde_json::from_reader(&mut file)?;
         Ok(data)
     }
@@ -180,3 +186,16 @@ pub struct Filter {
     #[serde(default)]
     pub forum: bool,
 }
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct File {
+    pub name: String,
+    pub path: PathBuf,
+
+    #[serde(default)]
+    pub overwrite: Option<PathBuf>,
+
+    #[serde(default)]
+    pub deleted: bool,
+}