Fix tests for CI, increase body limit for http

.github/workflows/test.yaml

@@ -49,8 +49,6 @@ jobs:
         run: RUN_POSTGRES=1 ./ci/scripts/build.sh
         env:
           PG_VERSION: ${{ matrix.postgres }}
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
       - name: Run tests
         run: cargo test --workspace --exclude lantern_extras -- --nocapture --test-threads=1
         env:

lantern_cli/src/http_server/collection.rs

@@ -2,7 +2,7 @@ use bytes::BytesMut;
 use futures::SinkExt;
 use itertools::Itertools;
 use regex::Regex;
-use std::collections::HashMap;
+use std::{collections::HashMap, time::Instant};
 
 use actix_web::{
     delete,
@@ -347,25 +347,40 @@ async fn insert_data(
         .map_err(ErrorInternalServerError)?;
 
     futures::pin_mut!(writer_sink);
-    let mut buf = BytesMut::new();
+    let chunk_size = 1024 * 1024 * 100; // 10 MB
+    let mut buf = BytesMut::with_capacity(chunk_size * 2);
+    let mut row_cap = 2000;
+    let now = Instant::now();
     for row in &body.rows {
-        let mut row_str = String::from("");
+        let mut row_str = String::with_capacity(row_cap);
+        // let now = Instant::now();
         for (idx, column) in columns.iter().enumerate() {
-            let entry = row[column].to_string().replace("[", "{").replace("]", "}");
+            let elem = row[column].to_string();
+
+            let mut chars = elem.chars();
+            let first_char = chars.next().unwrap();
+            let last_char = chars.next_back().unwrap();
+            let entry = if first_char == '[' && last_char == ']' {
+                format!("{{{}}}", chars.as_str())
+            } else {
+                elem
+            };
+
             row_str.push_str(&entry);
             if idx != columns.len() - 1 {
                 row_str.push_str("\t");
             }
         }
         row_str.push_str("\n");
 
-        if buf.len() > 4096 {
+        if buf.len() > chunk_size {
             writer_sink
                 .send(buf.split().freeze())
                 .await
                 .map_err(ErrorInternalServerError)?;
         }
 
+        row_cap = row_str.len();
         buf.extend_from_slice(row_str.as_bytes());
     }
 
@@ -375,13 +390,16 @@ async fn insert_data(
             .await
             .map_err(ErrorInternalServerError)?;
     }
+    println!("Copy took {}s", now.elapsed().as_secs());
 
+    let now = Instant::now();
     writer_sink.finish().await.map_err(ErrorBadRequest)?;
 
     transaction
         .commit()
         .await
         .map_err(ErrorInternalServerError)?;
+    println!("Commit took {}s", now.elapsed().as_secs());
 
     Ok(HttpResponse::new(StatusCode::from_u16(200).unwrap()))
 }

lantern_cli/src/http_server/index.rs

@@ -87,17 +87,21 @@ async fn create_index(
             )
             .await.map_err(ErrorInternalServerError)?;
     } else {
-        let data_dir = match client
-            .query_one(
-                "SELECT setting::text FROM pg_settings WHERE name = 'data_directory'",
-                &[],
-            )
-            .await
-        {
-            Err(e) => {
-                return Err(ErrorInternalServerError(e));
+        let data_dir = if !data.is_remote_database {
+            match client
+                .query_one(
+                    "SELECT setting::text FROM pg_settings WHERE name = 'data_directory'",
+                    &[],
+                )
+                .await
+            {
+                Err(e) => {
+                    return Err(ErrorInternalServerError(e));
+                }
+                Ok(row) => row.get::<usize, String>(0),
             }
-            Ok(row) => row.get::<usize, String>(0),
+        } else {
+            "/tmp".to_owned()
         };
 
         let mut rng = rand::thread_rng();

lantern_cli/src/http_server/mod.rs

@@ -114,6 +114,11 @@ pub async fn start(
         App::new()
             .wrap(Logger::new("%r - %s %Dms"))
             .app_data(state.clone())
+            .app_data(
+                web::JsonConfig::default()
+                    // limit request payload size to 1GB
+                    .limit(1024 * 1024 * 1024),
+            )
             .service(
                 SwaggerUi::new("/swagger-ui/{_:.*}")
                     .url("/api-docs/openapi.json", ApiDoc::openapi()),

lantern_cli/tests/http_server_test_with_db.rs

@@ -41,7 +41,7 @@ fn start_server(db_uri: String) -> Sender<()> {
             http_server::start(
                 HttpServerArgs {
                     db_uri,
-                    remote_database: false,
+                    remote_database: true,
                     host: "127.0.0.1".to_owned(),
                     port: 7777,
                 },