Merge pull request #91 from demarches-simplifiees/less_memory

Less memory

launch_demo.sh

@@ -13,7 +13,7 @@ echo 'building simple node server which mimics a backend storage'
 npm install --prefix tests/fixtures/server-static
 
 echo 'building keyring file'
-./target/debug/ds_proxy add-key --password-file <(echo -n "$PASSWORD") --keyring-file "$KEYRING_FILE" --salt "$SALT"
+./target/release/ds_proxy add-key --password-file <(echo -n "$PASSWORD") --keyring-file "$KEYRING_FILE" --salt "$SALT"
 
 if [ "$1" = "aws" ]; then
   echo 'launching ds_proxy in aws mode listenning on real s3 backend'

src/crypto/encoder.rs

@@ -75,11 +75,19 @@ impl<E> Encoder<E> {
                 Some(ref mut stream) => {
                     trace!("stream encoder present !");
                     if self.chunk_size <= self.buffer.len() {
-                        trace!("encoding a whole chunk");
-                        let encoded = stream
-                            .push(&self.buffer.split_to(self.chunk_size), None, Tag::Message)
-                            .unwrap();
-                        Poll::Ready(Some(Ok(Bytes::from(encoded))))
+                        let mut encoded_buff = BytesMut::with_capacity(self.buffer.len());
+
+                        while self.chunk_size <= self.buffer.len() {
+                            trace!("encoding a whole chunk");
+
+                            let encoded_message = stream
+                                .push(&self.buffer.split_to(self.chunk_size), None, Tag::Message)
+                                .unwrap();
+
+                            encoded_buff.extend_from_slice(&encoded_message);
+                        }
+
+                        Poll::Ready(Some(Ok(encoded_buff.freeze())))
                     } else {
                         trace!("the chunk is not complete");
                         if self.inner_ended {

src/http/handlers/encrypt_to_file.rs

@@ -1,6 +1,6 @@
+use tokio::{fs::OpenOptions, io::AsyncWriteExt};
+
 use super::*;
-use std::fs::File;
-use std::io::Write;
 
 pub async fn encrypt_to_file(
     req: HttpRequest,
@@ -16,19 +16,20 @@ pub async fn encrypt_to_file(
 
     let mut encrypted_stream = Encoder::new(key, id, config.chunk_size, Box::new(payload));
 
-    // File::create is blocking operation, use threadpool
-    let mut f = web::block(move || File::create(filepath))
+    let mut f = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(filepath)
         .await
-        .unwrap()
         .unwrap();
 
     while let Ok(Some(chunk)) = encrypted_stream.try_next().await {
-        f = web::block(move || f.write_all(&chunk).map(|_| f))
-            .await
-            .unwrap()
-            .unwrap();
+        f.write_all(&chunk).await.unwrap();
     }
 
+    f.sync_all().await.unwrap();
+
     HttpResponse::Ok()
         .insert_header((header::CONTENT_TYPE, "application/json"))
         .body("{}")