From 02e25573e6314e1d444a7853840fcf0516cd7754 Mon Sep 17 00:00:00 2001 From: TCB13 Date: Wed, 23 Oct 2019 12:32:29 +0100 Subject: [PATCH] New methods do deal with finished uploads - Implemented completeAndStream(), completeAndStream() and complete() - MongoDB Backend: increased download buffer size to 10M; - Better method documentation. --- Readme.md | 4 ++- src/Server.php | 55 +++++++++++++++++++++++++++--------- src/Store/FileSystem.php | 20 ++++++++++++- src/Store/MongoDB.php | 41 +++++++++++++++++++++++---- src/Store/Redis.php | 25 +++++++++++++++- src/Store/StorageBackend.php | 9 ++++-- src/Store/StoreInterface.php | 6 ++-- 7 files changed, 133 insertions(+), 27 deletions(-) diff --git a/Readme.md b/Readme.md index 74c7be2..dc75852 100644 --- a/Readme.md +++ b/Readme.md @@ -53,13 +53,15 @@ After the upload is finished you may retrieve the file in another script by call ````php $finalStorageDirectory = "/var/www/html/uploads"; $server = new ThunderTUS\Server(); -$status = $server->fetchFromStorage($filename, $finalStorageDirectory); +$status = $server->completeAndFetch($filename, $finalStorageDirectory); if (!$status) { throw new \Exception("Could not fetch ({$filename}) from storage backend: not found."); } ```` The file will be moved from the temporary storage backend to the `$finalStorageDirectory` directory. +You may also retrieve the final file as a stream with `ThunderTUS\Server::completeAndStream()` or keep on the same place as the temporary parts with `ThunderTUS\Server::complete()` + ## Storage Backends In order to use **ThunderTUS you must pick a storage backend**. Those are used to temporally store the uploaded parts until the upload is completed. Storage backends come in a variety of flavours from the local filesystem to MongoBD's GridFS: diff --git a/src/Server.php b/src/Server.php index 9c6a8bf..047d8c6 100644 --- a/src/Server.php +++ b/src/Server.php @@ -39,7 +39,7 @@ class Server * * @param ?\Psr\Http\Message\ServerRequestInterface $request * @param ?\Psr\Http\Message\ResponseInterface $response - * @param string $streamURI A stream URI from where to get uploaded data. + * @param string $streamURI A stream URI from where to get uploaded data. * */ public function __construct(?ServerRequestInterface $request = null, ?ResponseInterface $response = null, string $streamURI = "php://input") @@ -52,8 +52,8 @@ public function __construct(?ServerRequestInterface $request = null, ?ResponseIn public function loadHTTPInterfaces(ServerRequestInterface $request, ResponseInterface $response) { - $this->request = $request; - $this->response = $response; + $this->request = $request; + $this->response = $response; // Detect the ThunderTUS CrossCheck extension if ($this->request->getHeaderLine("CrossCheck") == true) { @@ -81,26 +81,53 @@ public function getStorageBackend() } /** - * Fetch a finished upload from the current backend storage. - * This method abstracts backend storage file retrivel in a way that the programmer doen't + * Completes an upload and fetches the finished file from the backend storage. + * This method abstracts backend storage file retrivel in a way that the user doen't * need to know what backend storage is being used at all times. * This is useful when the TUS Server is provided by some kind of Service Provider in a * dependency injection context. * - * @param string $filename - * @param string $destinationDirectory - * @param bool $removeAfter + * @param string $filename Name of your file + * @param string $destinationDirectory Where to place the finished file + * @param bool $removeAfter Remove the temporary files after this operation * * @return bool */ - public function fetchFromStorage(string $filename, string $destinationDirectory, bool $removeAfter = true): bool + public function completeAndFetch(string $filename, string $destinationDirectory, bool $removeAfter = true): bool { - return $this->backend->fetchFromStorage($filename, $destinationDirectory, $removeAfter); + return $this->backend->completeAndFetch($filename, $destinationDirectory, $removeAfter); } - public function streamFromStorage(string $filename, bool $removeAfter = true) + /** + * Completes an upload and returns the finished file in the form of a stream. + * Useful when you want to upload the file to another system without writting + * it to the disk most of the time. + * This method uses PHP's tmp stream to merge the file parts. Ajust it accordingly. + * + * @param string $filename Name of your file + * @param bool $removeAfter Remove the temporary files after this operation + * + * @return bool + */ + public function completeAndStream(string $filename, bool $removeAfter = true) + { + return $this->backend->completeAndStream($filename, $removeAfter); + } + + /** + * Completes an upload without fetching it. The file will be placed in the + * same backend storage you're using for the temporary part upload. + * Useful when you want to keep the finished file in the same storage backend + * you're using for the temporary part upload. + * This method uses PHP's tmp stream to merge the file parts. Ajust it accordingly. + * + * @param string $filename Name of your file + * + * @return bool + */ + public function complete(string $name): bool { - return $this->backend->streamFromStorage($filename, $removeAfter); + return $this->backend->complete($name); } /** @@ -191,7 +218,7 @@ protected function handlePOST(): ResponseInterface // Extension Thunder TUS CrossCheck: get complete upload checksum if ($this->extCrossCheck) { - $supportedAlgos = $this->backend->supportsCrossCheck() ? $this->backend->getCrossCheckAlgoritms() : hash_algos(); + $supportedAlgos = $this->backend->supportsCrossCheck() ? $this->backend->getCrossCheckAlgoritms() : hash_algos(); $cache->checksum = self::parseChecksum($this->request->getHeaderLine("Upload-CrossChecksum")); if ($cache->checksum === false || !in_array($cache->checksum->algorithm, $supportedAlgos)) { return $this->response->withStatus(400); @@ -267,7 +294,7 @@ protected function handlePATCH(): ResponseInterface // Check if the server supports the proposed checksum algorithm $supportedAlgos = $this->backend->supportsCrossCheck() ? $this->backend->getCrossCheckAlgoritms() : hash_algos(); - $checksum = self::parseChecksum($this->request->getHeaderLine("Upload-Checksum")); + $checksum = self::parseChecksum($this->request->getHeaderLine("Upload-Checksum")); if ($checksum === false || !in_array($checksum->algorithm, $supportedAlgos)) { return $this->response->withStatus(400); } diff --git a/src/Store/FileSystem.php b/src/Store/FileSystem.php index 6a5eec4..d7e2c4a 100644 --- a/src/Store/FileSystem.php +++ b/src/Store/FileSystem.php @@ -67,7 +67,7 @@ public function delete(string $name): bool return true; } - public function fetchFromStorage(string $name, string $destinationDirectory, bool $removeAfter = true): bool + public function completeAndFetch(string $name, string $destinationDirectory, bool $removeAfter = true): bool { $destinationDirectory = self::normalizePath($destinationDirectory); if ($destinationDirectory === $this->uploadDir) { @@ -81,6 +81,24 @@ public function fetchFromStorage(string $name, string $destinationDirectory, boo } } + public function completeAndStream(string $name, bool $removeAfter = true) + { + $stream = fopen($this->uploadDir . $name, "r"); + if ($removeAfter) { + $final = fopen("php://temp", "r+"); + stream_copy_to_stream($stream, $final); + fclose($stream); + return unlink($this->uploadDir . $name); + } else { + return $stream; + } + } + + public function complete(string $name): bool + { + return true; + } + public function supportsCrossCheck(): bool { return true; diff --git a/src/Store/MongoDB.php b/src/Store/MongoDB.php index 40ddc1d..1870fcc 100644 --- a/src/Store/MongoDB.php +++ b/src/Store/MongoDB.php @@ -12,6 +12,7 @@ class MongoDB extends StorageBackend private static $bucketName = "tus"; private static $containerPrefix = "container."; + private static $partBufferSize = 10000000; public function __construct(\MongoDB\Database $database) { @@ -61,7 +62,7 @@ public function delete(string $name): bool return true; } - public function fetchFromStorage(string $name, string $destinationDirectory, bool $removeAfter = true): bool + public function completeAndFetch(string $name, string $destinationDirectory, bool $removeAfter = true): bool { $parts = $this->get($name, true); if (empty($parts) || $parts === null) { @@ -69,15 +70,13 @@ public function fetchFromStorage(string $name, string $destinationDirectory, boo } $parts = array_column($parts, "_id"); - // Create or open the file to store fata + // Read the gridfs parts file into local storage 10MB at the time $destinationDirectory = self::normalizePath($destinationDirectory); $file = fopen($destinationDirectory . $name, 'w'); - - // Read the gridfs file into local storage 5MB at the time foreach ($parts as $part) { $stream = $this->bucket->openDownloadStream($part); while (!feof($stream)) { - fwrite($file, fread($stream, 10000000)); + fwrite($file, fread($stream, self::$partBufferSize)); } fclose($stream); // Delete part from mongodb @@ -90,7 +89,7 @@ public function fetchFromStorage(string $name, string $destinationDirectory, boo return true; } - public function streamFromStorage(string $name, bool $removeAfter = true) + public function completeAndStream(string $name, bool $removeAfter = true) { $parts = $this->get($name, true); if (empty($parts) || $parts === null) { @@ -98,6 +97,7 @@ public function streamFromStorage(string $name, bool $removeAfter = true) } $parts = array_column($parts, "_id"); + // Read the gridfs parts file a final local tmp stream $final = fopen("php://temp", "r+"); foreach ($parts as $part) { $partStream = $this->bucket->openDownloadStream($part); @@ -112,6 +112,35 @@ public function streamFromStorage(string $name, bool $removeAfter = true) return $final; } + public function complete(string $name): bool + { + $parts = $this->get($name, true); + if (empty($parts) || $parts === null) { + return false; + } + $parts = array_column($parts, "_id"); + + // Read the gridfs parts file into a local tmp 10MB at the time + $final = fopen("php://temp", "r+"); + foreach ($parts as $part) { + $stream = $this->bucket->openDownloadStream($part); + while (!feof($stream)) { + stream_copy_to_stream(fread($stream, self::$partBufferSize), $final); + } + fclose($stream); + // Delete part from mongodb + if ($removeAfter) { + $this->bucket->delete($part); + } + } + + // We now have a final tmp with the entrie file upload it to mongodb + rewind($final); + $this->bucket->uploadFromStream($name, $final); + + return true; + + } public function containerExists(string $name): bool { diff --git a/src/Store/Redis.php b/src/Store/Redis.php index c6941d1..c09bb27 100644 --- a/src/Store/Redis.php +++ b/src/Store/Redis.php @@ -63,7 +63,7 @@ public function delete(string $name): bool return $this->client->del([self::$prefix . $name]); } - public function fetchFromStorage(string $name, string $destinationDirectory, bool $removeAfter = true): bool + public function completeAndFetch(string $name, string $destinationDirectory, bool $removeAfter = true): bool { $data = $this->get($name); if ($data === null) { @@ -82,6 +82,29 @@ public function fetchFromStorage(string $name, string $destinationDirectory, boo return true; } + public function completeAndStream(string $name, bool $removeAfter = true) + { + $data = $this->get($name); + if ($data === null) { + return false; + } + + $final = fopen("php://temp", "r+"); + fwrite($stream, $string); + rewind($stream); + + if ($removeAfter) { + $this->delete($name); + } + + return $stream; + } + + public function complete(string $name): bool + { + return true; + } + public function containerExists(string $name): bool { return $this->client->exists(static::$containerPrefix . $name) == 1; diff --git a/src/Store/StorageBackend.php b/src/Store/StorageBackend.php index f44503f..eb25834 100644 --- a/src/Store/StorageBackend.php +++ b/src/Store/StorageBackend.php @@ -22,9 +22,14 @@ public function getCrossCheckAlgoritms(): array return []; } - public function streamFromStorage(string $name, bool $removeAfter = true) + public function completeAndStream(string $name, bool $removeAfter = true) { - throw new ThunderTUSException("The " . static::class . " storage backend hasn't implemented 'streamFromStorage'. Please use 'fetchFromStorage' to fetch the complete file into the local filesystem."); + throw new ThunderTUSException("The " . static::class . " storage backend hasn't implemented 'completeAndStream'. Please use 'fetchFromStorage' to fetch the complete file into the local filesystem."); + } + + public function complete(string $name): bool + { + throw new ThunderTUSException("The " . static::class . " storage backend hasn't implemented 'complete'. Please use 'fetchFromStorage' to fetch the complete file into the local filesystem."); } public static function normalizePath(string $path): string diff --git a/src/Store/StoreInterface.php b/src/Store/StoreInterface.php index d26d1b6..84badc9 100644 --- a/src/Store/StoreInterface.php +++ b/src/Store/StoreInterface.php @@ -9,8 +9,10 @@ public function create(string $name): bool; public function getSize(string $name): int; public function append(string $name, $data): bool; public function delete(string $name): bool; - public function fetchFromStorage(string $name, string $destinationDirectory, bool $removeAfter = true): bool; - public function streamFromStorage(string $name, bool $removeAfter = true); + + public function completeAndFetch(string $name, string $destinationDirectory, bool $removeAfter = true): bool; + public function completeAndStream(string $name, bool $removeAfter = true); + public function complete(string $name): bool; public function supportsCrossCheck(): bool; public function crossCheck(string $name, string $algo, string $expectedHash): bool;