Implement a naive chunked upload
authorAlexander Ebert <ebert@woltlab.com>
Tue, 26 Dec 2023 14:23:15 +0000 (15:23 +0100)
committerAlexander Ebert <ebert@woltlab.com>
Sat, 8 Jun 2024 10:19:37 +0000 (12:19 +0200)
com.woltlab.wcf/templates/shared_messageFormAttachments.tpl
ts/WoltLabSuite/Core/Component/File/Upload.ts
wcfsetup/install/files/js/WoltLabSuite/Core/Component/File/Upload.js
wcfsetup/install/files/lib/action/FileUploadAction.class.php
wcfsetup/install/files/lib/action/FileUploadPreflightAction.class.php [new file with mode: 0644]
wcfsetup/setup/db/install.sql

index b68bfdb2158f420a9e08eb4ffd7ef150adea45e3..8b190553d770168c5cd08c30f246a520b8e74d2d 100644 (file)
@@ -1,7 +1,7 @@
 <div class="messageTabMenuContent" id="attachments_{if $wysiwygSelector|isset}{$wysiwygSelector}{else}text{/if}">
        <button type="button">
                <woltlab-core-file-upload
-                       data-endpoint="{link controller='FileUpload'}{/link}"
+                       data-endpoint="{link controller='FileUploadPreflight'}{/link}"
                ></woltlab-core-file-upload>
        </button>
        <dl class="wide">
index d886849d3bd89346017faae4513f7e3f976dc2ef..261ab4d9ab145cd70dfcb60fcaca1e75ee307059 100644 (file)
@@ -1,15 +1,28 @@
 import { prepareRequest } from "WoltLabSuite/Core/Ajax/Backend";
 import { wheneverFirstSeen } from "WoltLabSuite/Core/Helper/Selector";
 
+type PreflightResponse = {
+  endpoints: string[];
+};
+
 async function upload(element: WoltlabCoreFileUploadElement, file: File): Promise<void> {
+  const response = (await prepareRequest(element.dataset.endpoint!)
+    .post({
+      filename: file.name,
+      filesize: file.size,
+    })
+    .fetchAsJson()) as PreflightResponse;
+  const { endpoints } = response;
+
   const chunkSize = 2_000_000;
   const chunks = Math.ceil(file.size / chunkSize);
 
   for (let i = 0; i < chunks; i++) {
-    const chunk = file.slice(i * chunkSize, i * chunkSize + chunkSize + 1);
+    const start = i * chunkSize;
+    const end = start + chunkSize;
+    const chunk = file.slice(start, end);
 
-    const response = await prepareRequest(element.dataset.endpoint!).post(chunk).fetchAsResponse();
-    console.log(response);
+    await prepareRequest(endpoints[i]).post(chunk).fetchAsResponse();
   }
 }
 
index 24b61c94e6fac5347f0e35ef64558893a56c7da7..4fdde035d673f430d42c64a1be5debf48f874431 100644 (file)
@@ -3,12 +3,20 @@ define(["require", "exports", "WoltLabSuite/Core/Ajax/Backend", "WoltLabSuite/Co
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.setup = void 0;
     async function upload(element, file) {
+        const response = (await (0, Backend_1.prepareRequest)(element.dataset.endpoint)
+            .post({
+            filename: file.name,
+            filesize: file.size,
+        })
+            .fetchAsJson());
+        const { endpoints } = response;
         const chunkSize = 2000000;
         const chunks = Math.ceil(file.size / chunkSize);
         for (let i = 0; i < chunks; i++) {
-            const chunk = file.slice(i * chunkSize, i * chunkSize + chunkSize + 1);
-            const response = await (0, Backend_1.prepareRequest)(element.dataset.endpoint).post(chunk).fetchAsResponse();
-            console.log(response);
+            const start = i * chunkSize;
+            const end = start + chunkSize;
+            const chunk = file.slice(start, end);
+            await (0, Backend_1.prepareRequest)(endpoints[i]).post(chunk).fetchAsResponse();
         }
     }
     function setup() {
index e17f42aa31369a92f985ea0b4674cbd52a759ca1..4c396ce744ed8022b7829fe9460a44dac592c177 100644 (file)
@@ -6,11 +6,79 @@ use Laminas\Diactoros\Response\EmptyResponse;
 use Psr\Http\Message\ResponseInterface;
 use Psr\Http\Message\ServerRequestInterface;
 use Psr\Http\Server\RequestHandlerInterface;
+use wcf\http\Helper;
+use wcf\system\exception\IllegalLinkException;
+use wcf\system\WCF;
 
 final class FileUploadAction implements RequestHandlerInterface
 {
     public function handle(ServerRequestInterface $request): ResponseInterface
     {
+        // TODO: `sequenceNo` should be of type `non-negative-int`, but requires Valinor 1.7+
+        $parameters = Helper::mapQueryParameters(
+            $request->getQueryParams(),
+            <<<'EOT'
+                    array {
+                        identifier: non-empty-string,
+                        sequenceNo: int,
+                    }
+                    EOT,
+        );
+
+        $sql = "SELECT  *
+                FROM    wcf1_file_temporary
+                WHERE   identifier = ?";
+        $statement = WCF::getDB()->prepare($sql);
+        $statement->execute([$parameters['identifier']]);
+        $row = $statement->fetchSingleRow();
+
+        if ($row === false) {
+            throw new IllegalLinkException();
+        }
+
+        // Check if this is a valid sequence no.
+        // TODO: The chunk calculation shouldn’t be based on a fixed number.
+        $chunkSize = 2_000_000;
+        $chunks = (int)\ceil($row['filesize'] / $chunkSize);
+        if ($parameters['sequenceNo'] >= $chunks) {
+            throw new IllegalLinkException();
+        }
+
+        // Check if the actual size matches the expectations.
+        if ($parameters['sequenceNo'] === $chunks - 1) {
+            // The last chunk is most likely smaller than our chunk size.
+            $expectedSize = $row['filesize'] - $chunkSize * ($chunks - 1);
+        } else {
+            $expectedSize = $chunkSize;
+        }
+
+        $chunk = \file_get_contents('php://input');
+        $actualSize = \strlen($chunk);
+
+        if ($actualSize !== $expectedSize) {
+            throw new IllegalLinkException();
+        }
+
+        $folderA = \substr($row['identifier'], 0, 2);
+        $folderB = \substr($row['identifier'], 2, 2);
+
+        $tmpPath = \sprintf(
+            \WCF_DIR . '_data/private/fileUpload/%s/%s/',
+            $folderA,
+            $folderB,
+        );
+        if (!\is_dir($tmpPath)) {
+            \mkdir($tmpPath, recursive: true);
+        }
+
+        $filename = \sprintf(
+            '%s-%d.bin',
+            $row['identifier'],
+            $parameters['sequenceNo'],
+        );
+
+        \file_put_contents($tmpPath . $filename, $chunk);
+
         // TODO: Dummy response to simulate a successful upload of a chunk.
         return new EmptyResponse();
     }
diff --git a/wcfsetup/install/files/lib/action/FileUploadPreflightAction.class.php b/wcfsetup/install/files/lib/action/FileUploadPreflightAction.class.php
new file mode 100644 (file)
index 0000000..1fd23b2
--- /dev/null
@@ -0,0 +1,67 @@
+<?php
+
+namespace wcf\action;
+
+use Laminas\Diactoros\Response\JsonResponse;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use Psr\Http\Server\RequestHandlerInterface;
+use wcf\http\Helper;
+use wcf\system\request\LinkHandler;
+use wcf\system\WCF;
+
+final class FileUploadPreflightAction implements RequestHandlerInterface
+{
+    public function handle(ServerRequestInterface $request): ResponseInterface
+    {
+        // TODO: For now we only require the filename and size to be provided.
+        $parameters = Helper::mapQueryParameters(
+            $request->getParsedBody(),
+            <<<'EOT'
+                    array {
+                        filename: non-empty-string,
+                        filesize: positive-int,
+                    }
+                    EOT,
+        );
+
+        // TODO: The chunk calculation shouldn’t be based on a fixed number.
+        $chunkSize = 2_000_000;
+        $chunks = (int)\ceil($parameters['filesize'] / $chunkSize);
+
+        $identifier = $this->createTemporaryFile($parameters);
+
+        $endpoints = [];
+        for ($i = 0; $i < $chunks; $i++) {
+            $endpoints[] = LinkHandler::getInstance()->getControllerLink(
+                FileUploadAction::class,
+                [
+                    'identifier' => $identifier,
+                    'sequenceNo' => $i,
+                ]
+            );
+        }
+
+        return new JsonResponse([
+            'endpoints' => $endpoints,
+        ]);
+    }
+
+    private function createTemporaryFile(array $parameters): string
+    {
+        $identifier = \bin2hex(\random_bytes(20));
+
+        $sql = "INSERT INTO     wcf1_file_temporary
+                                (identifier, time, filename, filesize)
+                         VALUES (?, ?, ?, ?)";
+        $statement = WCF::getDB()->prepare($sql);
+        $statement->execute([
+            $identifier,
+            \TIME_NOW,
+            $parameters['filename'],
+            $parameters['filesize'],
+        ]);
+
+        return $identifier;
+    }
+}
index a32722b7fc5b8707f64354671e8c6661630e6c8f..cb59bd13b4c5e1d0974559f7cbeb22ff29c57a00 100644 (file)
@@ -595,20 +595,18 @@ CREATE TABLE wcf1_event_listener (
 
 DROP TABLE IF EXISTS wcf1_file_temporary;
 CREATE TABLE wcf1_file_temporary (
-       uuidv4 CHAR(36) NOT NULL PRIMARY KEY,
-       prefix CHAR(40) NOT NULL,
-       lastModified INT NOT NULL,
+       identifier CHAR(40) NOT NULL PRIMARY KEY,
+       time INT NOT NULL,
        filename VARCHAR(255) NOT NULL,
-       filesize BIGINT NOT NULL,
-       chunks SMALLINT NOT NULL
+       filesize BIGINT NOT NULL
 );
 
 DROP TABLE IF EXISTS wcf1_file_chunk;
 CREATE TABLE wcf1_file_chunk (
-       uuidv4 CHAR(36) NOT NULL,
+       identifier CHAR(40) NOT NULL,
        sequenceNo SMALLINT NOT NULL,
 
-       PRIMARY KEY chunk (uuidv4, sequenceNo)
+       PRIMARY KEY chunk (identifier, sequenceNo)
 );
 
 /* As the flood control table can be a high traffic table and as it is periodically emptied,