Encrypt uploads in the browser and send them in chunks

A 6 GB upload kept a customer waiting long after its progress bar reached 100%. The server wrote every upload three times: PHP's temporary file, Livewire's copy of it ("Processing files...") and the encrypted file ("Create Share Link"), each a full rewrite of a slow disk. The unencrypted copy also stayed behind in livewire-tmp. Now the uploader's browser encrypts each file in 16 MB chunks with WebCrypto and PUTs them one at a time; the server checks each chunk in memory and writes it once, already encrypted. Creating the share only wraps its key and saves the options. A 200 MB upload through the Docker image took 2.8 s, and its download matched byte for byte. - SEALCHK2: a 19-byte header (chunk size, 7-byte nonce prefix), then ciphertext and tag per chunk. Each nonce holds the chunk index and a last-chunk flag (the STREAM construction), so cut or reordered files fail to decrypt. SEALCHK1 and the single-block format still read. - Envelope encryption: one random key per share. With a password it is wrapped with Argon2id (sodium, libsodium's interactive limits) in shares.wrapped_key, which names its parameters. Password shares from before keep their PBKDF2-derived key. - The upload page registers each selection with FileUploader into a pending share of its own, lists the files with their progress, retries a failed chunk after 1-16 s, then offers Retry; Remove and Cancel abort. UploadChunkController only accepts chunks from the session that started the share: a repeat is acknowledged, a skip gets 409 with the count stored. Chunks go out as Blobs, which Chromium sends about eight times faster than ArrayBuffers. - Uploads need a secure context: over plain HTTP the page says HTTPS is needed and takes no files. The Docker image gains AUTO_HTTPS, which serves Let's Encrypt on 443 for SERVER_NAME and redirects 80; without it the container stays on HTTP 80 behind a proxy. docker/Caddyfile was never loaded and is gone; docker/healthcheck.sh covers both modes. - "Download all" streams the ZIP with maennchen/zipstream-php (STORE, ZIP64) instead of decrypting whole files into memory and writing the archive unencrypted to /tmp. - Pending shares count towards the quota, stay out of the admin dashboard and 404 everywhere else. shares:cleanup deletes uploads idle for 4 hours and Livewire temporary files older than that. - PHP's upload limits no longer cap the admin's max file size and default to 64M; LIVEWIRE_MAX_UPLOAD_TIME is gone and UPLOAD_CHUNK_SIZE_MB is new. - Tests cover the format, key wrapping, registration limits, the chunk endpoint's answers, completing a share, the streamed ZIP, cleanup, and in Chromium a real chunked upload and the HTTPS warning; the selected-files overflow test runs again. README, website, CHANGELOG and .ai/rules follow. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 20:49:17 +02:00
parent 504971ad7f
commit 40e35bab0e
53 changed files with 2280 additions and 946 deletions
@@ -4,13 +4,41 @@ use App\Livewire\FileUploader;
 use App\Livewire\SystemPasswordPrompt;
 use App\Models\Setting;
 use App\Models\Share;
+use App\Models\ShareFile;
 use App\Services\ShareService;
-use Illuminate\Http\UploadedFile;
 use Illuminate\Support\Facades\Crypt;
-use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
+use Livewire\Features\SupportTesting\Testable;
 use Livewire\Livewire;

+/**
+ * Upload files through the page as the browser does: register them in one batch, then store each
+ * one's encrypted content (the chunk endpoint itself is covered by UploadChunkTest).
+ *
+ * @param  array<string, string>  $files  name => content
+ * @return array<int, array<string, mixed>|null> what the page handed the browser
+ */
+function uploadThroughPage(Testable $component, array $files): array
+{
+    $targets = [];
+
+    $component->call('registerFiles', collect($files)->map(fn (string $content, string $name): array => ['name' => $name, 'size' => strlen($content), 'path' => null])->values()->all())
+        ->assertReturned(function (array $returned) use (&$targets): bool {
+            $targets = $returned;
+
+            return true;
+        });
+
+    foreach (array_values($files) as $position => $content) {
+        if ($targets[$position] !== null) {
+            $file = ShareFile::query()->findOrFail($targets[$position]['id']);
+            app(ShareService::class)->storeChunk($file, 0, encryptedChunk($file, $content, 0, true));
+        }
+    }
+
+    return $targets;
+}
+
 test('upload page can be rendered', function () {
    $response = $this->get(route('upload'));

@@ -36,28 +64,100 @@ test('upload page accessible after system password verified', function () {

 test('file upload creates share', function () {
    Storage::fake('shares');
+    $component = Livewire::test(FileUploader::class);
+    uploadThroughPage($component, ['document.pdf' => 'the document']);

-    $file = UploadedFile::fake()->create('document.pdf', 1024);
-
-    Livewire::test(FileUploader::class)
-        ->set('files', [$file])
-        ->call('createShare')
+    $component->call('createShare')
        ->assertRedirectContains('/share/');

-    expect(Share::query()->count())->toBe(1);
+    $share = Share::query()->sole();
+    expect($share->isCompleted())->toBeTrue();
+    expect($share->files->pluck('original_name')->all())->toBe(['document.pdf']);
+    expect(session('pending_shares'))->not->toContain($share->token);
+});

-    $share = Share::query()->first();
-    expect($share->files)->toHaveCount(1);
-    expect($share->files->first()->original_name)->toBe('document.pdf');
+test('registering files hands the browser each file\'s chunk URL, the share key and the file\'s nonce prefix', function () {
+    Storage::fake('shares');
+    config(['uploads.chunk_size' => 4]);
+    $component = Livewire::test(FileUploader::class);
+
+    $targets = uploadThroughPage($component, ['a.txt' => 'abc']);
+
+    $file = ShareFile::query()->sole();
+    expect($targets[0])->toMatchArray([
+        'id' => $file->id,
+        'url' => url('upload/files/'.$file->id.'/chunks'),
+        'key' => $file->share->encryption_key,
+        'chunkSize' => 4,
+        'chunkCount' => 1,
+    ]);
+    expect($targets[0]['noncePrefix'])->toBe(bin2hex(substr(file_get_contents(app(ShareService::class)->storedFilePath($file)), 12, 7)));
+    expect(session('pending_shares'))->toBe([$file->share->token]);
+    $component->assertSet('pendingToken', $file->share->token)
+        ->assertSeeHtml('data-test="selected-file"');
+});
+
+test('each upload page gets its own pending share', function () {
+    Storage::fake('shares');
+    $first = Livewire::test(FileUploader::class);
+    $second = Livewire::test(FileUploader::class);
+
+    uploadThroughPage($first, ['one.txt' => 'one']);
+    uploadThroughPage($second, ['two.txt' => 'two']);
+
+    expect($first->get('pendingToken'))->not->toBe($second->get('pendingToken'));
+    expect(Share::query()->pluck('total_size')->all())->toBe([3, 3]);
+});
+
+test('a file an admin limit refuses gets no target and shows why, while the rest of the batch is registered', function () {
+    Storage::fake('shares');
+    Setting::set('max_file_size', 1024 * 1024);
+    $component = Livewire::test(FileUploader::class);
+
+    $component->call('registerFiles', [
+        ['name' => 'small.txt', 'size' => 3, 'path' => null],
+        ['name' => 'large.txt', 'size' => 2 * 1024 * 1024, 'path' => null],
+    ]);
+
+    $component->assertReturned(fn (array $targets): bool => $targets[0] !== null && $targets[1] === null);
+    expect($component->errors()->first('files'))->toBe('"large.txt" is too large (2 MB). Maximum file size is 1 MB.');
+    expect(ShareFile::query()->pluck('original_name')->all())->toBe(['small.txt']);
+});
+
+test('removing files takes them out of the pending share', function () {
+    Storage::fake('shares');
+    $component = Livewire::test(FileUploader::class);
+    $targets = uploadThroughPage($component, ['keep.txt' => 'keep', 'remove.txt' => 'remove']);
+
+    $component->call('removeFiles', [$targets[1]['id']]);
+
+    expect(ShareFile::query()->pluck('original_name')->all())->toBe(['keep.txt']);
+    expect(Share::query()->sole()->total_size)->toBe(4);
+});
+
+test('a share cannot be created while a file is still uploading', function () {
+    Storage::fake('shares');
+    $component = Livewire::test(FileUploader::class)
+        ->call('registerFiles', [['name' => 'unfinished.txt', 'size' => 10, 'path' => null]]);
+
+    $component->call('createShare');
+
+    expect($component->errors()->first('files'))->toBe('Wait until every file has finished uploading, or remove the ones that failed.');
+    expect(Share::query()->sole()->isCompleted())->toBeFalse();
+});
+
+test('the page offers a warning for browsers without a secure context', function () {
+    Livewire::test(FileUploader::class)
+        ->assertSeeHtml('data-test="insecure-context"')
+        ->assertSee('Uploads need a secure connection (HTTPS).');
 });

 test('file upload with password creates password-protected share', function () {
    Storage::fake('shares');
+    $component = Livewire::test(FileUploader::class);
+    uploadThroughPage($component, ['secret.txt' => 'secret']);

-    $file = UploadedFile::fake()->create('secret.txt', 512);
-
-    Livewire::test(FileUploader::class)
-        ->set('files', [$file])
+    $component
        ->set('usePassword', true)
        ->set('password', 'my-password')
        ->call('createShare')
@@ -79,9 +179,9 @@ test('switching password protection on leaves the field empty and offers a Gener
 test('a generated password protects the share and is flashed, encrypted, for the page the upload leads to', function () {
    Storage::fake('shares');

-    $component = Livewire::test(FileUploader::class)
-        ->set('files', [UploadedFile::fake()->create('secret.txt', 512)])
-        ->set('usePassword', true)
+    $component = Livewire::test(FileUploader::class);
+    uploadThroughPage($component, ['secret.txt' => 'secret']);
+    $component->set('usePassword', true)
        ->call('generatePassword');
    $password = $component->get('password');
    $component->call('createShare');
@@ -96,9 +196,10 @@ test('a generated password protects the share and is flashed, encrypted, for the
 test('a share without a password flashes no password', function () {
    Storage::fake('shares');

-    Livewire::test(FileUploader::class)
-        ->set('files', [UploadedFile::fake()->create('document.pdf', 100)])
-        ->call('createShare')
+    $component = Livewire::test(FileUploader::class);
+    uploadThroughPage($component, ['document.pdf' => 'the document']);
+
+    $component->call('createShare')
        ->assertRedirectContains('/share/');

    expect(session()->has('share_password'))->toBeFalse();
@@ -136,10 +237,10 @@ test('a generator switched off offers no Generate button and generates nothing',
 test('file upload with expiration sets expires_at', function () {
    Storage::fake('shares');

-    $file = UploadedFile::fake()->create('file.txt', 256);
+    $component = Livewire::test(FileUploader::class);
+    uploadThroughPage($component, ['file.txt' => 'content']);

-    Livewire::test(FileUploader::class)
-        ->set('files', [$file])
+    $component
        ->set('expiration', '24h')
        ->call('createShare')
        ->assertRedirectContains('/share/');
@@ -151,10 +252,10 @@ test('file upload with expiration sets expires_at', function () {
 test('file upload with max downloads sets limit', function () {
    Storage::fake('shares');

-    $file = UploadedFile::fake()->create('file.txt', 256);
+    $component = Livewire::test(FileUploader::class);
+    uploadThroughPage($component, ['file.txt' => 'content']);

-    Livewire::test(FileUploader::class)
-        ->set('files', [$file])
+    $component
        ->set('maxDownloads', 5)
        ->call('createShare')
        ->assertRedirectContains('/share/');
@@ -163,69 +264,26 @@ test('file upload with max downloads sets limit', function () {
    expect($share->max_downloads)->toBe(5);
 });

-test('every upload batch dispatches files-processed to clear the uploading state', function () {
-    Storage::fake('shares');
-
-    Livewire::test(FileUploader::class)
-        ->set('files', [UploadedFile::fake()->create('first.txt', 64)])
-        ->assertDispatched('files-processed')
-        ->set('files', [UploadedFile::fake()->create('second.txt', 64)])
-        ->assertDispatched('files-processed');
-});
-
-test('files added in multiple batches end up in the same share', function () {
-    Storage::fake('shares');
-
-    Livewire::test(FileUploader::class)
-        ->set('files', [UploadedFile::fake()->create('first.txt', 64)])
-        ->set('files', [UploadedFile::fake()->create('second.txt', 64)])
-        ->call('createShare')
-        ->assertHasNoErrors()
-        ->assertRedirectContains('/share/');
-
-    $share = Share::query()->first();
-
-    expect($share->files->pluck('original_name')->all())->toBe(['first.txt', 'second.txt']);
-});
-
-test('files larger than 4 GB can be shared when within the admin file size limit', function () {
+test('a file of 6 GB is accepted when the admin limits allow it', function () {
    Storage::fake('shares');
    Setting::set('max_file_size', 15000 * 1024 * 1024);
    Setting::set('max_size_per_share', 20 * 1024 * 1024 * 1024);
-
-    Livewire::test(FileUploader::class)
-        ->set('files', [UploadedFile::fake()->create('backup.dump', 6 * 1024 * 1024)])
-        ->assertHasNoErrors('files')
-        ->call('createShare')
-        ->assertHasNoErrors()
-        ->assertRedirectContains('/share/');
-
-    expect(Share::query()->first()->total_size)->toBe(6 * 1024 * 1024 * 1024);
-});
-
-test('a rejected upload logs the real reason instead of blaming the file size limit', function () {
-    Log::spy();
-    Setting::set('max_file_size', 15000 * 1024 * 1024);
-
-    $errors = ['files.0' => ['The files.0 failed to upload.']];
+    Setting::set('max_storage_quota', 50 * 1024 * 1024 * 1024);

    $component = Livewire::test(FileUploader::class)
-        ->call('_uploadErrored', 'files', json_encode(['errors' => $errors]), true)
-        ->assertDispatched('upload:errored');
+        ->call('registerFiles', [['name' => 'backup.dump', 'size' => 6 * 1024 * 1024 * 1024, 'path' => null]]);

-    expect($component->errors()->first('files'))
-        ->toBe('Upload failed: the server could not accept the file. Please try again or contact the administrator.');
-
-    Log::shouldHaveReceived('warning')
-        ->withArgs(fn (string $message, array $context): bool => $context['errors'] === $errors)
-        ->once();
+    $component->assertHasNoErrors('files');
+    expect(Share::query()->sole()->total_size)->toBe(6 * 1024 * 1024 * 1024);
+    expect(ShareFile::query()->sole()->file_size)->toBe(6 * 1024 * 1024 * 1024);
 });

 test('file upload requires at least one file', function () {
-    Livewire::test(FileUploader::class)
-        ->set('files', [])
-        ->call('createShare')
-        ->assertHasErrors(['files']);
+    $component = Livewire::test(FileUploader::class)
+        ->call('createShare');
+
+    expect($component->errors()->first('files'))->toBe('Please select at least one file to upload.');
+    expect(Share::query()->count())->toBe(0);
 });

 test('file upload blocks when storage is full', function () {
@@ -233,12 +291,11 @@ test('file upload blocks when storage is full', function () {
    Setting::set('max_storage_quota', 100);
    Share::factory()->create(['total_size' => 100]);

-    $file = UploadedFile::fake()->create('file.txt', 1);
+    $component = Livewire::test(FileUploader::class)
+        ->call('registerFiles', [['name' => 'file.txt', 'size' => 1, 'path' => null]]);

-    Livewire::test(FileUploader::class)
-        ->set('files', [$file])
-        ->call('createShare')
-        ->assertHasErrors(['files']);
+    expect($component->errors()->first('files'))->toBe('Storage is full. Please contact the administrator.');
+    expect(ShareFile::query()->count())->toBe(0);
 });

 test('system password prompt verifies correct password', function () {