Encrypt uploads in the browser and send them in chunks
A 6 GB upload kept a customer waiting long after its progress bar
reached 100%. The server wrote every upload three times: PHP's
temporary file, Livewire's copy of it ("Processing files...") and the
encrypted file ("Create Share Link"), each a full rewrite of a slow
disk. The unencrypted copy also stayed behind in livewire-tmp.
Now the uploader's browser encrypts each file in 16 MB chunks with
WebCrypto and PUTs them one at a time; the server checks each chunk in
memory and writes it once, already encrypted. Creating the share only
wraps its key and saves the options. A 200 MB upload through the
Docker image took 2.8 s, and its download matched byte for byte.
- SEALCHK2: a 19-byte header (chunk size, 7-byte nonce prefix), then
ciphertext and tag per chunk. Each nonce holds the chunk index and a
last-chunk flag (the STREAM construction), so cut or reordered files
fail to decrypt. SEALCHK1 and the single-block format still read.
- Envelope encryption: one random key per share. With a password it is
wrapped with Argon2id (sodium, libsodium's interactive limits) in
shares.wrapped_key, which names its parameters. Password shares from
before keep their PBKDF2-derived key.
- The upload page registers each selection with FileUploader into a
pending share of its own, lists the files with their progress, retries
a failed chunk after 1-16 s, then offers Retry; Remove and Cancel
abort. UploadChunkController only accepts chunks from the session that
started the share: a repeat is acknowledged, a skip gets 409 with the
count stored. Chunks go out as Blobs, which Chromium sends about eight
times faster than ArrayBuffers.
- Uploads need a secure context: over plain HTTP the page says HTTPS is
needed and takes no files. The Docker image gains AUTO_HTTPS, which
serves Let's Encrypt on 443 for SERVER_NAME and redirects 80; without
it the container stays on HTTP 80 behind a proxy. docker/Caddyfile was
never loaded and is gone; docker/healthcheck.sh covers both modes.
- "Download all" streams the ZIP with maennchen/zipstream-php (STORE,
ZIP64) instead of decrypting whole files into memory and writing the
archive unencrypted to /tmp.
- Pending shares count towards the quota, stay out of the admin
dashboard and 404 everywhere else. shares:cleanup deletes uploads idle
for 4 hours and Livewire temporary files older than that.
- PHP's upload limits no longer cap the admin's max file size and
default to 64M; LIVEWIRE_MAX_UPLOAD_TIME is gone and
UPLOAD_CHUNK_SIZE_MB is new.
- Tests cover the format, key wrapping, registration limits, the chunk
endpoint's answers, completing a share, the streamed ZIP, cleanup,
and in Chromium a real chunked upload and the HTTPS warning; the
selected-files overflow test runs again. README, website, CHANGELOG
and .ai/rules follow.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andreas Reinhold / reini
co-authored by
Claude Opus 5
Claude Opus 5
parent
504971ad7f
commit
40e35bab0e
@@ -1,3 +1,4 @@
|
||||
// Livewire Material. Alpine is bundled and started by Livewire 4: never import it here as well.
|
||||
import '../../vendor/nonameweb/livewire-material/resources/js/material.js'
|
||||
import './share-created.js'
|
||||
import './share-uploader.js'
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
/**
|
||||
* `shareUploader`: the upload page's queue. Files are registered with the Livewire component in one
|
||||
* batch per selection, then sent one at a time: each chunk is sliced from the file, encrypted here
|
||||
* with AES-256-GCM and PUT on its own, so the server writes it once, already encrypted.
|
||||
*
|
||||
* The encrypted format is App\Services\FileEncryptionService's SEALCHK2: chunk i's nonce is the
|
||||
* file's 7-byte prefix, i as a big-endian uint32 and a byte that is 1 on the last chunk; WebCrypto
|
||||
* appends the 16-byte tag to the ciphertext, which is how the server stores it.
|
||||
*
|
||||
* A failed request is retried after 1, 2, 4, 8 and 16 seconds; after that the file waits for its
|
||||
* Retry button, which picks up from the chunk the server last confirmed. The server answers 409
|
||||
* with its own count when a chunk skips ahead, and acknowledges a chunk it already has.
|
||||
*/
|
||||
const RETRY_DELAYS = [1000, 2000, 4000, 8000, 16000]
|
||||
|
||||
document.addEventListener('alpine:init', () => {
|
||||
window.Alpine.data('shareUploader', ({ csrfToken, messages }) => ({
|
||||
secure: window.isSecureContext && Boolean(window.crypto?.subtle),
|
||||
|
||||
dragging: false,
|
||||
|
||||
busy: false,
|
||||
|
||||
/** Files waiting to be sent, in order: { id, file, target, nextIndex }. */
|
||||
queue: [],
|
||||
|
||||
/** Every file this page registered, by id: { state: 'queued'|'uploading'|'uploaded'|'failed', sent, size }. */
|
||||
uploads: {},
|
||||
|
||||
/** The files that failed, by id, kept for their Retry button. */
|
||||
failed: {},
|
||||
|
||||
/** The request on its way, so Cancel and Remove can abort it. */
|
||||
request: null,
|
||||
|
||||
get progress() {
|
||||
const unfinished = Object.values(this.uploads).filter((upload) => upload.state !== 'failed')
|
||||
const size = unfinished.reduce((total, upload) => total + upload.size, 0)
|
||||
|
||||
return size === 0 ? 0 : (unfinished.reduce((total, upload) => total + upload.sent, 0) / size) * 100
|
||||
},
|
||||
|
||||
choose(event) {
|
||||
this.add([...event.target.files].map((file) => ({ file, path: null })))
|
||||
|
||||
event.target.value = ''
|
||||
},
|
||||
|
||||
handleDrop(event) {
|
||||
this.dragging = false
|
||||
|
||||
if (! this.secure) {
|
||||
return
|
||||
}
|
||||
|
||||
const items = event.dataTransfer.items
|
||||
const files = []
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const entry = items[i].webkitGetAsEntry?.()
|
||||
|
||||
if (entry) {
|
||||
this.traverseEntry(entry, '', files)
|
||||
} else if (items[i].kind === 'file') {
|
||||
files.push({ file: items[i].getAsFile(), path: null })
|
||||
}
|
||||
}
|
||||
|
||||
// Directory entries are read asynchronously.
|
||||
setTimeout(() => this.add(files), 500)
|
||||
},
|
||||
|
||||
traverseEntry(entry, path, files) {
|
||||
if (entry.isFile) {
|
||||
entry.file((file) => files.push({ file, path: path ? `${path}/${file.name}` : null }))
|
||||
} else if (entry.isDirectory) {
|
||||
entry.createReader().readEntries((entries) => {
|
||||
entries.forEach((child) => this.traverseEntry(child, path ? `${path}/${entry.name}` : entry.name, files))
|
||||
})
|
||||
}
|
||||
},
|
||||
|
||||
async add(selection) {
|
||||
if (! this.secure || selection.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const targets = await this.$wire.registerFiles(selection.map(({ file, path }) => ({ name: file.name, size: file.size, path })))
|
||||
|
||||
;(targets ?? []).forEach((target, position) => {
|
||||
if (! target) {
|
||||
return
|
||||
}
|
||||
|
||||
this.uploads[target.id] = { state: 'queued', sent: 0, size: selection[position].file.size }
|
||||
this.queue.push({ id: target.id, file: selection[position].file, target, nextIndex: 0 })
|
||||
})
|
||||
|
||||
this.run()
|
||||
},
|
||||
|
||||
async run() {
|
||||
if (this.busy) {
|
||||
return
|
||||
}
|
||||
|
||||
this.busy = true
|
||||
|
||||
while (this.queue.length > 0) {
|
||||
const item = this.queue[0]
|
||||
const uploaded = await this.upload(item)
|
||||
|
||||
if (this.queue[0] === item) {
|
||||
this.queue.shift()
|
||||
}
|
||||
|
||||
if (uploaded) {
|
||||
await this.$wire.$refresh()
|
||||
}
|
||||
}
|
||||
|
||||
this.busy = false
|
||||
},
|
||||
|
||||
/**
|
||||
* Send one file's remaining chunks; true once the server has them all.
|
||||
*/
|
||||
async upload(item) {
|
||||
const { id, file, target } = item
|
||||
const upload = this.uploads[id]
|
||||
|
||||
if (! upload) {
|
||||
return false
|
||||
}
|
||||
|
||||
upload.state = 'uploading'
|
||||
|
||||
try {
|
||||
const key = await crypto.subtle.importKey('raw', bytesFromHex(target.key), 'AES-GCM', false, ['encrypt'])
|
||||
const noncePrefix = bytesFromHex(target.noncePrefix)
|
||||
|
||||
while (item.nextIndex < target.chunkCount) {
|
||||
const index = item.nextIndex
|
||||
const start = index * target.chunkSize
|
||||
const plaintext = await file.slice(start, start + target.chunkSize).arrayBuffer()
|
||||
const ciphertext = await crypto.subtle.encrypt(
|
||||
{ name: 'AES-GCM', iv: chunkNonce(noncePrefix, index, index === target.chunkCount - 1), tagLength: 128 },
|
||||
key,
|
||||
plaintext,
|
||||
)
|
||||
|
||||
item.nextIndex = await this.send(upload, `${target.url}/${index}`, ciphertext, start, plaintext.byteLength)
|
||||
upload.sent = Math.min(item.nextIndex * target.chunkSize, upload.size)
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.name === 'AbortError') {
|
||||
return false
|
||||
}
|
||||
|
||||
upload.state = 'failed'
|
||||
this.failed[id] = item
|
||||
|
||||
if (error?.status === 419) {
|
||||
window.materialToast(messages.sessionExpired, { type: 'error' })
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
upload.state = 'uploaded'
|
||||
|
||||
return true
|
||||
},
|
||||
|
||||
/**
|
||||
* PUT one encrypted chunk, retrying transient failures; resolves with the number of chunks
|
||||
* the server holds for the file.
|
||||
*/
|
||||
async send(upload, url, body, offset, plaintextLength) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const response = await this.put(url, body, (loaded) => {
|
||||
upload.sent = Math.min(offset + (loaded / body.byteLength) * plaintextLength, upload.size)
|
||||
})
|
||||
|
||||
if ((response.status === 200 || response.status === 409) && Number.isInteger(response.uploadedChunks)) {
|
||||
return response.uploadedChunks
|
||||
}
|
||||
|
||||
if (response.status === 404 || response.status === 419 || attempt === RETRY_DELAYS.length) {
|
||||
throw Object.assign(new Error('Upload failed'), { status: response.status })
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAYS[attempt]))
|
||||
}
|
||||
},
|
||||
|
||||
put(url, body, onProgress) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest()
|
||||
|
||||
xhr.open('PUT', url)
|
||||
xhr.setRequestHeader('Content-Type', 'application/octet-stream')
|
||||
xhr.setRequestHeader('Accept', 'application/json')
|
||||
xhr.setRequestHeader('X-CSRF-TOKEN', csrfToken)
|
||||
xhr.upload.onprogress = (event) => onProgress(event.loaded)
|
||||
xhr.onload = () => {
|
||||
this.request = null
|
||||
resolve({ status: xhr.status, uploadedChunks: parseUploadedChunks(xhr.responseText) })
|
||||
}
|
||||
xhr.onerror = () => {
|
||||
this.request = null
|
||||
resolve({ status: 0 })
|
||||
}
|
||||
xhr.onabort = () => {
|
||||
this.request = null
|
||||
reject(new DOMException('Upload cancelled', 'AbortError'))
|
||||
}
|
||||
|
||||
this.request = xhr
|
||||
// Chromium sends a Blob body about eight times faster than the same ArrayBuffer.
|
||||
xhr.send(new Blob([body]))
|
||||
})
|
||||
},
|
||||
|
||||
retry(id) {
|
||||
const item = this.failed[id]
|
||||
|
||||
if (! item) {
|
||||
return
|
||||
}
|
||||
|
||||
delete this.failed[id]
|
||||
this.uploads[id].state = 'queued'
|
||||
this.queue.push(item)
|
||||
this.run()
|
||||
},
|
||||
|
||||
remove(id) {
|
||||
this.forget([id])
|
||||
this.$wire.removeFiles([id])
|
||||
},
|
||||
|
||||
/**
|
||||
* Stop everything still to send and take those files out of the share.
|
||||
*/
|
||||
cancel() {
|
||||
const unfinished = Object.entries(this.uploads)
|
||||
.filter(([, upload]) => upload.state !== 'uploaded')
|
||||
.map(([id]) => Number(id))
|
||||
|
||||
this.forget(unfinished)
|
||||
this.$wire.removeFiles(unfinished)
|
||||
},
|
||||
|
||||
forget(ids) {
|
||||
const current = this.queue[0]
|
||||
|
||||
this.queue = this.queue.filter((item) => ! ids.includes(item.id))
|
||||
|
||||
ids.forEach((id) => {
|
||||
delete this.uploads[id]
|
||||
delete this.failed[id]
|
||||
})
|
||||
|
||||
if (current && ids.includes(current.id)) {
|
||||
this.request?.abort()
|
||||
}
|
||||
},
|
||||
|
||||
statusOf(id, uploaded) {
|
||||
const upload = this.uploads[id]
|
||||
|
||||
if (uploaded || upload?.state === 'uploaded') {
|
||||
return messages.uploaded
|
||||
}
|
||||
|
||||
if (upload?.state === 'uploading') {
|
||||
return `${Math.round((upload.sent / Math.max(upload.size, 1)) * 100)}%`
|
||||
}
|
||||
|
||||
return upload?.state === 'failed' ? messages.failed : messages.queued
|
||||
},
|
||||
|
||||
warnBeforeLeaving(event) {
|
||||
if (this.busy) {
|
||||
event.preventDefault()
|
||||
event.returnValue = ''
|
||||
}
|
||||
},
|
||||
}))
|
||||
})
|
||||
|
||||
function bytesFromHex(hex) {
|
||||
return Uint8Array.from(hex.match(/.{2}/g), (pair) => parseInt(pair, 16))
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk `index`'s 12-byte nonce: the file's prefix, the index as a big-endian uint32 and the
|
||||
* last-chunk flag.
|
||||
*/
|
||||
function chunkNonce(prefix, index, isLast) {
|
||||
const nonce = new Uint8Array(12)
|
||||
|
||||
nonce.set(prefix, 0)
|
||||
new DataView(nonce.buffer).setUint32(7, index)
|
||||
nonce[11] = isLast ? 1 : 0
|
||||
|
||||
return nonce
|
||||
}
|
||||
|
||||
function parseUploadedChunks(responseText) {
|
||||
try {
|
||||
return JSON.parse(responseText).uploaded_chunks
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user