A 6 GB upload kept a customer waiting long after its progress bar
reached 100%. The server wrote every upload three times: PHP's
temporary file, Livewire's copy of it ("Processing files...") and the
encrypted file ("Create Share Link"), each a full rewrite of a slow
disk. The unencrypted copy also stayed behind in livewire-tmp.
Now the uploader's browser encrypts each file in 16 MB chunks with
WebCrypto and PUTs them one at a time; the server checks each chunk in
memory and writes it once, already encrypted. Creating the share only
wraps its key and saves the options. A 200 MB upload through the
Docker image took 2.8 s, and its download matched byte for byte.
- SEALCHK2: a 19-byte header (chunk size, 7-byte nonce prefix), then
ciphertext and tag per chunk. Each nonce holds the chunk index and a
last-chunk flag (the STREAM construction), so cut or reordered files
fail to decrypt. SEALCHK1 and the single-block format still read.
- Envelope encryption: one random key per share. With a password it is
wrapped with Argon2id (sodium, libsodium's interactive limits) in
shares.wrapped_key, which names its parameters. Password shares from
before keep their PBKDF2-derived key.
- The upload page registers each selection with FileUploader into a
pending share of its own, lists the files with their progress, retries
a failed chunk after 1-16 s, then offers Retry; Remove and Cancel
abort. UploadChunkController only accepts chunks from the session that
started the share: a repeat is acknowledged, a skip gets 409 with the
count stored. Chunks go out as Blobs, which Chromium sends about eight
times faster than ArrayBuffers.
- Uploads need a secure context: over plain HTTP the page says HTTPS is
needed and takes no files. The Docker image gains AUTO_HTTPS, which
serves Let's Encrypt on 443 for SERVER_NAME and redirects 80; without
it the container stays on HTTP 80 behind a proxy. docker/Caddyfile was
never loaded and is gone; docker/healthcheck.sh covers both modes.
- "Download all" streams the ZIP with maennchen/zipstream-php (STORE,
ZIP64) instead of decrypting whole files into memory and writing the
archive unencrypted to /tmp.
- Pending shares count towards the quota, stay out of the admin
dashboard and 404 everywhere else. shares:cleanup deletes uploads idle
for 4 hours and Livewire temporary files older than that.
- PHP's upload limits no longer cap the admin's max file size and
default to 64M; LIVEWIRE_MAX_UPLOAD_TIME is gone and
UPLOAD_CHUNK_SIZE_MB is new.
- Tests cover the format, key wrapping, registration limits, the chunk
endpoint's answers, completing a share, the streamed ZIP, cleanup,
and in Chromium a real chunked upload and the HTTPS warning; the
selected-files overflow test runs again. README, website, CHANGELOG
and .ai/rules follow.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
318 lines
11 KiB
JavaScript
318 lines
11 KiB
JavaScript
/**
|
|
* `shareUploader`: the upload page's queue. Files are registered with the Livewire component in one
|
|
* batch per selection, then sent one at a time: each chunk is sliced from the file, encrypted here
|
|
* with AES-256-GCM and PUT on its own, so the server writes it once, already encrypted.
|
|
*
|
|
* The encrypted format is App\Services\FileEncryptionService's SEALCHK2: chunk i's nonce is the
|
|
* file's 7-byte prefix, i as a big-endian uint32 and a byte that is 1 on the last chunk; WebCrypto
|
|
* appends the 16-byte tag to the ciphertext, which is how the server stores it.
|
|
*
|
|
* A failed request is retried after 1, 2, 4, 8 and 16 seconds; after that the file waits for its
|
|
* Retry button, which picks up from the chunk the server last confirmed. The server answers 409
|
|
* with its own count when a chunk skips ahead, and acknowledges a chunk it already has.
|
|
*/
|
|
const RETRY_DELAYS = [1000, 2000, 4000, 8000, 16000]
|
|
|
|
document.addEventListener('alpine:init', () => {
|
|
window.Alpine.data('shareUploader', ({ csrfToken, messages }) => ({
|
|
secure: window.isSecureContext && Boolean(window.crypto?.subtle),
|
|
|
|
dragging: false,
|
|
|
|
busy: false,
|
|
|
|
/** Files waiting to be sent, in order: { id, file, target, nextIndex }. */
|
|
queue: [],
|
|
|
|
/** Every file this page registered, by id: { state: 'queued'|'uploading'|'uploaded'|'failed', sent, size }. */
|
|
uploads: {},
|
|
|
|
/** The files that failed, by id, kept for their Retry button. */
|
|
failed: {},
|
|
|
|
/** The request on its way, so Cancel and Remove can abort it. */
|
|
request: null,
|
|
|
|
get progress() {
|
|
const unfinished = Object.values(this.uploads).filter((upload) => upload.state !== 'failed')
|
|
const size = unfinished.reduce((total, upload) => total + upload.size, 0)
|
|
|
|
return size === 0 ? 0 : (unfinished.reduce((total, upload) => total + upload.sent, 0) / size) * 100
|
|
},
|
|
|
|
choose(event) {
|
|
this.add([...event.target.files].map((file) => ({ file, path: null })))
|
|
|
|
event.target.value = ''
|
|
},
|
|
|
|
handleDrop(event) {
|
|
this.dragging = false
|
|
|
|
if (! this.secure) {
|
|
return
|
|
}
|
|
|
|
const items = event.dataTransfer.items
|
|
const files = []
|
|
|
|
for (let i = 0; i < items.length; i++) {
|
|
const entry = items[i].webkitGetAsEntry?.()
|
|
|
|
if (entry) {
|
|
this.traverseEntry(entry, '', files)
|
|
} else if (items[i].kind === 'file') {
|
|
files.push({ file: items[i].getAsFile(), path: null })
|
|
}
|
|
}
|
|
|
|
// Directory entries are read asynchronously.
|
|
setTimeout(() => this.add(files), 500)
|
|
},
|
|
|
|
traverseEntry(entry, path, files) {
|
|
if (entry.isFile) {
|
|
entry.file((file) => files.push({ file, path: path ? `${path}/${file.name}` : null }))
|
|
} else if (entry.isDirectory) {
|
|
entry.createReader().readEntries((entries) => {
|
|
entries.forEach((child) => this.traverseEntry(child, path ? `${path}/${entry.name}` : entry.name, files))
|
|
})
|
|
}
|
|
},
|
|
|
|
async add(selection) {
|
|
if (! this.secure || selection.length === 0) {
|
|
return
|
|
}
|
|
|
|
const targets = await this.$wire.registerFiles(selection.map(({ file, path }) => ({ name: file.name, size: file.size, path })))
|
|
|
|
;(targets ?? []).forEach((target, position) => {
|
|
if (! target) {
|
|
return
|
|
}
|
|
|
|
this.uploads[target.id] = { state: 'queued', sent: 0, size: selection[position].file.size }
|
|
this.queue.push({ id: target.id, file: selection[position].file, target, nextIndex: 0 })
|
|
})
|
|
|
|
this.run()
|
|
},
|
|
|
|
async run() {
|
|
if (this.busy) {
|
|
return
|
|
}
|
|
|
|
this.busy = true
|
|
|
|
while (this.queue.length > 0) {
|
|
const item = this.queue[0]
|
|
const uploaded = await this.upload(item)
|
|
|
|
if (this.queue[0] === item) {
|
|
this.queue.shift()
|
|
}
|
|
|
|
if (uploaded) {
|
|
await this.$wire.$refresh()
|
|
}
|
|
}
|
|
|
|
this.busy = false
|
|
},
|
|
|
|
/**
|
|
* Send one file's remaining chunks; true once the server has them all.
|
|
*/
|
|
async upload(item) {
|
|
const { id, file, target } = item
|
|
const upload = this.uploads[id]
|
|
|
|
if (! upload) {
|
|
return false
|
|
}
|
|
|
|
upload.state = 'uploading'
|
|
|
|
try {
|
|
const key = await crypto.subtle.importKey('raw', bytesFromHex(target.key), 'AES-GCM', false, ['encrypt'])
|
|
const noncePrefix = bytesFromHex(target.noncePrefix)
|
|
|
|
while (item.nextIndex < target.chunkCount) {
|
|
const index = item.nextIndex
|
|
const start = index * target.chunkSize
|
|
const plaintext = await file.slice(start, start + target.chunkSize).arrayBuffer()
|
|
const ciphertext = await crypto.subtle.encrypt(
|
|
{ name: 'AES-GCM', iv: chunkNonce(noncePrefix, index, index === target.chunkCount - 1), tagLength: 128 },
|
|
key,
|
|
plaintext,
|
|
)
|
|
|
|
item.nextIndex = await this.send(upload, `${target.url}/${index}`, ciphertext, start, plaintext.byteLength)
|
|
upload.sent = Math.min(item.nextIndex * target.chunkSize, upload.size)
|
|
}
|
|
} catch (error) {
|
|
if (error?.name === 'AbortError') {
|
|
return false
|
|
}
|
|
|
|
upload.state = 'failed'
|
|
this.failed[id] = item
|
|
|
|
if (error?.status === 419) {
|
|
window.materialToast(messages.sessionExpired, { type: 'error' })
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
upload.state = 'uploaded'
|
|
|
|
return true
|
|
},
|
|
|
|
/**
|
|
* PUT one encrypted chunk, retrying transient failures; resolves with the number of chunks
|
|
* the server holds for the file.
|
|
*/
|
|
async send(upload, url, body, offset, plaintextLength) {
|
|
for (let attempt = 0; ; attempt++) {
|
|
const response = await this.put(url, body, (loaded) => {
|
|
upload.sent = Math.min(offset + (loaded / body.byteLength) * plaintextLength, upload.size)
|
|
})
|
|
|
|
if ((response.status === 200 || response.status === 409) && Number.isInteger(response.uploadedChunks)) {
|
|
return response.uploadedChunks
|
|
}
|
|
|
|
if (response.status === 404 || response.status === 419 || attempt === RETRY_DELAYS.length) {
|
|
throw Object.assign(new Error('Upload failed'), { status: response.status })
|
|
}
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAYS[attempt]))
|
|
}
|
|
},
|
|
|
|
put(url, body, onProgress) {
|
|
return new Promise((resolve, reject) => {
|
|
const xhr = new XMLHttpRequest()
|
|
|
|
xhr.open('PUT', url)
|
|
xhr.setRequestHeader('Content-Type', 'application/octet-stream')
|
|
xhr.setRequestHeader('Accept', 'application/json')
|
|
xhr.setRequestHeader('X-CSRF-TOKEN', csrfToken)
|
|
xhr.upload.onprogress = (event) => onProgress(event.loaded)
|
|
xhr.onload = () => {
|
|
this.request = null
|
|
resolve({ status: xhr.status, uploadedChunks: parseUploadedChunks(xhr.responseText) })
|
|
}
|
|
xhr.onerror = () => {
|
|
this.request = null
|
|
resolve({ status: 0 })
|
|
}
|
|
xhr.onabort = () => {
|
|
this.request = null
|
|
reject(new DOMException('Upload cancelled', 'AbortError'))
|
|
}
|
|
|
|
this.request = xhr
|
|
// Chromium sends a Blob body about eight times faster than the same ArrayBuffer.
|
|
xhr.send(new Blob([body]))
|
|
})
|
|
},
|
|
|
|
retry(id) {
|
|
const item = this.failed[id]
|
|
|
|
if (! item) {
|
|
return
|
|
}
|
|
|
|
delete this.failed[id]
|
|
this.uploads[id].state = 'queued'
|
|
this.queue.push(item)
|
|
this.run()
|
|
},
|
|
|
|
remove(id) {
|
|
this.forget([id])
|
|
this.$wire.removeFiles([id])
|
|
},
|
|
|
|
/**
|
|
* Stop everything still to send and take those files out of the share.
|
|
*/
|
|
cancel() {
|
|
const unfinished = Object.entries(this.uploads)
|
|
.filter(([, upload]) => upload.state !== 'uploaded')
|
|
.map(([id]) => Number(id))
|
|
|
|
this.forget(unfinished)
|
|
this.$wire.removeFiles(unfinished)
|
|
},
|
|
|
|
forget(ids) {
|
|
const current = this.queue[0]
|
|
|
|
this.queue = this.queue.filter((item) => ! ids.includes(item.id))
|
|
|
|
ids.forEach((id) => {
|
|
delete this.uploads[id]
|
|
delete this.failed[id]
|
|
})
|
|
|
|
if (current && ids.includes(current.id)) {
|
|
this.request?.abort()
|
|
}
|
|
},
|
|
|
|
statusOf(id, uploaded) {
|
|
const upload = this.uploads[id]
|
|
|
|
if (uploaded || upload?.state === 'uploaded') {
|
|
return messages.uploaded
|
|
}
|
|
|
|
if (upload?.state === 'uploading') {
|
|
return `${Math.round((upload.sent / Math.max(upload.size, 1)) * 100)}%`
|
|
}
|
|
|
|
return upload?.state === 'failed' ? messages.failed : messages.queued
|
|
},
|
|
|
|
warnBeforeLeaving(event) {
|
|
if (this.busy) {
|
|
event.preventDefault()
|
|
event.returnValue = ''
|
|
}
|
|
},
|
|
}))
|
|
})
|
|
|
|
function bytesFromHex(hex) {
|
|
return Uint8Array.from(hex.match(/.{2}/g), (pair) => parseInt(pair, 16))
|
|
}
|
|
|
|
/**
|
|
* Chunk `index`'s 12-byte nonce: the file's prefix, the index as a big-endian uint32 and the
|
|
* last-chunk flag.
|
|
*/
|
|
function chunkNonce(prefix, index, isLast) {
|
|
const nonce = new Uint8Array(12)
|
|
|
|
nonce.set(prefix, 0)
|
|
new DataView(nonce.buffer).setUint32(7, index)
|
|
nonce[11] = isLast ? 1 : 0
|
|
|
|
return nonce
|
|
}
|
|
|
|
function parseUploadedChunks(responseText) {
|
|
try {
|
|
return JSON.parse(responseText).uploaded_chunks
|
|
} catch {
|
|
return undefined
|
|
}
|
|
}
|