Download large shares file by file, resumably
A 29 GB share with 254 files failed as one ZIP: the single request ran into the 300-second request limit and, behind a proxy we cannot configure, could be cut at any time with no way to resume. Single-file downloads now answer HTTP Range requests (206, ETag and If-Range, 416 past the end): a SEALCHK2 file is decrypted from the chunk holding the first byte asked for. On that, the download page's new "Download all to a folder" (File System Access API, Chrome and Edge on a computer) saves every file into a folder the recipient picks, one request per file, in its folder path; a broken connection is retried from the byte on disk, complete files are skipped, and failed files get a Retry button. Other browsers keep the ZIP and the per-file buttons. The ZIP is sized up front with ZipStream's strict simulation and sent with Content-Length, so the browser shows progress and notices a cut. Both download routes lift the time limit for their streams. The recipient's download window now restarts with each download, so a long file-by-file download never outlasts it. The download page shows the file count and total size and marks the files this browser has downloaded, and the upload page gains "Browse Folder", which keeps each file's path as a dropped folder does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andreas Reinhold / reini
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
f9a7839ad3
commit
b594fb2300
@@ -2,3 +2,4 @@
|
||||
import '../../vendor/nonameweb/livewire-material/resources/js/material.js'
|
||||
import './share-created.js'
|
||||
import './share-uploader.js'
|
||||
import './share-downloader.js'
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
/**
|
||||
* `shareDownloader`: the download page's file list. It remembers, in this browser, which files the
|
||||
* recipient has downloaded, and where the browser can write to a folder (the File System Access
|
||||
* API: Chrome and Edge on a computer) it downloads every file into the folder the recipient picks,
|
||||
* one at a time, in its folder path.
|
||||
*
|
||||
* One request per file keeps every request short enough for a proxy in front of the server. A
|
||||
* request that breaks off is retried after 1, 2, 4, 8 and 16 seconds from the byte the file on
|
||||
* disk has reached (an HTTP Range request); a file already there at its full size is skipped, so
|
||||
* running it again after a stop picks up where it stopped. A file that still fails is listed for
|
||||
* the Retry button; a share that is gone (404) stops the run.
|
||||
*/
|
||||
const RETRY_DELAYS = [1000, 2000, 4000, 8000, 16000]
|
||||
|
||||
const UNITS = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
|
||||
function formatSize(bytes) {
|
||||
let unit = 0
|
||||
|
||||
while (bytes >= 1024 && unit < UNITS.length - 1) {
|
||||
bytes /= 1024
|
||||
unit++
|
||||
}
|
||||
|
||||
return `${bytes.toFixed(unit === 0 ? 0 : 1)} ${UNITS[unit]}`
|
||||
}
|
||||
|
||||
function readStored(key) {
|
||||
try {
|
||||
return JSON.parse(window.localStorage.getItem(key)) ?? []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/** The file's handle under the picked folder, creating its subfolders; never a path out of it. */
|
||||
async function fileHandle(directory, path) {
|
||||
const segments = path.split('/').filter((segment) => segment !== '' && segment !== '.' && segment !== '..')
|
||||
const name = segments.pop()
|
||||
|
||||
for (const segment of segments) {
|
||||
directory = await directory.getDirectoryHandle(segment, { create: true })
|
||||
}
|
||||
|
||||
return directory.getFileHandle(name, { create: true })
|
||||
}
|
||||
|
||||
document.addEventListener('alpine:init', () => {
|
||||
window.Alpine.data('shareDownloader', ({ token, files, messages }) => {
|
||||
const storageKey = `sealshare.downloaded.${token}`
|
||||
const total = files.reduce((sum, file) => sum + file.size, 0)
|
||||
|
||||
// Kept out of Alpine's reactive data: a proxied native handle throws on its own methods.
|
||||
let directory = null
|
||||
let controller = null
|
||||
|
||||
return {
|
||||
supported: window.isSecureContext && 'showDirectoryPicker' in window,
|
||||
|
||||
/** The ids of the files this browser has downloaded. */
|
||||
done: readStored(storageKey),
|
||||
|
||||
running: false,
|
||||
|
||||
/** Bytes of the run's files on disk, for the progress bar. */
|
||||
received: 0,
|
||||
|
||||
/** How many of the run's files are in the folder. */
|
||||
saved: 0,
|
||||
|
||||
/** The file on its way. */
|
||||
current: null,
|
||||
|
||||
/** The files that failed, kept for the Retry button. */
|
||||
failed: [],
|
||||
|
||||
/** Why the run ended early, if it did. */
|
||||
error: null,
|
||||
|
||||
finished: false,
|
||||
|
||||
get progress() {
|
||||
return total === 0 ? 100 : (this.received / total) * 100
|
||||
},
|
||||
|
||||
get status() {
|
||||
return messages.progress
|
||||
.replace(':saved', this.saved)
|
||||
.replace(':count', files.length)
|
||||
.replace(':received', formatSize(this.received))
|
||||
.replace(':total', formatSize(total))
|
||||
},
|
||||
|
||||
isDone(id) {
|
||||
return this.done.includes(id)
|
||||
},
|
||||
|
||||
markDone(id) {
|
||||
if (this.done.includes(id)) {
|
||||
return
|
||||
}
|
||||
|
||||
this.done.push(id)
|
||||
|
||||
try {
|
||||
window.localStorage.setItem(storageKey, JSON.stringify(this.done))
|
||||
} catch {
|
||||
// Private mode or storage blocked: the marks last as long as the page.
|
||||
}
|
||||
},
|
||||
|
||||
async downloadAll() {
|
||||
try {
|
||||
directory = await window.showDirectoryPicker({ id: 'sealshare', mode: 'readwrite' })
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
this.run(files)
|
||||
},
|
||||
|
||||
retry() {
|
||||
this.run(this.failed)
|
||||
},
|
||||
|
||||
cancel() {
|
||||
controller?.abort()
|
||||
},
|
||||
|
||||
async run(list) {
|
||||
controller = new AbortController()
|
||||
this.running = true
|
||||
this.finished = false
|
||||
this.error = null
|
||||
this.failed = []
|
||||
this.saved = files.length - list.length
|
||||
this.received = total - list.reduce((sum, file) => sum + file.size, 0)
|
||||
this.downloaded = true
|
||||
|
||||
for (const file of list) {
|
||||
const before = this.received
|
||||
this.current = file.path
|
||||
|
||||
try {
|
||||
await this.save(file, controller.signal)
|
||||
this.saved++
|
||||
this.markDone(file.id)
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) {
|
||||
break
|
||||
}
|
||||
|
||||
if (error.status === 404) {
|
||||
this.error = messages.gone
|
||||
break
|
||||
}
|
||||
|
||||
this.failed.push(file)
|
||||
this.received = before + file.size
|
||||
}
|
||||
}
|
||||
|
||||
this.running = false
|
||||
this.current = null
|
||||
this.finished = ! controller.signal.aborted
|
||||
},
|
||||
|
||||
async save(file, signal) {
|
||||
const handle = await fileHandle(directory, file.path)
|
||||
const before = this.received
|
||||
// ponytail: a file is matched by name and size only, so a different file of the same name and size already in the picked folder is kept; compare a checksum if that ever bites.
|
||||
let written = (await handle.getFile()).size
|
||||
|
||||
if (written > file.size) {
|
||||
written = 0
|
||||
}
|
||||
|
||||
for (let attempt = 0; written < file.size; attempt++) {
|
||||
try {
|
||||
written = await this.fetchInto(handle, file, written, signal, (bytes) => (this.received = before + bytes))
|
||||
} catch (error) {
|
||||
if (signal.aborted || error.status === 404 || attempt >= RETRY_DELAYS.length) {
|
||||
throw error
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAYS[attempt]))
|
||||
written = (await handle.getFile()).size
|
||||
}
|
||||
}
|
||||
|
||||
this.received = before + file.size
|
||||
},
|
||||
|
||||
/** Write the file from byte `offset` on; the bytes that arrived are kept even when the connection breaks. */
|
||||
async fetchInto(handle, file, offset, signal, onProgress) {
|
||||
const response = await fetch(file.url, {
|
||||
headers: offset > 0 ? { Range: `bytes=${offset}-` } : {},
|
||||
signal,
|
||||
})
|
||||
|
||||
if (! response.ok) {
|
||||
throw Object.assign(new Error(`HTTP ${response.status}`), { status: response.status })
|
||||
}
|
||||
|
||||
if (response.status !== 206) {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
const writable = await handle.createWritable({ keepExistingData: offset > 0 })
|
||||
const reader = response.body.getReader()
|
||||
|
||||
try {
|
||||
await writable.truncate(offset)
|
||||
await writable.seek(offset)
|
||||
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
|
||||
if (done) {
|
||||
break
|
||||
}
|
||||
|
||||
await writable.write(value)
|
||||
offset += value.length
|
||||
onProgress(offset)
|
||||
}
|
||||
} finally {
|
||||
// Closing commits what was written, so a retry continues from there.
|
||||
await writable.close().catch(() => {})
|
||||
}
|
||||
|
||||
if (offset < file.size) {
|
||||
throw new Error('The download ended early')
|
||||
}
|
||||
|
||||
return offset
|
||||
},
|
||||
|
||||
warnBeforeLeaving(event) {
|
||||
if (this.running) {
|
||||
event.preventDefault()
|
||||
event.returnValue = ''
|
||||
}
|
||||
},
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -53,7 +53,8 @@ document.addEventListener('alpine:init', () => {
|
||||
},
|
||||
|
||||
choose(event) {
|
||||
this.add([...event.target.files].map((file) => ({ file, path: null })))
|
||||
// A chosen folder's files carry their path from it, as a dropped folder's do.
|
||||
this.add([...event.target.files].map((file) => ({ file, path: file.webkitRelativePath || null })))
|
||||
|
||||
event.target.value = ''
|
||||
},
|
||||
|
||||
@@ -45,9 +45,14 @@
|
||||
<p class="md-type-body-md md-ink-variant md-text-center">{{ __('or click to browse') }}</p>
|
||||
</x-stack>
|
||||
|
||||
{{-- The button is the tab stop and opens the browser's own picker; the input only carries the selection. --}}
|
||||
<x-button :label="__('Browse Files')" icon="folder_open" variant="outlined" x-on:click="$refs.picker.click()" x-bind:disabled="! secure" />
|
||||
{{-- Each button is a tab stop and opens the browser's own picker, for files or for a folder with
|
||||
everything in it; the inputs only carry the selection. --}}
|
||||
<x-row gap="space100" justify="center" wrap>
|
||||
<x-button :label="__('Browse Files')" icon="upload_file" variant="outlined" x-on:click="$refs.picker.click()" x-bind:disabled="! secure" />
|
||||
<x-button :label="__('Browse Folder')" icon="folder_open" variant="outlined" x-on:click="$refs.folderPicker.click()" x-bind:disabled="! secure" data-test="browse-folder" />
|
||||
</x-row>
|
||||
<input type="file" multiple hidden x-ref="picker" x-on:change="choose($event)" x-bind:disabled="! secure" data-test="file-input" />
|
||||
<input type="file" webkitdirectory hidden x-ref="folderPicker" x-on:change="choose($event)" x-bind:disabled="! secure" data-test="folder-input" />
|
||||
</x-stack>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -73,8 +73,27 @@
|
||||
@endif
|
||||
|
||||
@if ($files->isNotEmpty())
|
||||
<x-card :title="__('Shared Files')" heading="h2" variant="outlined">
|
||||
<x-stack gap="space200">
|
||||
{{-- Which files this browser has downloaded is remembered per share (resources/js/share-downloader.js),
|
||||
and where the browser can write to a folder, "Download all to a folder" fetches them one at a
|
||||
time, each resumable, which a single ZIP of a large share is not. --}}
|
||||
<x-card
|
||||
:title="__('Shared Files')"
|
||||
:subtitle="trans_choice('{1} :count file, :size|[2,*] :count files, :size', $files->count(), ['size' => Number::fileSize($files->sum('file_size'))])"
|
||||
heading="h2"
|
||||
variant="outlined"
|
||||
>
|
||||
<x-stack
|
||||
gap="space200"
|
||||
x-data="shareDownloader({
|
||||
token: {{ \Illuminate\Support\Js::from($share->token) }},
|
||||
files: {{ \Illuminate\Support\Js::from($downloads) }},
|
||||
messages: {{ \Illuminate\Support\Js::from([
|
||||
'progress' => __(':saved of :count files, :received of :total'),
|
||||
'gone' => __('This share is no longer available.'),
|
||||
]) }},
|
||||
})"
|
||||
x-on:beforeunload.window="warnBeforeLeaving($event)"
|
||||
>
|
||||
<x-list :label="__('Shared Files')">
|
||||
@foreach ($files as $file)
|
||||
<x-list-item
|
||||
@@ -84,17 +103,47 @@
|
||||
>
|
||||
<x-slot:description><span class="md-tabular">{{ Number::fileSize($file->file_size) }}</span></x-slot:description>
|
||||
<x-slot:end>
|
||||
<x-button icon="download" :link="route('share.download.file', [$share, $file])" no-wire-navigate :aria-label="__('Download :name', ['name' => $file->original_name])" x-on:click="downloaded = true" />
|
||||
<x-row gap="space100">
|
||||
<span x-show="isDone({{ $file->id }})" x-cloak data-test="file-downloaded">
|
||||
<x-icon name="download_done" class="md-ink-success" :label="__('Downloaded')" />
|
||||
</span>
|
||||
<x-button icon="download" :link="route('share.download.file', [$share, $file])" no-wire-navigate :aria-label="__('Download :name', ['name' => $file->original_name])" x-on:click="downloaded = true; markDone({{ $file->id }})" />
|
||||
</x-row>
|
||||
</x-slot:end>
|
||||
</x-list-item>
|
||||
@endforeach
|
||||
</x-list>
|
||||
|
||||
<x-row justify="end">
|
||||
<div x-show="running || finished" x-cloak data-test="folder-download">
|
||||
<x-stack gap="space100">
|
||||
<x-row justify="between">
|
||||
<span class="md-type-label-lg md-tabular" x-text="status"></span>
|
||||
<x-button :label="__('Cancel')" size="xs" x-show="running" x-on:click="cancel()" />
|
||||
</x-row>
|
||||
<x-progress bind="progress" :label="__('Downloading')" />
|
||||
<p class="md-type-body-sm md-ink-variant" x-show="running" x-text="current"></p>
|
||||
<p class="md-type-body-sm md-ink-success" x-show="finished && failed.length === 0">{{ __('All files are in the folder.') }}</p>
|
||||
</x-stack>
|
||||
</div>
|
||||
|
||||
<div x-show="error" x-cloak>
|
||||
<x-alert color="error" assertive><span x-text="error"></span></x-alert>
|
||||
</div>
|
||||
|
||||
<div x-show="! running && failed.length > 0" x-cloak>
|
||||
<x-alert color="error" assertive>
|
||||
{{ __('Files that could not be downloaded:') }} <span class="md-tabular" x-text="failed.length"></span>
|
||||
<x-slot:actions><x-button :label="__('Retry')" x-on:click="retry()" /></x-slot:actions>
|
||||
</x-alert>
|
||||
</div>
|
||||
|
||||
<x-row justify="end" gap="space100" wrap x-show="! running">
|
||||
@if ($files->count() > 1)
|
||||
<x-button :label="__('Download All as ZIP')" icon="download" variant="filled" :link="route('share.download.all', $share)" no-wire-navigate x-on:click="downloaded = true" />
|
||||
<x-button :label="__('Download All as ZIP')" icon="folder_zip" variant="tonal" :link="route('share.download.all', $share)" no-wire-navigate x-show="supported" x-cloak x-on:click="downloaded = true" />
|
||||
<x-button :label="__('Download all to a folder')" icon="download" variant="filled" x-show="supported" x-cloak x-on:click="downloadAll()" data-test="download-to-folder" />
|
||||
<x-button :label="__('Download All as ZIP')" icon="download" variant="filled" :link="route('share.download.all', $share)" no-wire-navigate x-show="! supported" x-on:click="downloaded = true" />
|
||||
@else
|
||||
<x-button :label="__('Download')" icon="download" variant="filled" :link="route('share.download.file', [$share, $files->first()])" no-wire-navigate x-on:click="downloaded = true" />
|
||||
<x-button :label="__('Download')" icon="download" variant="filled" :link="route('share.download.file', [$share, $files->first()])" no-wire-navigate x-on:click="downloaded = true; markDone({{ $files->first()->id }})" />
|
||||
@endif
|
||||
</x-row>
|
||||
</x-stack>
|
||||
|
||||
Reference in New Issue
Block a user