Hey everyone. So the lack of responsiveness from the FileInput module has presented a pretty frustrating user experience for my labmates on a tool I’m developing, so I decided to take a crack at making an improved module. Basically everything I tried in Panel lead to the same result: Progress was updated cleanly up until the file transfer event occurs, then everything would hang for ~2 min while the actual upload took place. I really wanted to have it where I could dynamically update a progress bar as the transfer itself took place. @Marc made a suggestion on a related GitHub issue of taking the Bokeh file_input.ts code and modifying it, which was extremely helpful, so thank you for that! The solution I came up with doubles as a “streaming” file input, where each file is transferred as soon as possible, so you could also potentially launch background processing tasks as data rolls in (I plan on using this for a multiprocessing queue), but the data is also saved in lists and can be handled in the normal way (This can be pretty easily removed if that functionality is not necessary). I plan on also implementing Pako to pre-compress the data before the transfer to see if that speeds things up, but that felt a bit beyond the scope of this issue. Here’s the code, my apologies if there is anything non-conventional, I’m extremely new to Typescript:
Demonstration
_uploadtest.py
#Bokeh version: 2.4.3
#Panel version: 0.14.1
import panel as pn
from bokeh.core.properties import List, String, Bool, Int
from bokeh.layouts import column
from bokeh.models import LayoutDOM
pn.extension()
class CustomFileInputStream(LayoutDOM):
__implementation__ = "assets/ts/custom_file_inputstream.ts"
filename = String(default = "")
value = String(default = "")
mime_type = String(default = "")
accept = String(default = "")
multiple = Bool(default=False)
is_loading = Bool(default=False)
num_files = Int(default=0)
load_progress = Int(default=0)
filenames = List(String, default=[])
values = List(String, default=[])
mime_types = List(String, default=[])
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.on_change("is_loading", self._reset_lists)
self.on_change("filename", self._filename_transfered)
self.on_change("value", self._value_transfered)
self.on_change("mime_type", self._mime_type_transfered)
def _reset_lists(self, attr, old, new):
if new:
self.filenames = []
self.values = []
self.mime_types = []
def _filename_transfered(self, attr, old, new):
self.filenames.append(new)
def _value_transfered(self, attr, old, new):
self.values.append(new)
def _mime_type_transfered(self, attr, old, new):
self.mime_types.append(new)
custom_file = CustomFileInputStream(multiple = True)
def _file_loading_callback(attr, old, new):
if new:
test_text.value = f"Loading {custom_file.num_files} files...\n"
else:
test_text.value += "Loading complete!"
custom_file.on_change("is_loading", _file_loading_callback)
def _file_loading_progress(attr, old, new):
progress_bar.value = custom_file.load_progress
custom_file.on_change("load_progress", _file_loading_progress)
def _file_contents_changed(attr, old, new):
test_text.value += f"{new}\n"
custom_file.on_change("filename", _file_contents_changed)
layout = column(custom_file)
bokeh_pane = pn.pane.Bokeh(layout)
progress_bar = pn.indicators.Progress(name="ProgressBar", value=1, max=100, active=False)
test_text = pn.widgets.TextAreaInput(width=500, height=300)
check_button = pn.widgets.Button(name="Check")
def check_callback(event):
test_text.value = f"Loaded {len(custom_file.filenames)} files\n"
for f in custom_file.filenames:
test_text.value += f"{f}\n"
check_button.on_click(check_callback)
ui = pn.Column(
test_text,
progress_bar,
bokeh_pane,
check_button
)
ui.servable()
custom_file_inputstream.ts
import { input } from "core/dom"
import * as p from "core/properties"
import {Widget, WidgetView} from "models/widgets/widget"
export class CustomFileInputStreamView extends WidgetView {
override model: CustomFileInputStream
protected dialog_el: HTMLInputElement
override connect_signals(): void {
super.connect_signals()
this.connect(this.model.change, () => this.render())
}
override render(): void {
const {multiple, accept, disabled, width} = this.model
if (this.dialog_el == null) {
this.dialog_el = input({type: "file", multiple: multiple})
this.dialog_el.onchange = () => {
const {files} = this.dialog_el
if (files != null) {
this.model.setv({num_files: files.length, is_loading: true})
this.load_files(files)
}
}
this.el.appendChild(this.dialog_el)
}
if (accept != null && accept != "") {
this.dialog_el.accept = accept
}
this.dialog_el.style.width = `${width}px`
this.dialog_el.disabled = disabled
}
async load_files(files: FileList): Promise<void> {
var progress: number = 0
for (const file of files) {
const data_url = await this._read_file(file)
const [, mime_type="",, value=""] = data_url.split(/[:;,]/, 4)
progress += 1
this.model.setv({
value: value,
filename: file.name,
mime_type: mime_type,
load_progress: Math.round(100 * (progress / this.model.num_files))
})
}
this.model.setv({is_loading: false})
}
protected _read_file(file: File): Promise<string> {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => {
const {result} = reader
if (result != null) {
resolve(result as string)
} else {
reject(reader.error ?? new Error(`unable to read '${file.name}'`))
}
}
reader.readAsDataURL(file)
})
}
}
export namespace CustomFileInputStream {
export type Attrs = p.AttrsOf<Props>
export type Props = Widget.Props & {
value: p.Property<string>
mime_type: p.Property<string>
filename: p.Property<string>
accept: p.Property<string>
multiple: p.Property<boolean>
is_loading: p.Property<boolean>
num_files: p.Property<number>
load_progress: p.Property<number>
values: p.Property<string[]>
mime_types: p.Property<string[]>
filenames: p.Property<string[]>
}
}
export interface CustomFileInputStream extends CustomFileInputStream.Attrs {}
export class CustomFileInputStream extends Widget {
override properties: CustomFileInputStream.Props
override __view_type__: CustomFileInputStreamView
constructor(attrs?: Partial<CustomFileInputStream.Attrs>) {
super(attrs)
}
static {
this.prototype.default_view = CustomFileInputStreamView
this.define<CustomFileInputStream.Props>(({Number, Boolean, String, Array}) => ({
value: [ String, "" ],
mime_type: [ String, "" ],
filename: [ String, "" ],
accept: [ String, "" ],
multiple: [ Boolean, false ],
is_loading: [ Boolean, false],
num_files: [ Number, 0],
load_progress: [ Number, 0],
values: [ Array(String) ],
mime_types: [ Array(String) ],
filenames: [ Array(String) ],
}))
}
}