Mixing streams and pipes does not work reliably

The following example is an attempt at mixing streams and pipes. I want an image that updates continuously from a pipe but also to be able to customize certain parameters (colormap, color_levels) of the image from the browser presumably using a stream.

   import param
   import panel as pn
   import holoviews as hv
   import numpy as np
   import time


   pn.extension()
   hv.extension('bokeh')

   options = {'axiswise': True, 'framewise': True, 'shared_axes': False, 'show_grid': True, 'tools': ['hover'],        
              'responsive': True, 'min_height': 200, 'min_width': 200}
   hv.opts.defaults(hv.opts.Image(**options),
                    hv.opts.Histogram(**options))


   class Image(param.Parameterized):                                                                                   
       colormap = param.ObjectSelector(default='viridis', objects=['viridis', 'plasma', 'magma'])                      
       color_levels = param.Integer(default=255, bounds=(1, 255))                                                      
                                                                                                                       
       @param.depends('colormap', 'color_levels')                                                                      
       def plot(self, data):                                                                                           
           return hv.Image(data).opts(colorbar=True, width=400, cmap=self.colormap, color_levels=self.color_levels)    


   img = Image()
   pipe = hv.streams.Pipe(data=[(np.random.randn(100, 100))])
   dmap = hv.DynamicMap(img.plot,                                                                                      
                        streams=[pipe]).hist()
   row = pn.Row(img.param, dmap)

   pn.serve(row, threaded=True)


   while True:                                                                                                         
       pipe.send((np.random.randn(100, 100)))                                                                          
       time.sleep(1) 

Sometimes when interacting with the plot, either changing the color_levels or the zooming/panning on the image or the histogram the backend seems to crash. The error changes depending on the action.

It usually looks something like this though:

WARNING:tornado.access:404 GET /favicon.ico (127.0.0.1) 0.45ms
Traceback (most recent call last):
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/panel/io/server.py", line 390, in unlocked
    WebSocketHandler.write_message(socket, msg.header_json)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/websocket.py", line 340, in write_message
    return self.ws_connection.write_message(message, binary=binary)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/websocket.py", line 1096, in write_message
ERROR:bokeh.server.views.ws:Handler or its work threw an exception: AssertionError(): Message 'PATCH-DOC' content: {'events': [{'kind': 'ModelChanged', 'model': {'id': '1724'}, 'attr': 'inner_width', 'new': 271}, {'kind': 'ModelChanged', 'model': {'id': '1724'}, 'attr': 'inner_height', 'new': 144}, {'kind': 'ModelChanged', 'model': {'id': '1724'}, 'attr': 'outer_width', 'new': 400}, {'kind': 'ModelChanged', 'model': {'id': '1724'}, 'attr': 'outer_height', 'new': 200}], 'references': []}
Traceback (most recent call last):
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/server/views/ws.py", line 247, in on_message
    await self._schedule(work)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/server/views/ws.py", line 319, in _schedule
    await self.send_message(work)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/server/views/ws.py", line 274, in send_message
    await message.send(self)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/protocol/message.py", line 259, in send
    await conn.write_message(self.header_json, locked=False)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/server/views/ws.py", line 289, in write_message
    await super().write_message(message, binary)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/websocket.py", line 340, in write_message
    return self.ws_connection.write_message(message, binary=binary)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/websocket.py", line 1096, in write_message
    fut = self._write_frame(True, opcode, message, flags=flags)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/websocket.py", line 1073, in _write_frame
    return self.stream.write(frame)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/iostream.py", line 544, in write
    self._handle_write()
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/iostream.py", line 974, in _handle_write
    self._write_buffer.advance(num_bytes)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/iostream.py", line 183, in advance
    assert 0 < size <= self._size
AssertionError
ERROR:bokeh.server.views.ws:Bokeh Server internal error: server failed to handle a message, closing connection
WARNING:bokeh.server.views.ws:Failed sending message as connection was closed
    fut = self._write_frame(True, opcode, message, flags=flags)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/websocket.py", line 1073, in _write_frame
    return self.stream.write(frame)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/iostream.py", line 544, in write
    self._handle_write()
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/iostream.py", line 974, in _handle_write
    self._write_buffer.advance(num_bytes)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/tornado/iostream.py", line 202, in advance
    del typing.cast(bytearray, b)[:pos]
BufferError: Existing exports of data: object cannot be re-sized

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "test4.py", line 38, in <module>
    pipe.send((np.random.randn(100, 100)))
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/holoviews/streams.py", line 492, in send
    self.event(data=data)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/holoviews/streams.py", line 430, in event
    self.trigger([self])
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/holoviews/streams.py", line 186, in trigger
    subscriber(**dict(union))
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/holoviews/plotting/plot.py", line 244, in refresh
    raise e
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/holoviews/plotting/plot.py", line 240, in refresh
    self._trigger_refresh(stream_key)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/holoviews/plotting/plot.py", line 257, in _trigger_refresh
    self.update(key)
  File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
    next(self.gen)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/panel/io/server.py", line 401, in unlocked
    curdoc.unhold()
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/document.py", line 669, in unhold
    self._trigger_on_change(event)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/document.py", line 1185, in _trigger_on_change
    self._with_self_as_curdoc(invoke_callbacks)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/document.py", line 1198, in _with_self_as_curdoc
    return f()
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/document.py", line 1184, in invoke_callbacks
    cb(event)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/document.py", line 724, in <lambda>
    self._callbacks[receiver] = lambda event: event.dispatch(receiver)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/events.py", line 269, in dispatch
    super().dispatch(receiver)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/document/events.py", line 124, in dispatch
    receiver._document_patched(self)
  File "/home/seshu/.virtualenvs/ami/lib/python3.7/site-packages/bokeh/server/session.py", line 218, in _document_patched
    raise RuntimeError("_pending_writes should be non-None when we have a document lock, and we should have the lock when the document changes")
RuntimeError: _pending_writes should be non-None when we have a document lock, and we should have the lock when the document changes