Highres quiver plot returning memory error

[iuryt]

I am following the example from Vectorfield — hvPlot 0.9.0 documentation
But I changed the shape to match the size of my current dataset.

import numpy as np
import xarray as xr
import cartopy.crs as ccrs

def sample_data(shape=(904, 999)):
    """
    Return ``(x, y, u, v, crs)`` of some vector data
    computed mathematically. The returned crs will be a rotated
    pole CRS, meaning that the vectors will be unevenly spaced in
    regular PlateCarree space.

    """
    crs = ccrs.UTM(10)

    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v, crs

xs, ys, U, V, crs = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})}, 
                attrs={'crs': crs, 'crs_wkt': doppscat.attrs["crs_wkt"]})
ds

Then the following code returns memory error with a number of points greater than the data points.

ds.hvplot.vectorfield(x='x', y='y', angle='angle', mag='mag', hover=False).opts(magnitude='mag')

I suspect that this is because of the projection. I tried to use Dask by chunking the data, but it doesn’t seem to be integrated with Dask.

---------------------------------------------------------------------------
MemoryError                               Traceback (most recent call last)
File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/util.py:1075, in get_min_distance(element)
   1074     from scipy.spatial.distance import pdist
-> 1075     return pdist(element.array([0, 1])).min()
   1076 except Exception:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/scipy/spatial/distance.py:2233, in pdist(X, metric, out, **kwargs)
   2232     pdist_fn = metric_info.pdist_func
-> 2233     return pdist_fn(X, out=out, **kwargs)
   2234 elif mstr.startswith("test_"):

MemoryError: Unable to allocate 2.97 TiB for an array with shape (407790741060,) and data type float64

During handling of the above exception, another exception occurred:

MemoryError                               Traceback (most recent call last)
File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/IPython/core/formatters.py:972, in MimeBundleFormatter.__call__(self, obj, include, exclude)
    969     method = get_real_method(obj, self.print_method)
    971     if method is not None:
--> 972         return method(include=include, exclude=exclude)
    973     return None
    974 else:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/dimension.py:1286, in Dimensioned._repr_mimebundle_(self, include, exclude)
   1279 def _repr_mimebundle_(self, include=None, exclude=None):
   1280     """
   1281     Resolves the class hierarchy for the class rendering the
   1282     object using any display hooks registered on Store.display
   1283     hooks.  The output of all registered display_hooks is then
   1284     combined and returned.
   1285     """
-> 1286     return Store.render(self)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/options.py:1428, in Store.render(cls, obj)
   1426 data, metadata = {}, {}
   1427 for hook in hooks:
-> 1428     ret = hook(obj)
   1429     if ret is None:
   1430         continue

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:287, in pprint_display(obj)
    285 if not ip.display_formatter.formatters['text/plain'].pprint:
    286     return None
--> 287 return display(obj, raw_output=True)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:255, in display(obj, raw_output, **kwargs)
    253 elif isinstance(obj, (CompositeOverlay, ViewableElement)):
    254     with option_state(obj):
--> 255         output = element_display(obj)
    256 elif isinstance(obj, (Layout, NdLayout, AdjointLayout)):
    257     with option_state(obj):

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:149, in display_hook.<locals>.wrapped(element)
    147 try:
    148     max_frames = OutputSettings.options['max_frames']
--> 149     mimebundle = fn(element, max_frames=max_frames)
    150     if mimebundle is None:
    151         return {}, {}

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:195, in element_display(element, max_frames)
    192 if type(element) not in Store.registry[backend]:
    193     return None
--> 195 return render(element)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:76, in render(obj, **kwargs)
     73 if renderer.fig == 'pdf':
     74     renderer = renderer.instance(fig='png')
---> 76 return renderer.components(obj, **kwargs)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/renderer.py:396, in Renderer.components(self, obj, fmt, comm, **kwargs)
    393 embed = (not (dynamic or streams or self.widget_mode == 'live') or config.embed)
    395 if embed or config.comms == 'default':
--> 396     return self._render_panel(plot, embed, comm)
    397 return self._render_ipywidget(plot)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/renderer.py:403, in Renderer._render_panel(self, plot, embed, comm)
    401 doc = Document()
    402 with config.set(embed=embed):
--> 403     model = plot.layout._render_model(doc, comm)
    404 if embed:
    405     return render_model(model, comm)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/viewable.py:737, in Viewable._render_model(self, doc, comm)
    735 if comm is None:
    736     comm = state._comm_manager.get_server_comm()
--> 737 model = self.get_root(doc, comm)
    739 if self._design and self._design.theme.bokeh_theme:
    740     doc.theme = self._design.theme.bokeh_theme

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/layout/base.py:306, in Panel.get_root(self, doc, comm, preprocess)
    302 def get_root(
    303     self, doc: Optional[Document] = None, comm: Optional[Comm] = None,
    304     preprocess: bool = True
    305 ) -> Model:
--> 306     root = super().get_root(doc, comm, preprocess)
    307     # ALERT: Find a better way to handle this
    308     if hasattr(root, 'styles') and 'overflow-x' in root.styles:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/viewable.py:659, in Renderable.get_root(self, doc, comm, preprocess)
    657 wrapper = self._design._wrapper(self)
    658 if wrapper is self:
--> 659     root = self._get_model(doc, comm=comm)
    660     if preprocess:
    661         self._preprocess(root)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/layout/base.py:174, in Panel._get_model(self, doc, root, parent, comm)
    172 root = root or model
    173 self._models[root.ref['id']] = (model, parent)
--> 174 objects, _ = self._get_objects(model, [], doc, root, comm)
    175 props = self._get_properties(doc)
    176 props[self._property_mapping['objects']] = objects

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/layout/base.py:156, in Panel._get_objects(self, model, old_objects, doc, root, comm)
    154 else:
    155     try:
--> 156         child = pane._get_model(doc, root, model, comm)
    157     except RerenderError as e:
    158         if e.layout is not None and e.layout is not self:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/pane/holoviews.py:411, in HoloViews._get_model(self, doc, root, parent, comm)
    409     plot = self.object
    410 else:
--> 411     plot = self._render(doc, comm, root)
    413 plot.pane = self
    414 backend = plot.renderer.backend

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/pane/holoviews.py:506, in HoloViews._render(self, doc, comm, root)
    503     if comm:
    504         kwargs['comm'] = comm
--> 506 return renderer.get_plot(self.object, **kwargs)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/bokeh/renderer.py:68, in BokehRenderer.get_plot(self_or_cls, obj, doc, renderer, **kwargs)
     61 @bothmethod
     62 def get_plot(self_or_cls, obj, doc=None, renderer=None, **kwargs):
     63     """
     64     Given a HoloViews Viewable return a corresponding plot instance.
     65     Allows supplying a document attach the plot to, useful when
     66     combining the bokeh model with another plot.
     67     """
---> 68     plot = super().get_plot(obj, doc, renderer, **kwargs)
     69     if plot.document is None:
     70         plot.document = Document() if self_or_cls.notebook_context else curdoc()

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/renderer.py:240, in Renderer.get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs)
    237     defaults = [kd.default for kd in plot.dimensions]
    238     init_key = tuple(v if d is None else d for v, d in
    239                      zip(plot.keys[0], defaults))
--> 240     plot.update(init_key)
    241 else:
    242     plot = obj

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/plot.py:955, in DimensionedPlot.update(self, key)
    953 def update(self, key):
    954     if len(self) == 1 and key in (0, self.keys[0]) and not self.drawn:
--> 955         return self.initialize_plot()
    956     item = self.__getitem__(key)
    957     self.traverse(lambda x: setattr(x, '_updated', True))

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/bokeh/element.py:1885, in ElementPlot.initialize_plot(self, ranges, plot, plots, source)
   1883 if self.autorange:
   1884     self._setup_autorange()
-> 1885 self._init_glyphs(plot, element, ranges, source)
   1886 if not self.overlaid:
   1887     self._update_plot(key, plot, style_element)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/bokeh/element.py:1804, in ElementPlot._init_glyphs(self, plot, element, ranges, source)
   1802 else:
   1803     style = self.style[self.cyclic_index]
-> 1804     data, mapping, style = self.get_data(element, ranges, style)
   1805     current_id = element._plot_id
   1807 with abbreviated_exception():

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/bokeh/chart.py:281, in VectorFieldPlot.get_data(self, element, ranges, style)
    279 else:
    280     xidx, yidx = (0, 1)
--> 281 lens = self._get_lengths(element, ranges)/input_scale
    282 cdim = element.get_dimension(self.color_index)
    283 cdata, cmapping = self._get_color_data(element, ranges, style,
    284                                        name='line_color')

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/bokeh/chart.py:256, in VectorFieldPlot._get_lengths(self, element, ranges)
    254             magnitudes = magnitudes / max_magnitude
    255     if self.rescale_lengths:
--> 256         base_dist = get_min_distance(element)
    257         magnitudes = magnitudes * base_dist
    258 else:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/util.py:1077, in get_min_distance(element)
   1075     return pdist(element.array([0, 1])).min()
   1076 except Exception:
-> 1077     return _get_min_distance_numpy(element)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/util.py:1060, in _get_min_distance_numpy(element)
   1058 warnings.filterwarnings('ignore', r'invalid value encountered in')
   1059 xys = xys.astype('float32').view(np.complex64)
-> 1060 distances = np.abs(xys.T-xys)
   1061 np.fill_diagonal(distances, np.inf)
   1062 distances = distances[distances>0]

MemoryError: Unable to allocate 5.93 TiB for an array with shape (903096, 903096) and data type complex64

UPDATE:

I just checked and even without a crs it gives the error.
Why do we have to calculate the distance between all points?

[iuryt]

Btw, related to this topic, is there a way to somehow use datashader to aggregate by mean and optimize this plot by showing increasing higher resolution data by zoom? (I hope this is understandable)

[jbednar]

Using the code above, it’s hard to see how a plot with that many quivers in it would be readable, since it’s unreadable even with fewer than 2% of those arrows:

import numpy as np
import xarray as xr

#def sample_data(shape=(904, 999)):
def sample_data(shape=(120, 130)):
    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v

xs, ys, U, V = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})})
ds

I would guess that it’s possible to overlay a few transparent dynamically rendered low-res datashader-generated aggregates that then drive the quiver plot, but I definitely haven’t tried that. Would be cool to see it! Alternatively, it should be possible to write a callback driven by RangeXY that just subsamples the data for the current resolution, which wouldn’t be quite as fancy but should be more straightforward to write. I can’t think of any current examples to guide you on this, though!

[iuryt]

Hi!

Thank you very much for the attention to this matter.
As an Earth Scientist, I understand that the physics of the ocean and atmosphere is inherently multi-scale and that observations and models are increasing their resolution with time.

Despite not readable as it is, it could be. The behavior I would expect would be to maybe specify max bin size and minimum bin size and show mean-aggregated values for the velocity field. That way, we would look at the large scale physics when zooming out and display smaller scale features with zooming in.

That would be awesome! And depending on that, I would be willing to create a showcase example with real data for the docs.

I just don’t know hwere to start.

I am feeling that HoloViz has so many packages that it makes difficult to navigate and understand their combinations from the various documentations, but I am willing to make an effort into making it as my default way of looking at data.

By the way, what do you think about implementing a quiver function that basically converts u and v to mag and angle and call vectorField?

[jbednar]

Right; I’m saying that the massive non-dynamic plot is unlikely to be useful, but I totally agree that a dynamically constructed quiver appropriate for the current viewport would be very useful. And I think HoloViews can be used to create that, starting from RangeXY — HoloViews v1.18.0. That example creates a histogram driven by the currently visible range, and here instead you’d have a quiver plot driven by the currently visible range. But I don’t think it’s obvious how to build such a plot; basically it needs the sort of logic we already use for datashading, decimation, and other dynamic operations in holoviews.operation Package — HoloViews v1.18.0 . So I think it’s doable, but not obvious how to do it! Maybe someone who wrote one of those other dynamic operations could chime in here.

Does Add VectorField/WindBarbs project operation by philippjfr · Pull Request #296 · holoviz/geoviews · GitHub address your u/v->r/theta conversion request?

[ahuang11]

Could probably use this depending on the range. Rather than rasterize, subset by [::i]

Show individual points when zoomed, else datashade

Worth noting that there’s now a helper for this in the form of apply_when: import holoviews as hv from holoviews.operation.datashader import rasterize from holoviews.operation import apply_when points = hv.Points(np.random.randn(100000, 2)) apply_when(points, operation=rasterize, predicate=lambda x: len(x) > 5000)

and

def filter_points(points, x_range, y_range):
    if x_range is None or y_range is None:
        return points
    return points[x_range, y_range]  # add i here

[jbednar]

Yes, you could use apply_when to set up specific ranges with specific behavior, but for generality what I think would be more useful is to update the display every time one zooms or pans, with a fixed number of arrows, dynamically subselected (using ::i) from the full array.

[iuryt]

The best option for me would be the average instead of the subset, but I think that even the subset would be cool.
If I understand correctly, the idea would be to do something like this?

def filter_points(points, x_range, y_range, nmax = 1000):
    if x_range is None or y_range is None:
        return points
    selected_points = points[x_range, y_range]
    xs, ys = selected_points.shape
    ix, iy = nmax // xs, nmax // ys
    return selected_points[::ix, ::iy]

nmax is the number max of points in each dimension.

[ahuang11]

Yes I believe so.

[iuryt]

I am not sure I understand every step in the code as the reference has fragmented code snippets and I am not very familiar with holoviz packages. So, beforehand, sorry if there is something weird here. The code should work as it is in terms of imports and variable definition, but this is returning error. Any suggestion?

import numpy as np
import xarray as xr
import hvplot.xarray #noqa
import holoviews as hv
from holoviews.operation.datashader import datashade

#def sample_data(shape=(904, 999)):
def sample_data(shape=(90, 90)):
    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v

xs, ys, U, V = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})})


quiver = ds.hvplot.vectorfield(x = "x", y = "y", mag = "mag", angle = "angle", hover = False).opts(magnitude = "mag")

streams = [hv.streams.RangeXY(source = quiver)]

def filter_points(points, x_range, y_range, nmax = 10):
    if x_range is None or y_range is None:
        return points
    selected_points = points[x_range, y_range]
    xs, ys = selected_points.shape
    ix, iy = nmax // xs, nmax // ys
    return selected_points[::ix, ::iy]
    
filtered = quiver.apply(filter_points, streams = streams)
datashade(filtered, width = 400, height = 400, streams = streams)

returns

WARNING:param.dynamic_operation: Callable raised "AttributeError("'VectorField' object has no attribute 'xdensity'")".
Invoked as dynamic_operation(x_range=None, y_range=None)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/IPython/core/formatters.py:972, in MimeBundleFormatter.__call__(self, obj, include, exclude)
    969     method = get_real_method(obj, self.print_method)
    971     if method is not None:
--> 972         return method(include=include, exclude=exclude)
    973     return None
    974 else:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/dimension.py:1286, in Dimensioned._repr_mimebundle_(self, include, exclude)
   1279 def _repr_mimebundle_(self, include=None, exclude=None):
   1280     """
   1281     Resolves the class hierarchy for the class rendering the
   1282     object using any display hooks registered on Store.display
   1283     hooks.  The output of all registered display_hooks is then
   1284     combined and returned.
   1285     """
-> 1286     return Store.render(self)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/options.py:1428, in Store.render(cls, obj)
   1426 data, metadata = {}, {}
   1427 for hook in hooks:
-> 1428     ret = hook(obj)
   1429     if ret is None:
   1430         continue

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:287, in pprint_display(obj)
    285 if not ip.display_formatter.formatters['text/plain'].pprint:
    286     return None
--> 287 return display(obj, raw_output=True)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:261, in display(obj, raw_output, **kwargs)
    259 elif isinstance(obj, (HoloMap, DynamicMap)):
    260     with option_state(obj):
--> 261         output = map_display(obj)
    262 elif isinstance(obj, Plot):
    263     output = render(obj)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:149, in display_hook.<locals>.wrapped(element)
    147 try:
    148     max_frames = OutputSettings.options['max_frames']
--> 149     mimebundle = fn(element, max_frames=max_frames)
    150     if mimebundle is None:
    151         return {}, {}

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:209, in map_display(vmap, max_frames)
    206     max_frame_warning(max_frames)
    207     return None
--> 209 return render(vmap)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/ipython/display_hooks.py:76, in render(obj, **kwargs)
     73 if renderer.fig == 'pdf':
     74     renderer = renderer.instance(fig='png')
---> 76 return renderer.components(obj, **kwargs)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/renderer.py:396, in Renderer.components(self, obj, fmt, comm, **kwargs)
    393 embed = (not (dynamic or streams or self.widget_mode == 'live') or config.embed)
    395 if embed or config.comms == 'default':
--> 396     return self._render_panel(plot, embed, comm)
    397 return self._render_ipywidget(plot)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/renderer.py:403, in Renderer._render_panel(self, plot, embed, comm)
    401 doc = Document()
    402 with config.set(embed=embed):
--> 403     model = plot.layout._render_model(doc, comm)
    404 if embed:
    405     return render_model(model, comm)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/viewable.py:737, in Viewable._render_model(self, doc, comm)
    735 if comm is None:
    736     comm = state._comm_manager.get_server_comm()
--> 737 model = self.get_root(doc, comm)
    739 if self._design and self._design.theme.bokeh_theme:
    740     doc.theme = self._design.theme.bokeh_theme

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/layout/base.py:306, in Panel.get_root(self, doc, comm, preprocess)
    302 def get_root(
    303     self, doc: Optional[Document] = None, comm: Optional[Comm] = None,
    304     preprocess: bool = True
    305 ) -> Model:
--> 306     root = super().get_root(doc, comm, preprocess)
    307     # ALERT: Find a better way to handle this
    308     if hasattr(root, 'styles') and 'overflow-x' in root.styles:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/viewable.py:659, in Renderable.get_root(self, doc, comm, preprocess)
    657 wrapper = self._design._wrapper(self)
    658 if wrapper is self:
--> 659     root = self._get_model(doc, comm=comm)
    660     if preprocess:
    661         self._preprocess(root)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/layout/base.py:174, in Panel._get_model(self, doc, root, parent, comm)
    172 root = root or model
    173 self._models[root.ref['id']] = (model, parent)
--> 174 objects, _ = self._get_objects(model, [], doc, root, comm)
    175 props = self._get_properties(doc)
    176 props[self._property_mapping['objects']] = objects

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/layout/base.py:156, in Panel._get_objects(self, model, old_objects, doc, root, comm)
    154 else:
    155     try:
--> 156         child = pane._get_model(doc, root, model, comm)
    157     except RerenderError as e:
    158         if e.layout is not None and e.layout is not self:

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/pane/holoviews.py:411, in HoloViews._get_model(self, doc, root, parent, comm)
    409     plot = self.object
    410 else:
--> 411     plot = self._render(doc, comm, root)
    413 plot.pane = self
    414 backend = plot.renderer.backend

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/panel/pane/holoviews.py:506, in HoloViews._render(self, doc, comm, root)
    503     if comm:
    504         kwargs['comm'] = comm
--> 506 return renderer.get_plot(self.object, **kwargs)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/bokeh/renderer.py:68, in BokehRenderer.get_plot(self_or_cls, obj, doc, renderer, **kwargs)
     61 @bothmethod
     62 def get_plot(self_or_cls, obj, doc=None, renderer=None, **kwargs):
     63     """
     64     Given a HoloViews Viewable return a corresponding plot instance.
     65     Allows supplying a document attach the plot to, useful when
     66     combining the bokeh model with another plot.
     67     """
---> 68     plot = super().get_plot(obj, doc, renderer, **kwargs)
     69     if plot.document is None:
     70         plot.document = Document() if self_or_cls.notebook_context else curdoc()

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/renderer.py:217, in Renderer.get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs)
    214     raise SkipRendering(msg.format(dims=dims))
    216 # Initialize DynamicMaps with first data item
--> 217 initialize_dynamic(obj)
    219 if not renderer:
    220     renderer = self_or_cls

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/plotting/util.py:270, in initialize_dynamic(obj)
    268     continue
    269 if not len(dmap):
--> 270     dmap[dmap._initial_key()]

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/spaces.py:1216, in DynamicMap.__getitem__(self, key)
   1214 # Not a cross product and nothing cached so compute element.
   1215 if cache is not None: return cache
-> 1216 val = self._execute_callback(*tuple_key)
   1217 if data_slice:
   1218     val = self._dataslice(val, data_slice)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/spaces.py:983, in DynamicMap._execute_callback(self, *args)
    980     kwargs['_memoization_hash_'] = hash_items
    982 with dynamicmap_memoization(self.callback, self.streams):
--> 983     retval = self.callback(*args, **kwargs)
    984 return self._style(retval)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/spaces.py:581, in Callable.__call__(self, *args, **kwargs)
    578     args, kwargs = (), dict(pos_kwargs, **kwargs)
    580 try:
--> 581     ret = self.callable(*args, **kwargs)
    582 except KeyError:
    583     # KeyError is caught separately because it is used to signal
    584     # invalid keys on DynamicMap and should not warn
    585     raise

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/util/__init__.py:1034, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs)
   1032 def dynamic_operation(*key, **kwargs):
   1033     key, obj = resolve(key, kwargs)
-> 1034     return apply(obj, *key, **kwargs)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/util/__init__.py:1026, in Dynamic._dynamic_operation.<locals>.apply(element, *key, **kwargs)
   1024 def apply(element, *key, **kwargs):
   1025     kwargs = dict(util.resolve_dependent_kwargs(self.p.kwargs), **kwargs)
-> 1026     processed = self._process(element, key, kwargs)
   1027     if (self.p.link_dataset and isinstance(element, Dataset) and
   1028         isinstance(processed, Dataset) and processed._dataset is None):
   1029         processed._dataset = element.dataset

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/util/__init__.py:1008, in Dynamic._process(self, element, key, kwargs)
   1006 elif isinstance(self.p.operation, Operation):
   1007     kwargs = {k: v for k, v in kwargs.items() if k in self.p.operation.param}
-> 1008     return self.p.operation.process_element(element, key, **kwargs)
   1009 else:
   1010     return self.p.operation(element, **kwargs)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/operation.py:194, in Operation.process_element(self, element, key, **params)
    191 else:
    192     self.p = param.ParamOverrides(self, params,
    193                                   allow_extra_keywords=self._allow_extra_keywords)
--> 194 return self._apply(element, key)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/operation.py:141, in Operation._apply(self, element, key)
    139     if not in_method:
    140         element._in_method = True
--> 141 ret = self._process(element, key)
    142 if hasattr(element, '_in_method') and not in_method:
    143     element._in_method = in_method

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/operation/datashader.py:1526, in datashade._process(self, element, key)
   1524 def _process(self, element, key=None):
   1525     agg = rasterize._process(self, element, key)
-> 1526     shaded = shade._process(self, agg, key)
   1527     return shaded

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/operation/datashader.py:1268, in shade._process(self, element, key)
   1266     return element.map(partial(shade._process, self), [Element])
   1267 else:
-> 1268     xdensity = element.xdensity
   1269     ydensity = element.ydensity
   1270     bounds = element.bounds

AttributeError: 'VectorField' object has no attribute 'xdensity'

[ahuang11]

Nice! You’re 99.9% there. The issue is I don’t think vectorfield can be datashaded (see Datashade support for VectorField · Issue #5690 · holoviz/holoviews (github.com)) I suppose the error message could be improved so if you can, please submit a GitHub issue for that.

Anyways, all you have to do is drop the datashade line, and return filtered.

import numpy as np
import xarray as xr
import hvplot.xarray #noqa
import holoviews as hv

#def sample_data(shape=(904, 999)):
def sample_data(shape=(90, 90)):
    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v

xs, ys, U, V = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})})


quiver = ds.hvplot.vectorfield(x = "x", y = "y", mag = "mag", angle = "angle", hover = False).opts(magnitude = "mag")

streams = [hv.streams.RangeXY(source = quiver)]

def filter_points(points, x_range, y_range, nmax = 10):
    if x_range is None or y_range is None:
        return points
    selected_points = points[x_range, y_range]
    xs, ys = selected_points.shape
    ix, iy = nmax // xs, nmax // ys
    return selected_points[::ix, ::iy]
    
filtered = quiver.apply(filter_points, streams = streams)
filtered

You should see something like: which is pretty cool honestly.

One other thing you can do is tweak the number of quivers and also the quiver’s size on zoom in/out.

[iuryt]

That is awesome! I can make a PR for the better error message. I belive that would mean to create a new error message before even reaching the part that check if xdensity is in the attributes list. Any suggestion?

[iuryt]

@ahuang11

I just noticed that this doesn’t seem like it is working.
It was supposed to keep at max 10 x 10 arrows in the screen.
Any suggestion?

[iuryt]

I added some print statements and fixed a problem with the proportion. It should be xs // nmax, not nmax // xs I think. I also added an if statement to convert to 1 if ix or iy are zero.

But I don’t understand these prints.

import numpy as np
import xarray as xr
import hvplot.xarray #noqa
import holoviews as hv

#def sample_data(shape=(904, 999)):
def sample_data(shape=(90, 90)):
    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v

xs, ys, U, V = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})})


quiver = ds.hvplot.vectorfield(x = "x", y = "y", mag = "mag", angle = "angle", hover = False).opts(magnitude = "mag")

streams = [hv.streams.RangeXY(source = quiver)]

def filter_points(points, x_range, y_range, nmax = 10):
    if x_range is None or y_range is None:
        return points
    print(points.shape)
    print((x_range, y_range))
    selected_points = points[x_range, y_range]
    xs, ys = selected_points.shape
    print((xs, ys))
    ix, iy = xs // nmax, ys // nmax
    print((ix, iy))
    ix = 1 if ix <= 0 else ix
    iy = 1 if iy <= 0 else iy
    filtered_points = selected_points[::ix, ::iy]
    print(filtered_points.shape)
    return filtered_points
    
filtered = quiver.apply(filter_points, streams = streams)
filtered

It prints only once, no matter the zoom.

(8100, 4)
((310.2028571428571, 392.7971428571429), (-26.020000000000003, 27.220000000000002))
(8100, 4)
(810, 0)
(8100, 4)

I understand that it looks like it is getting the x and y range correctly, but first of all, these are absolute values, not indices, thus, it does not make sense to do points[x_range, y_range]. Btw, this command should give an error and it is not, for some reason. Also, the shape of points is actually the total number of points versus 4, which I don’t understand. Maybe it is actually something like [x, y, mag, angle]?

Anyway, it looks like this command is not being executed filtered_points = selected_points[::ix, ::iy]. Any reason why?

[iuryt]

I tried a different solution for now as I realized points is the same object as quiver, thus, containing the .data attribute as a pandas.dataframe (btw, why? if we used xarray).

Therefore, I am trying to do a simple thing for now. Once this is working it will be easy to use groupby to resample and aggregate by mean as the original idea. For now, I first tried to basically do

quiver.data = quiver.data[::10]
quiver

and it returns somewhat what I would expect to, as converting to a dataframe looses the two-dimensionality, I am just resampling by one of the coordinates.

Then I simply tried to implement that on filter_points():

def filter_points(points, x_range, y_range, nmax = 10):
    if x_range is None or y_range is None:
        return points
    
    data = points.data
    print(data.shape)
    
    inside = (
        (data.x > x_range[0]) & (data.x < x_range[1]) &
        (data.y > y_range[0]) & (data.y < y_range[1])
    )
    
    data = data[inside]

    ip = data.shape[0] // nmax
    ip = 1 if ip <= 0 else ip
    
    data = data[::ip].reset_index(drop = True)

    print(data.shape)
    
    points.data = data

    print(points.data.shape)
    
    return points

which prints the correct new shape for the data and affected points, but doesn’t change the plot.

Any thoughts?

Full reproducible code:

import numpy as np
import xarray as xr
import hvplot.xarray #noqa
import holoviews as hv

#def sample_data(shape=(904, 999)):
def sample_data(shape=(90, 90)):
    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v

xs, ys, U, V = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})})


quiver = ds.hvplot.vectorfield(x = "x", y = "y", mag = "mag", angle = "angle", hover = False).opts(magnitude = "mag")

streams = [hv.streams.RangeXY(source = quiver)]

def filter_points(points, x_range, y_range, nmax = 10):
    if x_range is None or y_range is None:
        return points
    
    data = points.data
    print(data.shape)
    
    inside = (
        (data.x > x_range[0]) & (data.x < x_range[1]) &
        (data.y > y_range[0]) & (data.y < y_range[1])
    )
    
    data = data[inside]

    ip = data.shape[0] // nmax
    ip = 1 if ip <= 0 else ip
    
    data = data[::ip].reset_index(drop = True)

    print(data.shape)
    
    points.data = data

    print(points.data.shape)
    
    return points


filtered = quiver.apply(filter_points, streams = streams)
filtered

[ahuang11]

Similar to

    selected_points = points[x_range, y_range]
    xs, ys = selected_points.shape
    ix, iy = nmax // xs, nmax // ys
    return selected_points[::ix, ::iy]

[iuryt]

Thanks for the clarification. Can you explain what you mean by modifying points directly and how I could do that?

[iuryt]

Maybe this is something specific to VectorField or the way I am implementing it, but the points object that is passed to filter_points is not indexable like this. It is actually the same class as the quiver object (holoviews.element.geom.VectorField) which I couldn’t find a documentation about the indexing. But reading the code I could find out that they work as slicing, thus, points[x_range, y_range] should work, but could not find any way to skip indices, i.e., quiver[::10, ::10] return exactly the same plot. It would be cool to be able to change the plot by altering .data, because then I could use everything pandas provide, such as grouping and aggregation… I don’t know how to proceed. Should I report to holoviews?

[ahuang11]

Indexing should theoretically work; docs on that:
https://holoviews.org/user_guide/Indexing_and_Selecting_Data.html

[iuryt]

I am confused, because indexing doesn’t work as I would expect and it might be just a case of not being used to the package yet.

import numpy as np
import xarray as xr
import hvplot.xarray #noqa
import holoviews as hv

#def sample_data(shape=(904, 999)):
def sample_data(shape=(90, 90)):
    x = np.linspace(311.9, 391.1, shape[1])
    y = np.linspace(-23.6, 24.8, shape[0])

    x2d, y2d = np.meshgrid(x, y)
    u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
    v = 20 * np.cos(6 * np.deg2rad(x2d))

    return x, y, u, v

xs, ys, U, V = sample_data()

mag = np.sqrt(U**2 + V**2)
angle = (np.pi/2.) - np.arctan2(U/mag, V/mag)

ds = xr.Dataset({'mag': xr.DataArray(mag, dims=('y', 'x'), coords={'y': ys, 'x': xs}),
                 'angle': xr.DataArray(angle, dims=('y', 'x'), coords={'y': ys, 'x': xs})})

quiver = ds.hvplot.vectorfield(x = "x", y = "y", mag = "mag", angle = "angle", hover = False).opts(magnitude = "mag")
quiver

returns

if I do

quiver[(320,360), (-10,10)]

it returns

But all the options for skipping values doesn’t seem to work.

quiver[(320,360,2), (-10,10,2)]
# or
# quiver.select(x = (320,360,2), y = (-10,10,2))

return exactly the same object as quiver[(320,360), (-10,10)]. I tried skipping values as 2, 5, 10 etc …
The only way I could skip values was by .iloc, but it only works as if the data was onedimensional, i.e.

quiver.select(x = (320,360), y = (-10,10)).iloc[::5]

subsets, but not as I wanted.

and

quiver.select(x = (320,360), y = (-10,10)).iloc[::5,::5]

returns

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[72], line 1
----> 1 quiver.select(x = (320,360), y = (-10,10)).iloc[::5,::5]

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/data/interface.py:33, in Accessor.__getitem__(self, index)
     31     self.dataset._in_method = True
     32 try:
---> 33     res = self._perform_getitem(self.dataset, index)
     34     if not in_method and isinstance(res, Dataset):
     35         getitem_op = method.instance(
     36             input_type=type(self),
     37             output_type=type(self.dataset),
     38             method_name='_perform_getitem',
     39             args=[index],
     40         )

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/data/interface.py:98, in iloc._perform_getitem(cls, dataset, index)
     95 datatype = [dt for dt in datatypes if dt in Interface.interfaces and
     96             not Interface.interfaces[dt].gridded]
     97 if not datatype: datatype = ['dataframe', 'dictionary']
---> 98 return dataset.clone(data, kdims=kdims, vdims=vdims, datatype=datatype)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/data/__init__.py:1203, in Dataset.clone(self, data, shared_data, new_type, link, *args, **overrides)
   1200 elif self._in_method and 'dataset' not in overrides:
   1201     overrides['dataset'] = self.dataset
-> 1203 return super().clone(data, shared_data, new_type, *args, **overrides)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/dimension.py:563, in LabelledData.clone(self, data, shared_data, new_type, link, *args, **overrides)
    561 # Apply name mangling for __ attribute
    562 pos_args = getattr(self, '_' + type(self).__name__ + '__pos_params', [])
--> 563 return clone_type(data, *args, **{k:v for k,v in settings.items()
    564                                   if k not in pos_args})

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/element/selection.py:24, in SelectionIndexExpr.__init__(self, *args, **kwargs)
     23 def __init__(self, *args, **kwargs):
---> 24     super().__init__(*args, **kwargs)
     25     self._index_skip = False

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/data/__init__.py:332, in Dataset.__init__(self, data, kdims, vdims, **kwargs)
    329 initialized = Interface.initialize(type(self), data, kdims, vdims,
    330                                    datatype=kwargs.get('datatype'))
    331 (data, self.interface, dims, extra_kws) = initialized
--> 332 super().__init__(data, **dict(kwargs, **dict(dims, **extra_kws)))
    333 self.interface.validate(self, validate_vdims)
    335 # Handle _pipeline property

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/dimension.py:841, in Dimensioned.__init__(self, data, kdims, vdims, **params)
    838 if 'cdims' in params:
    839     params['cdims'] = {d if isinstance(d, Dimension) else Dimension(d): val
    840                        for d, val in params['cdims'].items()}
--> 841 super().__init__(data, **params)
    842 self.ndims = len(self.kdims)
    843 cdims = [(d.name, val) for d, val in self.cdims.items()]

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/holoviews/core/dimension.py:500, in LabelledData.__init__(self, data, id, plot_id, **params)
    497     util.group_sanitizer.add_aliases(**{alias:long_name})
    498     params['group'] = long_name
--> 500 super().__init__(**params)
    501 if not util.group_sanitizer.allowable(self.group):
    502     raise ValueError("Supplied group %r contains invalid characters." %
    503                      self.group)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/parameterized.py:3173, in Parameterized.__init__(self, **params)
   3170 self._dynamic_watchers = defaultdict(list)
   3172 self.param._generate_name()
-> 3173 self.param._setup_params(**params)
   3174 object_count += 1
   3176 self.param._update_deps(init=True)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/parameterized.py:1387, in as_uninitialized.<locals>.override_initialization(self_, *args, **kw)
   1385 original_initialized = parameterized_instance.initialized
   1386 parameterized_instance.initialized = False
-> 1387 fn(parameterized_instance, *args, **kw)
   1388 parameterized_instance.initialized = original_initialized

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/parameterized.py:1641, in Parameters._setup_params(self_, **params)
   1639     self.param.warning("Setting non-parameter attribute %s=%s using a mechanism intended only for parameters", name, val)
   1640 # i.e. if not desc it's setting an attribute in __dict__, not a Parameter
-> 1641 setattr(self, name, val)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/parameterized.py:369, in instance_descriptor.<locals>._f(self, obj, val)
    367     instance_param.__set__(obj, val)
    368     return
--> 369 return f(self, obj, val)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/parameterized.py:1201, in Parameter.__set__(self, obj, val)
   1198 if hasattr(self, 'set_hook'):
   1199     val = self.set_hook(obj,val)
-> 1201 self._validate(val)
   1203 _old = NotImplemented
   1204 # obj can be None if __set__ is called for a Parameterized class

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/__init__.py:1442, in List._validate(self, val)
   1437 """
   1438 Checks that the value is numeric and that it is within the hard
   1439 bounds; if not, an exception is raised.
   1440 """
   1441 self._validate_value(val, self.allow_None)
-> 1442 self._validate_bounds(val, self.bounds)
   1443 self._validate_item_type(val, self.item_type)

File ~/miniforge3/envs/coringa/lib/python3.9/site-packages/param/__init__.py:1453, in List._validate_bounds(self, val, bounds)
   1451 if min_length is not None and max_length is not None:
   1452     if not (min_length <= l <= max_length):
-> 1453         raise ValueError("%s: list length must be between %s and %s (inclusive)"%(self.name,min_length,max_length))
   1454 elif min_length is not None:
   1455     if not min_length <= l:

ValueError: kdims: list length must be between 2 and 2 (inclusive)

Update:

After discussing with a colleague, she gave me an idea, but it doesn’t seem like this is the best solution.

quiver.select(x = (320,360), y = (-10,10)).iloc[::5].iloc[::5]