How to set unchanging ranges on overlays when interacting using panel

I am making a dashboard that combines a few different data sets together and I would like to be able to zoom in and change features of the data without losing the zoom. However, whenever I go to a desirable zoom level and then adjust anything on the widgets I am back to the original size.

I am posting a minimal reproducible example below. It is a bit complex because I am able to get the desired behavior when plotting a single variable using hvplot and setting framewise=False. However, when I combine things and create overlays I am not sure how to achieve it.

P.S. - any advice on whether I am doing anything silly in the code block below, which can be improved will be helpful.

import xarray as xr
import holoviews as hv
import numpy as np
import panel as pn
import hvplot.xarray # noqa: API import


# Create two fields
test_array_1 = xr.DataArray(np.ones((460,500)) + np.linspace(0,3,500),
                         coords=[np.arange(460), np.arange(500)],
                         dims=['x','y'], name='z1')

test_array_2 = xr.DataArray(np.random.rand(460,500) + np.linspace(-7,0,500),
                         coords=[np.arange(460), np.arange(500)],
                         dims=['x','y'], name='z2')

# Create a trajectory
test_array_3 = xr.DataArray(200*(np.sin(np.linspace(0,12, 12))+1), dims=['t'], name='x')
test_array_4 = xr.DataArray(150*(np.cos(np.linspace(0,12, 12))+1),  dims=['t'], name='y')
test_traj = xr.merge([test_array_3, test_array_4])

sel_field = pn.widgets.Select(name='Field', options=['Z1', 'Z2'])
sel_time = pn.widgets.IntRangeSlider(name='Time', start=0, end=12)

@pn.depends(sel_field, sel_time)
def combined_plot(var_sel, time_rng):
    var_plot = plot_var(var_sel)
    traj_plot =  plot_traj(time_rng)
    final_plot = var_plot*traj_plot
    return final_plot.opts(xlim=(0,500), ylim=(0,460))
def plot_var(var_sel):
    if var_sel == 'Z1':
        plot_gen = test_array_1.hvplot.contourf().opts(cmap='RdBu_r', clabel='Z1')
    elif var_sel == 'Z2':
        plot_gen = test_array_2.hvplot.image().opts(cmap='Blues_r',clabel='Z2')
    return plot_gen

def plot_traj(time_rng):
    plot_gen = test_traj.isel(t =slice(time_rng[0], time_rng[1])).hvplot(x='x', y='y',                                    
                                   line_width=2, line_color="orange")
    return plot_gen

pn.Row(pn.WidgetBox(sel_field, sel_time), combined_plot)