Wavesurfer.js time series and spectrograms, just how hard can it be?

[ea42gh]

wavesurfer.js can be used with jupyter lab.
wavesurfer uses Ipython display(HTML) and works.

I’d like to have a solution using panel, so just how difficult could it be?
Well, I can’t manage to load data into wavesurfer, no matter what I tried:

import panel as pn; pn.extension();
import param

%%html
<script src="https://cdn.jsdelivr.net/npm/wavesurfer.js@7"></script>

(I gave up struggling with importing scripts!)

class WaveSurferForPanel(pn.custom.JSComponent):
    width = param.Integer(default=800, doc="Width of the canvas")
    height = param.Integer(default=300, doc="Height of the canvas")
    url = param.String(default="", doc="audio file")

    _esm = """
    export function render({ model, el }) {
        console.log("START");

        // Create a container div for the waveform
        const id = 'waveform_' + Math.random().toString(36).substr(2, 9);
        const container = document.createElement('div');
        container.id = id;
        container.style.width = model.width + 'px';
        container.style.height = model.height + 'px';
        container.style.border = '1px solid black';

        el.appendChild(container);
        console.log("Created container:", container);

        // Initialize WaveSurfer
        const wavesurfer = WaveSurfer.create({
            container: container,
            waveColor: '#4F4A85',
            progressColor: '#383351',
            backend: 'WebAudio',
            responsive: true,
        });

        console.log("WaveSurfer instance created:", wavesurfer);

        // Function to generate an AudioBuffer with sine wave data
        function generateAudioBuffer(audioContext) {
            const duration = 2; // 2 seconds
            const sampleRate = audioContext.sampleRate;
            const length = sampleRate * duration;

            const buffer = audioContext.createBuffer(1, length, sampleRate);
            const channelData = buffer.getChannelData(0);

            for (let i = 0; i < length; i++) {
                channelData[i] = Math.sin(2 * Math.PI * 440 * i / sampleRate); // Generate sine wave at 440 Hz
            }

            return buffer;
        }
        wavesurfer.on('ready', () => {
            console.log("WaveSurfer is ready!");

            const audioContext = wavesurfer.backend.ac; // Access the AudioContext here
            const audioBuffer = generateAudioBuffer(audioContext);


            wavesurfer.backend.setBuffer(audioBuffer); // Set the AudioBuffer
            wavesurfer.drawBuffer(); // Draw the waveform
            console.log("AudioBuffer set and waveform drawn");
        });
    console.log("WaveSurfer setup complete.");
    }
    """
ws = WaveSurferForPanel()
pn.Column( ws, height=ws.height+10 ).servable()

The darn thing won’t load data no matter what I try!
Anybody know how to fix this?

[Marc]

I tried to help but could not get wavesurfer.js to render. I’ve reported it here wavesurfer.js not showing. but playing · Issue #3985 · katspaugh/wavesurfer.js.

[ea42gh]

The darn thing is that
pip install wavesurfer
installs a library that does work, i.e., there is proof it can be made to work.

Trouble is that the code is extensive and sophisticated enough that it will take time to figure out!

PS @Marc Happy New Year!

[ea42gh]

Made some progress looking at wavesurfer.py:
Instead of loading a url into wavesurfer. load pcm:

import panel as pn; pn.extension();
import param
from IPython.display import Audio
import soundfile as sf
import base64

class WaveSurferForPanel(pn.custom.JSComponent):
    width      = param.Integer(default=800, doc="Width of the canvas")
    height     = param.Integer(default=300, doc="Height of the canvas")
    url        = param.String(default="data/test_16k.wav", doc="audio file")
    audio_rate = param.Integer(default=0, doc="sample rate of audio")
    data       = param.String( default="");

    @staticmethod
    def encode(data, rate=None, with_header=True):
        """Transform a wave file or a numpy array to a PCM bytestring"""
        if with_header:
            return Audio(data, rate=rate).src_attr()
        data = np.clip(data, -1, 1)
        scaled, nchan = Audio._validate_and_normalize_with_numpy(data, False)
        return base64.b64encode(scaled).decode("ascii")

    def load_audio(self):
        rate = sf.info(self.url).samplerate if self.audio_rate <= 0 else self.audio_rate
        print( "LOAD ", self.url, "sample rate ", rate);
        self.data = self.encode(self.url, rate)

    _esm = """
    export function render({ model, el }) {
        console.log("START");
let path = window.location.pathname;
let directory = path.substring(0, path.lastIndexOf('/'));
console.log("Current directory:", directory);

        // Create a container div for the waveform
        const id = 'waveform_' + Math.random().toString(36).substr(2, 9);
        const container = document.createElement('div');
        container.id = id;
        container.style.width = model.width + 'px';
        container.style.height = model.height + 'px';
        container.style.border = '1px solid black';

        el.appendChild(container);
        console.log("Created container:", container);

        // Initialize WaveSurfer
        const wavesurfer = WaveSurfer.create({
            container: container,
            waveColor: '#4F4A85',
            progressColor: '#383351',
            backend: 'WebAudio',
            responsive: true,
        });

        console.log("WaveSurfer instance created:", wavesurfer);

        wavesurfer.on('error', function(e) {
            console.error('Wavesurfer error:', e);
            // Display user-friendly error message
        });

        model.on('change:data', () => {
            console.log("got data");
            wavesurfer.load( model.data );
        });
        console.log("WaveSurfer setup complete.");
        return container;
    }
    """

ws = WaveSurferForPanel()

pn.Column( ws, height=ws.height+10 ).servable()

Loading a wavefile will result in the display of the time series:

ws.load_audio()

With this as a starting point, one can now add plugins, e.g., a spectrogram display
and create a gui!
The wavesurfer python package is a great starting point to figure out the details!

[ea42gh]

Add the line
wavesurfer.on('interaction', () => { wavesurfer.playPause() });
after instantiating wavesurfer to enable play start stop with click on waveform

[ea42gh]

I have enough of this working to provide a starting point for audio applications.
I still can’t figure out how to include scripts in JSComponent, so:

%%html
<!-- script src="https://cdn.jsdelivr.net/npm/wavesurfer.js@7"></script -->
<!-- script src="https://unpkg.com/wavesurfer.js/dist/plugin/wavesurfer.plugin/wavesurfer.envelope.min.js"></script-->

<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/wavesurfer.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/envelope.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/hover.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/minimap.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/record.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/regions.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/spectrogram.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/timeline.min.js"></script>
<script src="https://unpkg.com/wavesurfer.js@7.8.14/dist/plugins/zoom.min.js"></script>

import panel as pn; pn.extension();
import param
from IPython.display import Audio
import soundfile as sf
import base64

class WaveSurferForPanel(pn.custom.JSComponent):
    width      = param.Integer(default=500, doc="Width of the canvas")
    height     = param.Integer(default=320, doc="Height of the canvas")
    audio_rate = param.Integer(default=0,   doc="sample rate of audio")
    data       = param.String( default="");
    zoom_level = param.Integer(default= 100, bounds=(10,1000), doc='time series zoom level')
    options    = param.ListSelector(default=['timeline', 'spectrogram', 'zoom', 'hover'], objects=['spectrogram', 'timeline', 'minimap', 'envelope', 'hover', 'record', 'zoom'])
                                            # 'regions',
    @staticmethod
    def encode(data, rate=None, with_header=True):
        """Transform a wave file or a numpy array to a PCM bytestring"""
        if with_header:
            return Audio(data, rate=rate).src_attr()
        data = np.clip(data, -1, 1)
        scaled, nchan = Audio._validate_and_normalize_with_numpy(data, False)
        return base64.b64encode(scaled).decode("ascii")

    def load_audio(self, url):
        rate = sf.info(url).samplerate if self.audio_rate <= 0 else self.audio_rate
        print( "LOAD ", url, "sample rate ", rate);
        self.data = self.encode(url, rate)

    _esm = """
    export function render({ model, el }) {
        // Create a container div for the waveform
        const id = Math.random().toString(36).substr(2, 9);
        const container = document.createElement('div');
        container.id = 'waveform_' + id;
        container.style.width = model.width + 'px';
        container.style.height = model.height + 'px';
        container.style.border = '1px solid black';

        el.appendChild(container);

        function mk_div(nm) {
            const d = document.createElement('div');
            d.id    = nm + id;
            container.appendChild(d);
            return d;
        }
        const tl = mk_div('timeline_');

        // ===================================================================================  WaveSurfer
        // Initialize WaveSurfer
        const wavesurfer = WaveSurfer.create({
            container: container,
            waveColor: '#4F4A85',
            progressColor: '#383351',
            backend: 'WebAudio',
            responsive: true,
            minPxPerSec: 100,
            dragToSeek: true,
        });

        wavesurfer.on('interaction', () => { wavesurfer.playPause() });

        // ===================================================================================  Timeline
        // add a time axis
        function create_timeline() {
          return WaveSurfer.Timeline.create({
            height: 15,
            timeInterval: 0.1,
            primaryLabelInterval: 1,
            insertPosition: 'beforebegin',
            style: {
              fontSize: '11px',
              color: 'black',
            },
          });
        }
        if (model.options.includes('timeline')) {
            wavesurfer.registerPlugin(create_timeline());
        }
        // =================================================================================== Minimap
        // add a small view of the whole file for easy reference and navigation
        function create_minimap() {
          return WaveSurfer.Minimap.create({
            height: 30,
            waveColor: '#DDDD',
            progressColor: '#9999',
            normalize: true,
            plugins: [WaveSurfer.Hover.create({
                lineColor: '#DDDD',
                lineWidth: 1,
                labelBackground: '#5555',
                labelColor: '#FFFF',
                labelSize: '11px',
            })],
          });
        }
        if (model.options.includes('minimap')) {
            wavesurfer.registerPlugin(create_minimap());
        }
        // =================================================================================== Spectrogram
        function create_spectrogram() {
          return WaveSurfer.Spectrogram.create({
            labels: true,
            colorMap: 'igray',  // 'gray', 'roseus'
            fftSamples: 2048,
          });
        }
        if (model.options.includes('spectrogram')) {
            wavesurfer.registerPlugin(create_spectrogram());
        }
        // =================================================================================== Zoom
        // Zoom in or out on the waveform. If spectrogram is displayed, this takes a while!
        function create_zoom() {
          return WaveSurfer.Zoom.create({
              scale:   0.5,
              maxZoom: 1000,
          });
        }
        if (model.options.includes('zoom')) {
            wavesurfer.registerPlugin(create_zoom());
            model.on('change:zoom_level', () => {
                 wavesurfer.zoom(model.zoom_level);
            });
        }
        // ===================================================================================
        function create_hover() {
          return WaveSurfer.Hover.create({
            lineColor: 'red',
            lineWidth: 1,
            labelBackground: '#5555',
            labelColor: '#FFFF',
            labelSize: '11px',
          });
        }
        if (model.options.includes('hover')) {
            wavesurfer.registerPlugin(create_hover());
        }
        // ===================================================================================
        wavesurfer.on('error', function(e) {
            console.error('Wavesurfer error:', e);
            // Display user-friendly error message
        });

        // ===================================================================================
        model.on('change:data', () => {
            console.log("got data");
            wavesurfer.load( model.data );
        });
        return container;

        // ===================================================================================
        wavesurfer.on('error', function(e) {
            console.error('Wavesurfer error:', e);
            // Display user-friendly error message
        });

        // ===================================================================================
        model.on('change:data', () => {
            console.log("got data");
            wavesurfer.load( model.data );
        });
        return container;
    }
    """
# ---------------------------------------------------------------------------------------
ws = WaveSurferForPanel()

pn.Column( ws, height=ws.height+10 ).servable()