Change exclude noise (#13)

* Update and update tests.

* Tidy up old noise references.

* Fix tests.

* Fix CI.

* Add Qt deps to linux CI.

Author: JoeZiminski

.github/workflows/test_and_deploy.yml

@@ -37,11 +37,46 @@ jobs:
           python-version: "3.13"
 
     steps:
-      # Run tests
-      - uses: neuroinformatics-unit/actions/test@v2
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install Qt system libraries (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            libegl1 \
+            libgl1 \
+            libxkbcommon-x11-0 \
+            libdbus-1-3 \
+            libxcb-cursor0 \
+            libxcb-icccm4 \
+            libxcb-image0 \
+            libxcb-keysyms1 \
+            libxcb-randr0 \
+            libxcb-render-util0 \
+            libxcb-shape0 \
+            libxcb-sync1 \
+            libxcb-xfixes0 \
+            libxcb-xinerama0 \
+            libxcb-xkb1 \
+            libxkbcommon0 \
+            xvfb
+
+      - name: Install package
+        run: python -m pip install -e ".[dev]"
+
+      - name: Run tests (Linux, headless)
+        if: runner.os == 'Linux'
+        run: xvfb-run -a python -m pytest .
+
+      - name: Run tests
+        if: runner.os != 'Linux'
+        run: python -m pytest .
+
   build_sdist_wheels:
     name: Build source distribution
     needs: [test]

docs/source/pages/how-to-use-driftplots.md

@@ -7,10 +7,14 @@ unit that spike is assigned to.
 Below we will cover the main ways to use `driftplots`.
 See the [API Reference](/pages/api_index) for a full list of arguments.
 
-:::{note}
 See [here](terminology) for a glossary of key terms.
+
+::: {warning}
+`driftplots` was designed and tested with Neuropixels probes, but it should also work with most other probes.
+Please raise a [GitHub issue](https://github.com/neuroinformatics-unit/driftplots) if you have any problems.
 :::
 
+
 ## Inputs
 
 `driftplots` accepts either a path to the output of Kilosort, or a SpikeInterface `SortingAnalyzer` as input.
@@ -29,15 +33,13 @@ on the argument set for `max_spikes_per_unit` used when computing `"random_spike
 
 By default, the number of spikes displayed will be decimated to around `100,000`.
 
-::: {warning}
-`driftplots` was designed and tested with Neuropixels probes, but it should also work with most other probes.
-Please raise a [GitHub issue](https://github.com/neuroinformatics-unit/driftplots) if you have any problems.
+::: {tip}
+`good_units_only=True` is a useful way of excluding spikes from noise and MUA units, tidying up the drift map.
 :::
 
-
 ## Interactive Viewer
 
-{py:meth}`~driftplots.DriftPlotter.drift_map_plot_matplotlib` generates an interactive viewer
+{py:meth}`~driftplots.DriftPlotter.drift_map_plot_interactive` generates an interactive viewer
 allowing the selection of individual spikes on the driftmap. Once selected, the template for the
 unit that spike is associated with will be displayed on the right-hand side.
 
@@ -54,7 +56,7 @@ plotter = DriftPlotter(
 )
 
 driftmap = plotter.drift_map_plot_interactive(
-    exclude_noise=True,
+    good_units_only=True,
 )
 
 driftmap.plot()
@@ -160,7 +162,6 @@ across all sessions and applied to all plots.
 ```python
 import numpy as np
 import spikeinterface as si
-from PySide6 import QtWidgets[how-parameters-are-calculated.md](how-parameters-are-calculated.md)
 
 from driftplots import DriftPlotter, MultiSessionDriftmapWidget, get_amplitudes
 
@@ -172,7 +173,7 @@ SORTING_SESSIONS = [
 ]
 
 all_spike_amplitudes = get_amplitudes(
-    SORTING_SESSIONS, exclude_noise=False, concatenate=True
+    SORTING_SESSIONS, good_units_only=True, concatenate=True
 )
 
 min_cutoff, max_cutoff = np.percentile(all_spike_amplitudes, (0, 95))

driftplots/amplitudes.py

@@ -10,7 +10,7 @@
 
 def get_amplitudes(
     list_of_path_or_analyzer: list[Path | si.SortingAnalyzer],
-    exclude_noise: bool = False,
+    good_units_only: bool | str = False,
     concatenate: bool = False,
     verbose: bool = True,
 ) -> np.ndarray | list[np.ndarray]:
@@ -21,9 +21,13 @@ def get_amplitudes(
     list_of_path_or_analyzer
         Kilosort output directory paths or SpikeInterface ``SortingAnalyzer``
         objects to load amplitudes from. Can be a mix of both.
-    exclude_noise
-        If ``True``, remove spikes belonging to clusters labelled "noise"
-        before returning amplitudes.
+    good_units_only
+        If ``True``, only spikes belonging to "good" units are displayed.
+        For Kilosort, this is taken from the
+        cluster_groups.csv / cluster_group.tsv file that reflects labels
+        set in Phy. For a SortingAnalyzer, a string must be passed.
+        The labels are taken from the sorting property with the
+        passed name (e.g. "KSLabel").
     concatenate
         If ``True``, concatenate all per-session amplitude arrays into a
         single 1-D array.  If ``False`` (default), return a list with one
@@ -44,7 +48,7 @@ def get_amplitudes(
         loader = DataLoader(path_or_analyzer, verbose)
 
         processed_data = loader.get_processed_data(
-            exclude_noise,
+            good_units_only,
             decimate=False,
             filter_amplitude_mode=None,
             filter_amplitude_values=None,

driftplots/data_loader.py

@@ -69,23 +69,18 @@ def __init__(
 
     def get_processed_data(
         self,
-        exclude_noise,
+        good_units_only,
         decimate,
         filter_amplitude_mode,
         filter_amplitude_values,
         verbose: bool,
     ):
         """Filter and subsample the loaded spike data.
 
-        Operations are applied in order: decimation → noise exclusion →
-        amplitude filtering → masking. Decimation is applied first as a
-        performance knob to thin the full dataset before further filtering.
-
         Parameters
         ----------
-        exclude_noise :
-            If ``True``, spikes belonging to clusters labelled "noise" in
-            the Kilosort cluster groups file are removed.
+        good_units_only
+            If ``True``, only spikes belonging to "good" units are kept.
         decimate :
             Keep every *n*-th spike. Applied first to reduce the dataset
             before noise/amplitude filters. ``False`` disables decimation.
@@ -115,19 +110,19 @@ def get_processed_data(
 
         keep_bool_mask = None
 
-        # First, exclude spikes from units labeled as "noise"
-        if exclude_noise:
+        # First, exclude spikes from units that are not labelled "good"
+        if good_units_only:
             if isinstance(self.path_or_analyzer, si.SortingAnalyzer):
-                keep_bool_mask = ~analyzer_helpers.get_noise_mask(
-                    exclude_noise, spike_templates, self.path_or_analyzer
+                keep_bool_mask = analyzer_helpers.get_good_unit_mask(
+                    good_units_only, spike_templates, self.path_or_analyzer
                 )
             else:
-                keep_bool_mask = ~kilosort_helpers.get_noise_mask(
+                keep_bool_mask = kilosort_helpers.get_good_unit_mask(
                     spike_templates, self.path_or_analyzer
                 )
 
             self._print(
-                f"Excluding noise spikes. {keep_bool_mask.sum()} spikes remaining.",
+                f"Keeping good spikes only. {keep_bool_mask.sum()} spikes remaining.",
                 verbose,
             )
 
@@ -154,7 +149,7 @@ def get_processed_data(
                 verbose,
             )
 
-        # mask exclude_noise / filtered amplitudes
+        # mask good_units_only / filtered amplitudes
         if keep_bool_mask is not None:
             spike_times = spike_times[keep_bool_mask]
             spike_amplitudes = spike_amplitudes[keep_bool_mask]

driftplots/driftplotter.py

@@ -11,9 +11,6 @@
 from driftplots.data_loader import DataLoader
 from driftplots.interactive.driftmap_plot_widget import DriftmapPlotWidget
 
-# test ideas:
-# check signatures match default args between interactive and matplotlib
-
 
 class DriftPlotter:
     """Load Kilosort or SpikeInterface output and plot drift maps.
@@ -46,7 +43,7 @@ def __init__(self, path_or_analyzer: str | Path, verbose: bool = True) -> None:
     def drift_map_plot_interactive(
         self,
         decimate: int | bool | None | Literal["estimate"] = "estimate",
-        exclude_noise: bool | str = False,
+        good_units_only: bool | str = False,
         amplitude_cmap_scaling: str | tuple[float, float] = "linear",
         n_color_bins: int = 20,
         point_size: float = 5.0,
@@ -64,12 +61,13 @@ def drift_map_plot_interactive(
             automatically reduce spikes to a reasonable count (≈ 100 000).
             Pass ``False``, ``None``, or ``0`` to disable decimation.  Pass
             an integer *n* to keep every *n*-th spike.
-        exclude_noise
-            If ``True``, remove all spikes belonging to clusters labelled
-            "noise". For Kilosort, this is taken from the cluster_groups.csv /
-            cluster_group.tsv file that reflects labels set in Phy. For a
-            SortingAnalyzer, a string must be passed. The labels are taken from
-            the sorting property with the passed name (e.g. "KSLabel").
+        good_units_only
+            If ``True``, only spikes belonging to "good" units are displayed.
+            For Kilosort, this is taken from the
+            cluster_groups.csv / cluster_group.tsv file that reflects labels
+            set in Phy. For a SortingAnalyzer, a string must be passed.
+            The labels are taken from the sorting property with the
+            passed name (e.g. "KSLabel").
         amplitude_cmap_scaling
             Controls how spike amplitudes are mapped to the greyscale
             colormap.  Pass ``"linear"`` or ``"log2"`` or ``"log10"`` for automatic
@@ -105,7 +103,7 @@ def drift_map_plot_interactive(
         app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([])
 
         processed_data = self._data_loader.get_processed_data(
-            exclude_noise,
+            good_units_only,
             decimate,
             filter_amplitude_mode,
             filter_amplitude_values,
@@ -126,7 +124,7 @@ def drift_map_plot_interactive(
     def drift_map_plot_matplotlib(
         self,
         decimate: int | bool | None | Literal["estimate"] = "estimate",
-        exclude_noise: bool | str = False,
+        good_units_only: bool | str = False,
         amplitude_cmap_scaling: str | tuple[float, float] = "linear",
         n_color_bins: int = 20,
         point_size: float = 5.0,
@@ -147,12 +145,13 @@ def drift_map_plot_matplotlib(
             automatically reduce spikes to a reasonable count (≈ 100 000).
             Pass ``False``, ``None``, or ``0`` to disable decimation.  Pass
             an integer *n* to keep every *n*-th spike.
-        exclude_noise
-            If ``True``, remove all spikes belonging to clusters labelled
-            "noise". For Kilosort, this is taken from the cluster_groups.csv /
-            cluster_group.tsv file that reflects labels set in Phy. For a
-            SortingAnalyzer, a string must be passed. The labels are taken from
-            the sorting property with the passed name (e.g. "KSLabel").
+        good_units_only
+            If ``True``, only spikes belonging to "good" units are displayed.
+            For Kilosort, this is taken from the
+            cluster_groups.csv / cluster_group.tsv file that reflects labels
+            set in Phy. For a SortingAnalyzer, a string must be passed.
+            The labels are taken from the sorting property with the
+            passed name (e.g. "KSLabel").
         amplitude_cmap_scaling
             Controls how spike amplitudes are mapped to the greyscale
             colormap.  Pass ``"linear"`` or ``"log2"`` or ``"log10"`` for automatic
@@ -196,7 +195,7 @@ def drift_map_plot_matplotlib(
             The populated Matplotlib figure.
         """
         processed_data = self._data_loader.get_processed_data(
-            exclude_noise,
+            good_units_only,
             decimate,
             filter_amplitude_mode,
             filter_amplitude_values,

driftplots/extractors/analyzer_helpers.py

@@ -56,27 +56,28 @@ def get_sorting_analyzer(
     )
 
 
-def get_noise_mask(
-    exclude_noise: bool | str,
+def get_good_unit_mask(
+    good_units_only: bool | str,
     spike_templates: np.ndarray,
     analyzer: si.SortingAnalyzer,
 ) -> np.ndarray:
     """ """
-    if exclude_noise is True:
+    if good_units_only is True:
         raise ValueError(
-            f"When using SortingAnalyzer, `exclude_noise` must be a string of the "
-            f"name of the labels to use, passed to `analyzer.get_sorting_property()."
+            f"When using SortingAnalyzer, `good_units_only` must be a string of the "
+            f"name of the sorting property to use."
             f"Properties on this analyzer are: {analyzer.sorting.get_property_keys()}"
         )
 
-    assert isinstance(exclude_noise, str), "`exclude_noise` must be a string"
-    labels = analyzer.get_sorting_property(exclude_noise)
+    assert isinstance(good_units_only, str), "`good_units_only` must be a string"
+    labels = analyzer.get_sorting_property(good_units_only)
 
     if labels is None:
         raise ValueError(
-            f"The analyzer does not contain a sorting property called: {exclude_noise}"
+            "The analyzer does not contain a sorting property called: "
+            f"{good_units_only}"
         )
 
-    noise_mask = (labels == "noise")[spike_templates]
+    good_mask = (labels == "good")[spike_templates]
 
-    return noise_mask
+    return good_mask

driftplots/extractors/kilosort_helpers.py

@@ -50,12 +50,12 @@ def load_cluster_groups(cluster_path: Path) -> tuple[np.ndarray, np.ndarray]:
     return cluster_ids, cluster_groups
 
 
-def get_noise_mask(spike_templates: np.ndarray, sorter_output: Path) -> np.ndarray:
-    """Build a boolean mask identifying spikes that belong to noise-labelled templates.
+def get_good_unit_mask(spike_templates: np.ndarray, sorter_output: Path) -> np.ndarray:
+    """Build a boolean mask identifying spikes that belong to good-labelled templates.
 
     Loads the cluster-groups file (``cluster_groups.csv`` or
     ``cluster_group.tsv``) from the sorter output directory.  Spikes
-    whose cluster is labelled *noise* (group == 0) are marked ``True``.
+    whose cluster is labelled *good* (group == 2) are marked ``True``.
 
     Parameters
     ----------
@@ -68,30 +68,24 @@ def get_noise_mask(spike_templates: np.ndarray, sorter_output: Path) -> np.ndarr
     -------
     np.ndarray
         (num_spikes,) boolean array — ``True`` for spikes belonging to
-        a noise-labelled template.
-
-    Raises
-    ------
-    ValueError
-        If neither ``cluster_groups.csv`` nor ``cluster_group.tsv``
-        exists in ``sorter_output``.
+        a good-labelled template.
     """
     if not (
         (cluster_path := sorter_output / "cluster_groups.csv").is_file()
         or (cluster_path := sorter_output / "cluster_group.tsv").is_file()
     ):
         raise ValueError(
-            f"`exclude_noise` is `True` but there is no `cluster_groups.csv/.tsv` "
+            f"`good_units_only` is `True` but there is no `cluster_groups.csv/.tsv` "
             f"in the sorting output at: {sorter_output}"
         )
 
     cluster_ids, cluster_groups = load_cluster_groups(cluster_path)
 
-    noise_cluster_ids = cluster_ids[cluster_groups == 0]
+    good_cluster_ids = cluster_ids[cluster_groups == 2]
 
-    exclude_bool_mask = np.isin(spike_templates.ravel(), noise_cluster_ids)
+    good_mask = np.isin(spike_templates.ravel(), good_cluster_ids)
 
-    return exclude_bool_mask
+    return good_mask
 
 
 def get_ks_version(sorter_path: Path) -> str:

examples/example_interactive.py

@@ -20,7 +20,6 @@
 
     plot = plotter.drift_map_plot_interactive(
         decimate=False,
-        exclude_noise=False,
         filter_amplitude_mode="percentile",
         filter_amplitude_values=(1, 99),
         amplitude_cmap_scaling="linear",