Sync with main (#19)

Author: alejoe91

.gitignore

@@ -169,3 +169,11 @@ cython_debug/
 
 # PyPI configuration file
 .pypirc
+
+# VSCODE
+.vscode/
+
+# CLAUDE
+.claude/
+CLAUDE.md
+*MEMORY.md

Dockerfile

@@ -22,12 +22,23 @@ RUN wget "https://www.wavpack.com/wavpack-${WAVPACK_VERSION}.tar.bz2" && \
 # Install
 RUN pip install wavpack-numcodecs
 
+# Install spikeinterface from source
+RUN git clone https://github.com/SpikeInterface/spikeinterface.git && \
+    cd spikeinterface && \
+    git checkout f732780fd88f5802033b57c9bb9b06229ec7de30 && \
+    pip install . && cd ..
+
+# Force scikit-learn to 1.6.1 to avoid issues with newer versions
+RUN pip install scikit-learn==1.6.1
+
 # Install spikeinterface-gui from source
-RUN git clone https://github.com/SpikeInterface/spikeinterface-gui.git && \
+RUN git clone https://github.com/alejoe91/spikeinterface-gui.git && \
     cd spikeinterface-gui && \
-    git checkout e7c37434169df1ad107acdc2b7a6b4d3fd18821a && \
+    git checkout 47999372e405f5d7a435072ca3015a9fd1b9812c && \
     pip install . && cd ..
 
 
+ENV PYTHONUNBUFFERED=1
+
 EXPOSE 8000
-ENTRYPOINT ["sh", "-c", "panel serve src/aind_ephys_portal/ephys_portal_app.py src/aind_ephys_portal/ephys_gui_app.py --static-dirs images=src/aind_ephys_portal/images --address 0.0.0.0 --port 8000 --allow-websocket-origin ${ALLOW_WEBSOCKET_ORIGIN} --keep-alive 10000 --index ephys_portal_app.py --warm"]
+ENTRYPOINT ["sh", "-c", "panel serve src/aind_ephys_portal/ephys_gui_app.py src/aind_ephys_portal/ephys_portal_app.py src/aind_ephys_portal/ephys_launcher_app.py src/aind_ephys_portal/ephys_monitor_app.py --setup src/aind_ephys_portal/setup.py --static-dirs images=src/aind_ephys_portal/images --address 0.0.0.0 --port 8000 --allow-websocket-origin ${ALLOW_WEBSOCKET_ORIGIN} --index ephys_portal_app.py --check-unused-sessions 2000 --unused-session-lifetime 5000 --num-threads 8"]

README.md

@@ -24,11 +24,24 @@ pip install -e .
 
 ## Usage
 
-To run the Ephys Portal:
+To run the Ephys Portal you can either use tha `launch.sh` script:
 
 ```bash
-# Run using Panel CLI (recommended for server deployment)
-panel serve src/aind_ephys_portal/ephys_portal_app.py src/aind_ephys_portal/ephys_gui_app.py --static-dirs images=src/aind_ephys_portal/images --autoreload
+bash launch.sh
+```
+
+or serve panel apps directly:
+
+```bash
+panel serve \
+    src/aind_ephys_portal/ephys_gui_app.py \
+    src/aind_ephys_portal/ephys_launcher_app.py \
+    src/aind_ephys_portal/ephys_monitor_app.py \
+    src/aind_ephys_portal/ephys_log_app.py \
+    --setup src/aind_ephys_portal/setup.py \
+    --static-dirs images=src/aind_ephys_portal/images \
+    --index ephys_portal_app.py \
+    --num-procs 4
 ```
 
 This will start a Panel server and make the application available in your web browser.
@@ -45,7 +58,7 @@ pip install -e ".[dev]"
 1. Build the Docker image locally and run a Docker container:
 ```sh
 docker build -t aind-ephys-portal .
-docker run -e ALLOW_WEBSOCKET_ORIGIN=0.0.0.0:8000 -p 8000:8000 aind-ephys-portal
+docker run --rm -e ALLOW_WEBSOCKET_ORIGIN=0.0.0.0:8000 -v ~/.aws:/root/.aws:ro -p 8000:8000 aind-ephys-portal
 ```
 2. Navigate to '0.0.0.0:8000` to view the app.
 

launch.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+PORT=${1:-5006}
+
+# Clean up old logs
+rm -rf /tmp/aind_ephys_logs
+
+panel serve \
+    src/aind_ephys_portal/ephys_gui_app.py \
+    src/aind_ephys_portal/ephys_portal_app.py \
+    src/aind_ephys_portal/ephys_launcher_app.py \
+    src/aind_ephys_portal/ephys_monitor_app.py \
+    --setup src/aind_ephys_portal/setup.py \
+    --static-dirs images=src/aind_ephys_portal/images \
+    --check-unused-sessions 2000 \
+    --unused-session-lifetime 5000 \
+    --port $PORT \
+    --num-threads 8

pyproject.toml

@@ -13,15 +13,13 @@ authors = [
     {name = "Allen Institute for Neural Dynamics"}
 ]
 dependencies = [
-    "panel>=1.0.0,<1.6.2",
+    "panel>=1.0.0",
     "param>=1.12.0",
-    "aind-data-access-api>=0.1.0",
-    "spikeinterface[full]>=0.102.0",
-    "pandas>=1.2.0",
-    "numpy>=1.20.0",
+    "psutil",
     "boto3",
     "s3fs>=2025.2.0",
-    "numcodecs<0.16.0",
+    "aind-data-access-api>=0.1.0",
+    "spikeinterface[full]>=0.103.0",
     "spikeinterface-gui"
 ]
 

src/aind_ephys_portal/docdb/database.py

@@ -102,16 +102,27 @@ def get_raw_asset_by_name(asset_name: str):
 
 
 @pn.cache(ttl=TIMEOUT_1H)
-def get_all_ecephys_derived() -> List[Dict[str, Any]]:
+def get_all_ecephys_derived(additional_includes_in_name: str | None = None) -> List[Dict[str, Any]]:
     """Get a limited set of all records from the database.
 
     Returns
     -------
     list[dict]
         List of records, limited to 50 entries.
+    additional_includes_in_name : str, optional
+        Comma-separated list of additional fields to include in the results, by default None
     """
     filter_query = {"data_description.modality.abbreviation": "ecephys", "data_description.data_level": "derived"}
-    response = client.retrieve_docdb_records(
+    responses = client.retrieve_docdb_records(
         filter_query=filter_query,
     )
-    return response
+    responses_filt = []
+    if additional_includes_in_name:
+        additional_fields = [field.strip() for field in additional_includes_in_name.split(",")]
+        for record in responses:
+            for field in additional_fields:
+                if field in record["name"]:
+                    responses_filt.append(record)
+    else:
+        responses_filt = responses
+    return responses_filt

src/aind_ephys_portal/ephys_gui_app.py

@@ -3,32 +3,77 @@
 import urllib
 import param
 
-from aind_ephys_portal.panel.ephys_gui import EphysGuiView
-
 import panel as pn
 
 pn.extension("tabulator", "gridstack")
 
-from spikeinterface_gui.utils_panel import KeyboardShortcuts
+# Import for side effects: registers SpikeInterface GUI custom Panel/Bokeh models/resources
+import spikeinterface_gui.utils_panel  # noqa: F401
 
+from aind_ephys_portal.panel.ephys_gui import EphysGuiView
+
+def _get_arg(name: str, default: str = "") -> str:
+    """Read a query-parameter from the raw HTTP request."""
+    val = pn.state.session_args.get(name, [default.encode()])
+    if isinstance(val, list):
+        val = val[0]
+    if isinstance(val, bytes):
+        val = val.decode()
+    return val
+    
 
 # State sync
 class Settings(param.Parameterized):
     """Top-level settings for QC app"""
 
     analyzer_path = param.String(default="")
     recording_path = param.String(default="")
+    identifier = param.String(default="")
+    fast_mode = param.Boolean(
+        default=False,
+        doc="Whether to enable fast mode (skips waveforms and principal components)"
+    )
+    preload_curation = param.Boolean(
+        default=False,
+        doc="Whether to preload existing curation from disk (if available)"
+    )
 
 
 settings = Settings()
-pn.state.location.sync(settings, {"analyzer_path": "analyzer_path", "recording_path": "recording_path"})
+pn.state.location.sync(settings, {"analyzer_path": "analyzer_path", "recording_path": "recording_path", "identifier": "identifier", "fast_mode": "fast_mode", "preload_curation": "preload_curation"})
+
+
+
+analyzer_path = urllib.parse.unquote(_get_arg("analyzer_path"))
+recording_path = urllib.parse.unquote(_get_arg("recording_path"))
+identifier = urllib.parse.unquote(_get_arg("identifier"))
+fast_mode = _get_arg("fast_mode", "false").lower() in ("true", "1", "yes")
+preload_curation = _get_arg("preload_curation", "false").lower() in ("true", "1", "yes")
+
+print(f"Parsed arguments:")
+print(f"\tanalyzer_path={analyzer_path}\n\trecording_path={recording_path}\n\tidentifier={identifier}\n\tfast_mode={fast_mode}\n\tpreload_curation={preload_curation}")
+
+ephys_gui = EphysGuiView(analyzer_path=analyzer_path, recording_path=recording_path, identifier=identifier, fast_mode=fast_mode, preload_curation=preload_curation)
 
-# Manually decode stream_name after syncing
-settings.analyzer_path = urllib.parse.unquote(settings.analyzer_path)
-settings.recording_path = urllib.parse.unquote(settings.recording_path)
+ephys_gui.panel().servable(title="AIND Ephys GUI")
 
+# Register session cleanup on the document. The document is definitively bound
+# to this session at module-execution time, so curdoc is correct here.
+# Factory function creates a proper closure (cell variable) for view_ref,
+# which is immune to Panel's exec() scoping (exec uses separate globals/locals
+# dicts, so plain module-level names are inaccessible inside callbacks).
+def _make_cleanup_callback(view):
+    view_ref = [view]
+    def _on_session_destroyed(session_context):
+        v = view_ref[0]
+        view_ref[0] = None  # always break the reference, even if cleanup raises
+        if v is not None:
+            v.cleanup()
+    return _on_session_destroyed
 
-ephys_gui = EphysGuiView(analyzer_path=settings.analyzer_path, recording_path=settings.recording_path)
-ephys_gui_panel = ephys_gui.panel()
+pn.state.curdoc.on_session_destroyed(_make_cleanup_callback(ephys_gui))
+print(f"[GUI] Cleanup registered on doc {id(pn.state.curdoc)}")
 
-ephys_gui_panel.servable(title="AIND Ephys GUI")
+# Don't keep a module-level reference to the heavy GUI object.
+# The servable layout is now owned by Panel/Bokeh's document.
+del ephys_gui

src/aind_ephys_portal/ephys_launcher_app.py

@@ -0,0 +1,115 @@
+"""Standalone launcher page to generate GUI links."""
+
+import urllib.parse
+
+import panel as pn
+
+from aind_ephys_portal.panel.logging import setup_logging
+from aind_ephys_portal.panel.utils import EPHYSGUI_LINK_PREFIX
+
+pn.extension()
+
+
+class EphysLauncher:
+    def __init__(self):
+        setup_logging()
+
+        self.analyzer_input = pn.widgets.TextInput(
+            name="Analyzer path",
+            value="",
+            height=50,
+            sizing_mode="stretch_width",
+        )
+        self.recording_input = pn.widgets.TextInput(
+            name="Recording path (optional)",
+            value="",
+            height=50,
+            sizing_mode="stretch_width",
+        )
+
+        self.generate_button = pn.widgets.Button(
+            name="Generate Link",
+            button_type="primary",
+            height=45,
+            sizing_mode="stretch_width",
+        )
+
+        self.fast_mode_checkbox = pn.widgets.Checkbox(
+            name="Enable Fast Mode (skip waveforms and PCs)",
+            value=False,
+            height=30,
+            sizing_mode="stretch_width",
+        )
+
+        self.preload_curation = pn.widgets.Checkbox(
+            name="Preload Curation",
+            value=True,
+            height=30,
+            sizing_mode="stretch_width",
+        )
+
+        self.link_pane = pn.pane.HTML("No link generated yet.", sizing_mode="stretch_width")
+        self.url_output = pn.widgets.TextInput(
+            name="GUI URL",
+            value="",
+            disabled=True,
+            sizing_mode="stretch_width",
+        )
+
+        self.generate_button.on_click(self._update_link)
+
+        self.layout = pn.Column(
+            pn.pane.Markdown("## AIND Ephys Launcher"),
+            self.analyzer_input,
+            self.recording_input,
+            pn.Row(self.generate_button, self.fast_mode_checkbox, self.preload_curation, sizing_mode="stretch_width"),
+            self.link_pane,
+            self.url_output,
+            sizing_mode="stretch_width",
+        )
+
+    def _build_gui_url(self):
+        analyzer_path = self.analyzer_input.value.strip()
+        recording_path = self.recording_input.value.strip()
+        if not analyzer_path:
+            return None
+
+        analyzer_path_q = urllib.parse.quote(analyzer_path, safe="")
+        recording_path_q = urllib.parse.quote(recording_path, safe="") if recording_path else ""
+        path = EPHYSGUI_LINK_PREFIX.format(analyzer_path_q, recording_path_q)
+        if self.fast_mode_checkbox.value:
+            path += "&fast_mode=true"
+        if self.preload_curation.value:
+            path += "&preload_curation=true"
+
+        location = pn.state.location
+        if location is not None:
+            href = getattr(location, "href", None)
+            if href:
+                parsed = urllib.parse.urlparse(href)
+                if parsed.scheme and parsed.netloc:
+                    return f"{parsed.scheme}://{parsed.netloc}{path}"
+
+            protocol = getattr(location, "protocol", "")
+            hostname = getattr(location, "hostname", "")
+            port = getattr(location, "port", "")
+            if protocol and hostname:
+                netloc = f"{hostname}:{port}" if port else hostname
+                return f"{protocol}//{netloc}{path}"
+
+        return path
+
+    def _update_link(self, event=None):
+        url = self._build_gui_url()
+        if not url:
+            self.link_pane.object = "<span style='color: orange;'>⚠️ Analyzer path is required.</span>"
+            self.url_output.value = ""
+            return
+        print(f"Generated URL: {url}")
+
+        self.link_pane.object = f'<a href="{url}" target="_blank">Ephys Curation GUI</a>'
+        self.url_output.value = url
+
+
+app = EphysLauncher()
+app.layout.servable(title="AIND Ephys Launcher")