Improve timeline MIDI editing and instrument panels (#5683)

* feat(timeline): improve MIDI editing and instrument panels

* feat(timeline): allow optional instrument UI parameters

Author: georgi

packages/timeline/src/types.ts

@@ -623,6 +623,8 @@ export interface WavetableOscillator {
  */
 export interface WavetableMidiInstrument {
   type: "wavetable";
+  /** UI parameters from the WT-1 rack that are not part of the render core. */
+  fableParams?: Record<string, number>;
   oscA: WavetableOscillator;
   oscB: WavetableOscillator;
   /** Sine sub-oscillator level, 0..1. */
@@ -653,6 +655,8 @@ export interface WavetableMidiInstrument {
  */
 export interface BassMidiInstrument {
   type: "bass";
+  /** UI parameters from the BL-1 rack that are not part of the render core. */
+  fableParams?: Record<string, number>;
   table: MidiWavetableName;
   /** Morph position across the table's frames, 0..1. */
   position: number;
@@ -740,6 +744,8 @@ export interface DrumPad {
  */
 export interface DrumMidiInstrument {
   type: "drum";
+  /** UI parameters from the DR-1 rack that are not part of the render core. */
+  fableParams?: Record<string, number>;
   /** The MIDI note pad 0 answers to. Pads run `baseNote`..`baseNote + 15`. */
   baseNote: number;
   pads: DrumPad[];

web/src/components/panels/PanelBottom.tsx

@@ -29,6 +29,7 @@ import WorkerStatusIndicator from "../workers/WorkerStatusIndicator";
 import { VersionHistoryPanel } from "../version";
 import PanelHeadline from "../ui/PanelHeadline";
 import { useCombo } from "../../stores/KeyPressedStore";
+import { useWorkspaceTabsStore } from "../../stores/WorkspaceTabsStore";
 import { TOOLTIP_ENTER_DELAY } from "../../config/constants";
 import { useWorkflowManager } from "../../contexts/WorkflowManagerContext";
 import { ContextMenuProvider } from "../../providers/ContextMenuProvider";
@@ -469,7 +470,11 @@ const PanelBottom: React.FC = () => {
   const systemStats = useSystemStatsStore((state) => state.stats);
 
   useCombo(["Control", "Shift", "T"], () => handlePanelToggle("trace"), false);
-  useCombo(["l"], () => handlePanelToggle("logs"), false);
+  const timelineEditing = useWorkspaceTabsStore((state) => {
+    const tab = state.tabs.find((item) => item.id === state.activeTabId);
+    return tab?.type === "timeline" && tab.mode === "edit";
+  });
+  useCombo(["l"], () => handlePanelToggle("logs"), false, !timelineEditing);
 
   // Shown in the legacy editor (/editor) and the unified workspace (/workspace).
   if (!path.startsWith("/editor") && !path.startsWith("/workspace")) {

web/src/components/timeline/SourceViewerPanel.test.tsx

@@ -43,7 +43,7 @@ jest.mock("../ui_primitives", () => ({
   Text: ({ children }: React.PropsWithChildren) => <span>{children}</span>,
   Tooltip: ({ children }: React.PropsWithChildren) => <>{children}</>,
   TruncatedText: ({ children }: React.PropsWithChildren) => <span>{children}</span>,
-  VideoPlayer: ({ onDurationChange }: { onDurationChange?: (duration: number) => void }) => <button aria-label="Video player" onClick={() => onDurationChange?.(5)}>Video</button>,
+  VideoPlayer: ({ onDurationChange, onTimeUpdate }: { onDurationChange?: (duration: number) => void; onTimeUpdate?: (time: number) => void }) => <button aria-label="Video player" onClick={() => { onDurationChange?.(5); onTimeUpdate?.(2); }}>Video</button>,
   SPACING: { md: 1, xs: 1, sm: 1 },
   getSpacingPx: () => "1px"
 }));
@@ -77,3 +77,25 @@ describe("SourceViewerPanel", () => {
     expect(mockPerformSourceEdit).toHaveBeenCalledWith("append", expect.objectContaining({ ui: expect.objectContaining({ sourceRange: { inMs: 1000, outMs: 2000 } }) }));
   });
 });
+
+
+it("marks the source playhead with I/O without forwarding to the timeline", () => {
+  render(<SourceViewerPanel />);
+  fireEvent.click(screen.getByRole("button", { name: "Video player" }));
+  mockSetSourceRange.mockClear();
+  const forwarded = jest.fn();
+  window.addEventListener("keydown", forwarded);
+  try {
+    fireEvent.keyDown(screen.getByTestId("source-viewer"), { key: "i" });
+    expect(mockSetSourceRange).toHaveBeenLastCalledWith({ inMs: 2000, outMs: 5000 });
+    fireEvent.keyDown(screen.getByTestId("source-viewer"), { key: "o" });
+    expect(mockSetSourceRange).toHaveBeenLastCalledWith({ inMs: 2000, outMs: 2000 });
+    expect(forwarded).not.toHaveBeenCalled();
+    mockSetSourceRange.mockClear();
+    fireEvent.keyDown(screen.getByLabelText("Source in point"), { key: "i" });
+    fireEvent.keyDown(screen.getByTestId("source-viewer"), { key: "i", ctrlKey: true });
+    expect(mockSetSourceRange).not.toHaveBeenCalled();
+  } finally {
+    window.removeEventListener("keydown", forwarded);
+  }
+});

web/src/components/timeline/SourceViewerPanel.tsx

@@ -79,7 +79,6 @@ export const SourceViewerPanel: React.FC = memo(() => {
 
   // The player's own time, so "mark in/out here" reads where it is parked.
   const playerTimeRef = useRef(0);
-  const [playerTimeMs, setPlayerTimeMs] = useState(0);
   const [sourceDurationMs, setSourceDurationMs] = useState<number | null>(null);
   const onDurationChange = useCallback(
     (seconds: number) => {
@@ -93,15 +92,13 @@ export const SourceViewerPanel: React.FC = memo(() => {
   );
   const onTimeUpdate = useCallback((sec: number) => {
     playerTimeRef.current = Math.round(sec * 1000);
-    setPlayerTimeMs(playerTimeRef.current);
   }, []);
 
   // A new asset starts with no range.
   const assetId = asset?.id;
   useEffect(() => {
     setSourceRange(null);
     setSourceDurationMs(null);
-    setPlayerTimeMs(0);
     playerTimeRef.current = 0;
   }, [assetId, setSourceRange]);
 
@@ -135,8 +132,41 @@ export const SourceViewerPanel: React.FC = memo(() => {
   const keys = (action: "sourceAppend" | "sourceInsert" | "sourceOverwrite") =>
     bindingKeys(TIMELINE_KEYMAPS[preset][action][0]);
 
+  const markSource = (edge: "in" | "out") => {
+    const currentRange = sourceRangeFor(asset, uiApi.getState().sourceRange);
+    const timeMs = Math.max(
+      0, Math.min(playerTimeRef.current, sourceDurationMs ?? Infinity)
+    );
+    setSourceRange(
+      edge === "in"
+        ? { inMs: timeMs, outMs: Math.max(timeMs, currentRange.outMs) }
+        : { inMs: Math.min(timeMs, currentRange.inMs), outMs: timeMs }
+    );
+  };
+
+  const handleKeyDown = (event: React.KeyboardEvent<HTMLDivElement>) => {
+    if (
+      event.ctrlKey || event.metaKey || event.altKey || event.shiftKey ||
+      event.target instanceof HTMLInputElement ||
+      event.target instanceof HTMLTextAreaElement ||
+      (event.target instanceof HTMLElement && event.target.isContentEditable)
+    ) return;
+    if (mediaType === "video" && (event.key === "i" || event.key === "o")) {
+      event.preventDefault();
+      event.stopPropagation();
+      markSource(event.key === "i" ? "in" : "out");
+    }
+  };
+
   return (
-    <div css={panelStyles(theme)} data-testid="source-viewer">
+    <div
+      css={panelStyles(theme)}
+      data-testid="source-viewer"
+      role="region"
+      aria-label="Source monitor"
+      tabIndex={0}
+      onKeyDown={handleKeyDown}
+    >
       <FlexColumn gap={SPACING.md}>
         <TruncatedText variant="body2" sx={{ fontWeight: 500 }} showTooltip>
           {asset.name}
@@ -172,7 +202,7 @@ export const SourceViewerPanel: React.FC = memo(() => {
               }}
             />
             {mediaType === "video" && (
-              <Button size="small" variant="text" onClick={() => setSourceRange({ inMs: playerTimeMs, outMs: range.outMs })}>
+              <Button size="small" variant="text" onClick={() => markSource("in")} aria-label="Mark source in (I)">
                 Mark here
               </Button>
             )}
@@ -193,7 +223,7 @@ export const SourceViewerPanel: React.FC = memo(() => {
               }}
             />
             {mediaType === "video" && (
-              <Button size="small" variant="text" onClick={() => setSourceRange({ inMs: range.inMs, outMs: playerTimeMs })}>
+              <Button size="small" variant="text" onClick={() => markSource("out")} aria-label="Mark source out (O)">
                 Mark here
               </Button>
             )}

web/src/components/timeline/TimelineEditor.tsx

@@ -76,6 +76,8 @@ import { TimelineProvider } from "../../stores/timeline/TimelineInstance";
 import { VideoLandingStrip } from "../setup/video/VideoLandingStrip";
 import { useReattachSequenceJobs } from "../../hooks/timeline/useReattachSequenceJobs";
 import { PreviewArea } from "./preview/PreviewArea";
+import { SelectFieldDensityContext } from "../ui_primitives";
+import { TimelineInstrumentsPanel } from "./TimelineInstrumentsPanel";
 import { TimelineInspector } from "./Inspector/TimelineInspector";
 import { SourceViewerPanel } from "./SourceViewerPanel";
 import MovieFilterOutlinedIcon from "@mui/icons-material/MovieFilterOutlined";
@@ -125,7 +127,13 @@ const editorStyles = (theme: Theme) =>
     width: "100%",
     height: "100%",
     overflow: "hidden",
-    backgroundColor: theme.vars.palette.background.default
+    backgroundColor: theme.vars.palette.background.default,
+    "&& .MuiOutlinedInput-root:not(.Mui-focused):not(.Mui-error)": {
+      "& .MuiOutlinedInput-notchedOutline": { borderColor: "transparent" },
+      "&:hover:not(.Mui-disabled) .MuiOutlinedInput-notchedOutline": {
+        borderColor: theme.vars.palette.divider
+      }
+    }
   });
 
 const middleAreaStyles = (theme: Theme) =>
@@ -157,7 +165,8 @@ const dragHandleStyles = (theme: Theme, tall: boolean) =>
     height: tall ? TOUCH_HANDLE_HEIGHT_PX : HANDLE_HEIGHT_PX,
     cursor: "ns-resize",
     flexShrink: 0,
-    backgroundColor: theme.vars.palette.divider,
+    backgroundColor: theme.vars.palette.background.default,
+    boxShadow: `inset 0 1px ${theme.vars.palette.divider}`,
     transition: MOTION.background,
     outline: "none",
     // The handle is a drag target, not a scroll surface: without this a touch
@@ -299,10 +308,11 @@ const PreviewRegion: React.FC<{
 });
 PreviewRegion.displayName = "PreviewRegion";
 
-type InspectorTab = "inspector" | "source" | "agent" | "history" | "script";
+type InspectorTab = "inspector" | "source" | "instrument" | "agent" | "history" | "script";
 
 const INSPECTOR_TABS = [
   { value: "inspector", label: "Inspector", icon: <TuneOutlinedIcon /> },
+  { value: "instrument", label: "Instruments", icon: <TuneOutlinedIcon /> },
   { value: "source", label: "Source", icon: <MovieFilterOutlinedIcon /> },
   { value: "agent", label: "Assistant", icon: <AutoAwesomeIcon /> },
   { value: "history", label: "History", icon: <HistoryOutlinedIcon /> }
@@ -317,7 +327,8 @@ const SCRIPT_TAB = {
 const InspectorRegion: React.FC<{ sequenceId: string | undefined }> = memo(
   ({ sequenceId }) => {
   const theme = useTheme();
-  const [tab, setTab] = useState<InspectorTab>("inspector");
+  const tab = useTimelineUIStore(s => s.panelTab);
+  const setTab = useTimelineUIStore(s => s.setPanelTab);
 
   const tabs = INSPECTOR_TABS;
 
@@ -333,7 +344,6 @@ const InspectorRegion: React.FC<{ sequenceId: string | undefined }> = memo(
           value={tab}
           onChange={(value) => setTab(value as InspectorTab)}
           size="small"
-          fullWidth
           sx={{
             flexShrink: 0,
             borderBottom: `1px solid ${theme.vars.palette.divider}`
@@ -342,6 +352,8 @@ const InspectorRegion: React.FC<{ sequenceId: string | undefined }> = memo(
         <FlexColumn fullWidth sx={{ flex: 1, minHeight: 0, overflow: "hidden" }}>
           {tab === "inspector" ? (
             <TimelineInspector />
+          ) : tab === "instrument" ? (
+            <TimelineInstrumentsPanel />
           ) : tab === "source" ? (
             <SourceViewerPanel />
           ) : tab === "agent" ? (
@@ -406,7 +418,6 @@ const MobilePanelSheet: React.FC<{
       value={activeTab}
       onChange={(value) => onTabChange(value as InspectorTab)}
       size="small"
-      fullWidth
     />
   );
 
@@ -425,6 +436,10 @@ const MobilePanelSheet: React.FC<{
       <FlexColumn fullWidth sx={{ height: "52vh", minHeight: 0 }}>
         {activeTab === "inspector" ? (
           <TimelineInspector />
+        ) : activeTab === "instrument" ? (
+          <TimelineInstrumentsPanel />
+        ) : activeTab === "source" ? (
+          <SourceViewerPanel />
         ) : activeTab === "agent" ? (
           <TimelineAgentPanel />
         ) : activeTab === "script" ? (
@@ -504,7 +519,9 @@ const TimelineEditorBody: React.FC<
 
   // Phone panel sheet (Inspector / Assistant / History / Script).
   const [panelSheetOpen, setPanelSheetOpen] = useState(false);
-  const [panelTab, setPanelTab] = useState<InspectorTab>("inspector");
+  const panelTab = useTimelineUIStore(s => s.panelTab);
+  const setPanelTab = useTimelineUIStore(s => s.setPanelTab);
+  useEffect(() => { if (isMobile && panelTab === "instrument") setPanelSheetOpen(true); }, [isMobile, panelTab]);
   const openPanelSheet = useCallback(() => setPanelSheetOpen(true), []);
   const closePanelSheet = useCallback(() => setPanelSheetOpen(false), []);
   const hasSelection = useTimelineUIStore((s) => s.selectedClipIds.size > 0);
@@ -608,6 +625,13 @@ const TimelineEditorBody: React.FC<
 
   // Tracks resize ─────────────────────────────────────────────────────────
   const [tracksHeight, setTracksHeight] = useState(DEFAULT_TRACKS_HEIGHT_PX);
+  const expandedInstrumentTrackId = useTimelineUIStore((s) => s.expandedInstrumentTrackId);
+  useEffect(() => {
+    if (expandedInstrumentTrackId && !pianoRollOpen) setTracksHeight((height) => Math.max(height, 420));
+  }, [expandedInstrumentTrackId, pianoRollOpen]);
+  useEffect(() => {
+    if (pianoRollOpen) setTracksHeight((height) => Math.min(height, DEFAULT_TRACKS_HEIGHT_PX));
+  }, [pianoRollOpen]);
   const [isDragging, setIsDragging] = useState(false);
   const dragStartYRef = useRef(0);
   const dragStartHeightRef = useRef(DEFAULT_TRACKS_HEIGHT_PX);
@@ -801,6 +825,7 @@ const TimelineEditorBody: React.FC<
   );
 
   return (
+    <SelectFieldDensityContext.Provider value="compact">
     <FlexColumn fullWidth fullHeight css={editorStyles(theme)}>
       {/* ── Top bar ───────────────────────────────────────────────── */}
       <TopBar
@@ -827,7 +852,9 @@ const TimelineEditorBody: React.FC<
         onSelectFolder={(folderId) => void saveAsAsset(folderId, sequence?.name)}
       />
 
-      {/* ── Middle: assets + preview + inspector ──────────────────── */}
+      <FlexRow fullWidth sx={{ flex: 1, minHeight: 0, overflow: "hidden" }}>
+      <FlexColumn sx={{ flex: 1, minWidth: 0, minHeight: 0 }}>
+      {/* ── Middle: assets + preview ──────────────────────────────── */}
       {/* Basis 0 (not `auto`): the middle row absorbs all leftover height via
        *  flex-grow, but its *content* never contributes to the column's size.
        *  With `auto`, a tall inspector (clip selected) inflated this row's
@@ -850,7 +877,6 @@ const TimelineEditorBody: React.FC<
           createSequenceErrorMessage={createErrorMessage}
           fullWidth={isMobile}
         />
-        {!isMobile && <InspectorRegion sequenceId={sequenceId} />}
       </FlexRow>
 
       {/* ── Horizontal drag handle (pointer + keyboard resizable) ─── */}
@@ -877,6 +903,9 @@ const TimelineEditorBody: React.FC<
 
       {/* ── Clip editor (piano roll) ──────────────────────────────── */}
       <PianoRollPanel fullHeight={pianoRollFullScreen} />
+      </FlexColumn>
+      {!isMobile && <InspectorRegion sequenceId={sequenceId} />}
+      </FlexRow>
 
       {/* ── Bottom status bar ─────────────────────────────────────── */}
       <TimelineStatusBar
@@ -948,6 +977,7 @@ const TimelineEditorBody: React.FC<
       </Dialog>
 
     </FlexColumn>
+    </SelectFieldDensityContext.Provider>
   );
 });
 

web/src/components/timeline/TimelineInstrumentsPanel.tsx

@@ -0,0 +1,25 @@
+import React from "react";
+import { useTimelineStore } from "../../stores/timeline/TimelineStore";
+import { useTimelineUIStore } from "../../stores/timeline/TimelineUIStore";
+import { Caption, FlexColumn, SelectField, SPACING } from "../ui_primitives";
+import { TrackInstrumentPanel } from "./Tracks/TrackInstrumentPanel";
+
+export function TimelineInstrumentsPanel() {
+  const tracks = useTimelineStore(state => state.tracks);
+  const clips = useTimelineStore(state => state.clips);
+  const selected = useTimelineUIStore(state => state.selectedClipIds);
+  const instrumentTrackId = useTimelineUIStore(state => state.expandedInstrumentTrackId);
+  const openInstrument = useTimelineUIStore(state => state.toggleExpandedInstrument);
+  const midiTracks = tracks.filter(track => track.type === "midi");
+  const selectedTrack = clips.find(clip => selected.has(clip.id))?.trackId;
+  const track = midiTracks.find(track => track.id === instrumentTrackId)
+    ?? midiTracks.find(track => track.id === selectedTrack) ?? midiTracks[0];
+  if (!track) return <Caption sx={{ p: SPACING.md }}>Add a MIDI track to choose an instrument.</Caption>;
+  return <FlexColumn fullHeight sx={{ minHeight: 0 }} data-testid="timeline-instruments-panel">
+    <FlexColumn sx={{ p: SPACING.sm }}>
+      <SelectField label="Instrument track" hideLabel size="small" value={track.id}
+        options={midiTracks.map(track => ({value: track.id, label: track.name}))} onChange={openInstrument} />
+    </FlexColumn>
+    <TrackInstrumentPanel key={track.id} trackId={track.id} />
+  </FlexColumn>;
+}

web/src/components/timeline/TimelineShortcutsDialog.tsx

@@ -102,6 +102,8 @@ const GROUPS: Group[] = [
     title: "Playback",
     rows: [
       { keys: ["Space"], label: "Play / pause" },
+      { action: "stepFrameBack", label: "Step back one frame" },
+      { action: "stepFrameForward", label: "Step forward one frame" },
       { action: "shuttleBack", label: "Shuttle backwards (again: faster)" },
       { action: "shuttleStop", label: "Stop shuttle" },
       { action: "shuttleForward", label: "Shuttle forwards (again: faster)" },