perf: background write batcher for DuckDB telemetry ingest

CLAUDE.md

@@ -16,7 +16,7 @@ Traceway is an error tracking and monitoring platform consisting of:
 
 - **No pointless comments**: Do not add comments that simply describe what the code does. The code should be self-explanatory. Only add comments when explaining non-obvious "why" decisions.
 - **No `py-4` in dialog form content**: Do not add `py-4` on the content wrapper inside `AlertDialog` or `Dialog` components — it creates too much blank space between the form and the action buttons.
-- **Dialog button labels & toasts**: For form dialogs, use descriptive button labels with icons instead of generic "Create"/"Update". The `{Entity}` is always a platform entity, capitalized (Project, Widget, Widget Group, Channel, Rule, Token, Invitation, ...). Create actions: `<Plus icon> New {Entity}` with `variant="success"`. Update actions: `<Check icon> Update {Entity}` with the default (primary) variant. Delete/revoke/remove confirm buttons: `<Trash2 icon> Delete {Entity}` (or `Revoke {Entity}` / `Remove {Entity}`) with `variant="destructive"`. After success, show `toast.success('Successfully created the {Entity}', { position: 'top-center' })` for creates and `'Successfully updated the {Entity}'` for updates. The button should only be `disabled` during the loading state — never disable it to enforce validation; let the backend return 422 and show the error in the dialog instead.
+- **Dialog button labels & toasts**: For form dialogs, use descriptive button labels with icons instead of generic "Create"/"Update". The `{Entity}` is always a platform entity, capitalized (Project, Widget, Dashboard, Channel, Rule, Token, Invitation, ...). Create actions: `<Plus icon> New {Entity}` with `variant="success"`. Update actions: `<Check icon> Update {Entity}` with the default (primary) variant. Delete/revoke/remove confirm buttons: `<Trash2 icon> Delete {Entity}` (or `Revoke {Entity}` / `Remove {Entity}`) with `variant="destructive"`. After success, show `toast.success('Successfully created the {Entity}', { position: 'top-center' })` for creates and `'Successfully updated the {Entity}'` for updates. The button should only be `disabled` during the loading state — never disable it to enforce validation; let the backend return 422 and show the error in the dialog instead.
 
 ---
 
@@ -488,7 +488,7 @@ const handleClick = createRowClickHandler('/issues/abc123', 'preset', 'from', 't
 /endpoints/[endpoint]       Single endpoint details
 /tasks                      Background tasks list
 /tasks/[task]               Single task details
-/metrics                    System metrics dashboard (CPU, memory, etc.)
+/dashboards                 Dashboards page (tabs of org dashboards; /metrics redirects here)
 /connection                 SDK integration guide
 ```
 
@@ -620,22 +620,40 @@ backend/
 | GET | `/api/metrics/discover/tags` | App | Discover metric tags |
 | PUT | `/api/metrics/registry` | App+Write | Update metric registry entry |
 
-**Widget Groups & Widgets**
+**Dashboards & Templates**
+
+Dashboards are org-owned JSON documents (`{schemaVersion, widgets: [{id, title, widgetType, config}]}`, widget ids are server-generated `w_xxxxxxxx` strings, array order = display order) applied to projects via `project_dashboards`. Dashboard mutations require org role above `readonly` (checked in-handler); project-scoped routes (list/star/reorder/populate) use the standard middleware chains; apply/unapply also check the effective role of each affected project. The old per-project widget-group tables are converted once at startup by `backfill.RunDashboards` (advisory-locked on PG) and retained for rollback.
+
 | Method | Endpoint | Auth | Purpose |
 |--------|----------|------|---------|
-| GET | `/api/widget-groups` | App | List widget groups |
-| POST | `/api/widget-groups` | App+Write | Create widget group |
-| GET | `/api/widget-groups/:id` | App | Get group with widgets |
-| PUT | `/api/widget-groups/:id` | App+Write | Update widget group |
-| DELETE | `/api/widget-groups/:id` | App+Write | Delete widget group |
-| POST | `/api/widget-groups/:id/widgets` | App+Write | Add widget |
-| PUT | `/api/widget-groups/:id/widgets/:wid` | App+Write | Update widget |
-| PUT | `/api/widget-groups/:id/reorder` | App+Write | Reorder widgets (explicit id order) |
-| PUT | `/api/widget-groups/:id/widgets/:wid/star` | App+Write | Star/unstar widget for the homepage |
-| DELETE | `/api/widget-groups/:id/widgets/:wid` | App+Write | Delete widget |
-| GET | `/api/widget-groups/starred` | App | List starred widgets with homepage layout |
-| PUT | `/api/starred-widgets/reorder` | App+Write | Reorder homepage starred widgets (explicit id order) |
-| PUT | `/api/starred-widgets/:wid` | App+Write | Update homepage layout (colSpan/size) of a starred widget |
+| GET | `/api/dashboards` | App | Dashboards applied to the project (tab order) |
+| GET | `/api/dashboards/library` | App | All dashboards across the user's orgs with applied project ids |
+| POST | `/api/dashboards` | App | Create in org (auto-applies to current project unless `applyToProjectIds` given) |
+| GET | `/api/dashboards/:id` | App | Meta + widgets (+ per-project `isStarred`, `appliedProjectIds`) |
+| PUT | `/api/dashboards/:id` | App | Update name/description and/or full `definition` (the as-code path) |
+| DELETE | `/api/dashboards/:id` | App | Delete everywhere (assignments + stars cascade) |
+| PUT | `/api/dashboards/:id/apply` | App | Set the full project assignment list |
+| DELETE | `/api/dashboards/:id/apply/:projectId` | App | Unassign from one project |
+| POST | `/api/dashboards/:id/copy` | App | Copy (also cross-org) with optional apply |
+| PUT | `/api/dashboards/reorder` | App+Write | Tab order for a project (explicit id order) |
+| POST | `/api/dashboards/:id/widgets` | App | Add widget |
+| PUT | `/api/dashboards/:id/widgets/reorder` | App | Reorder widgets (explicit id order) |
+| PUT | `/api/dashboards/:id/widgets/:wid` | App | Update widget |
+| DELETE | `/api/dashboards/:id/widgets/:wid` | App | Delete widget (+ its stars) |
+| PUT | `/api/dashboards/:id/widgets/:wid/star` | App+Write | Star/unstar for the project homepage |
+| GET | `/api/dashboards/starred` | App | Starred widgets with homepage layout |
+| PUT | `/api/starred-widgets/reorder` | App+Write | Reorder homepage starred widgets (`{ids}` = starred row ids) |
+| PUT | `/api/starred-widgets/:id` | App+Write | Update homepage layout (colSpan/size) |
+| GET | `/api/dashboards/:id/export` | App | Export one dashboard as JSON |
+| GET | `/api/dashboards/export?organizationId=` | App | Export the org bundle |
+| POST | `/api/dashboards/import` | App | Import doc/bundle (`mode: create\|upsert`, upsert matches by name) |
+| POST | `/api/dashboards/import/grafana` | App | Convert a Grafana export (best-effort, returns `warnings[]`) |
+| GET | `/api/dashboard-templates` | App | Marketplace list/search (`search`, `category` params) |
+| POST | `/api/dashboard-templates/:key/install` | App | Copy a template into the org and apply |
+| POST | `/api/dashboards/populate-defaults` | App+Write | Install the framework-default template set for an empty project |
+| GET | `/api/metrics/discover/org` | App | Metric names across all org projects (command palette) |
+
+Templates are DB rows seeded by migrations (`traceway-otel-agent` for the OTel host agent, `golang` for Go SDK apps, `traceway-clickhouse`/`traceway-duckdb` for the telemetry stores of a monitored Traceway instance; SQLite emits no store-specific metrics so it has no template); cloud can insert more rows without a release. The OTLP metric ingest allowlists per-resource grouping tags (`container.name`, `k8s.pod.name`, `k8s.node.name`, `postgresql.database.name`, ...) in `otelcontrollers/metric_converter.go` for custom widgets.
 
 **Endpoints**
 | Method | Endpoint | Auth | Purpose |
@@ -741,9 +759,11 @@ func (c *ReportController) Report(ctx *gin.Context) {
 | `invitations` | Team invitations with token, role, expiry |
 | `source_maps` | Uploaded source map files (project, version, storage key) |
 | `metric_registry` | Custom metric definitions (type, unit, description) |
-| `widget_groups` | Dashboard widget groups (name, default flag) |
-| `widget_group_widgets` | Individual widgets within groups (type, config, position) |
-| `starred_widgets` | Homepage layout for starred widgets (position, col_span, size per widget) |
+| `dashboards` | Org-owned dashboards (name, JSONB definition with widgets, template provenance) |
+| `project_dashboards` | Which projects show a dashboard, and tab order |
+| `dashboard_templates` | Marketplace templates (key, category, definition), seeded by migrations |
+| `starred_dashboard_widgets` | Homepage layout per project (dashboard id + widget id, position, col_span, size) |
+| `widget_groups` / `widget_group_widgets` / `starred_widgets` | Legacy pre-dashboards tables, retained read-only for rollback until a follow-up drop |
 
 #### ClickHouse vs PostgreSQL Decision Guide
 - **PostgreSQL**: Relational/config data needing ACID, frequent updates, JOINs, low volume (users, organizations, projects, invitations, widgets, source maps, metric registry)
@@ -761,7 +781,7 @@ In SQLite mode (`DB_TYPE=sqlite`), the backend uses **two separate SQLite databa
 
 **Main DB tables** (`db.DB` — transactional, uses lit with `*sql.Tx`):
 - `users`, `organizations`, `organization_users`, `projects`, `invitations`
-- `source_maps`, `metric_registry`, `widget_groups`, `widget_group_widgets`, `starred_widgets`
+- `source_maps`, `metric_registry`, `dashboards`, `project_dashboards`, `dashboard_templates`, `starred_dashboard_widgets` (plus the legacy `widget_groups`/`widget_group_widgets`/`starred_widgets`)
 - `notification_channels`, `notification_rules`
 
 **Telemetry DB tables** (`db.TelemetryDB` — non-transactional, uses lit with `db.TelemetryDB` directly):

backend/app/backfill/dashboards.go

@@ -0,0 +1,263 @@
+package backfill
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/tracewayapp/lit/v2"
+	"github.com/tracewayapp/traceway/backend/app/config"
+	"github.com/tracewayapp/traceway/backend/app/db"
+	"github.com/tracewayapp/traceway/backend/app/models"
+	"github.com/tracewayapp/traceway/backend/app/repositories/transactional"
+	dashboardsvc "github.com/tracewayapp/traceway/backend/app/services/dashboards"
+
+	"github.com/google/uuid"
+)
+
+const dashboardsBackfillName = "dashboards"
+
+const maxDashboardNameRunes = 100
+
+type widgetGroupWithOrg struct {
+	Id             int       `lit:"id"`
+	ProjectId      uuid.UUID `lit:"project_id"`
+	Name           string    `lit:"name"`
+	Description    string    `lit:"description"`
+	IsDefault      bool      `lit:"is_default"`
+	CreatedBy      *int      `lit:"created_by"`
+	CreatedAt      time.Time `lit:"created_at"`
+	UpdatedAt      time.Time `lit:"updated_at"`
+	OrganizationId int       `lit:"organization_id"`
+	ProjectName    string    `lit:"project_name"`
+}
+
+func init() {
+	models.ExtensionModelRegistrations = append(models.ExtensionModelRegistrations, func(driver lit.Driver) {
+		lit.RegisterModel[widgetGroupWithOrg](driver)
+	})
+}
+
+func RunDashboards() error {
+	converted, err := db.ExecuteTransaction(func(tx *sql.Tx) (int, error) {
+		if !db.IsSQLite() {
+			if _, err := tx.Exec("SELECT pg_advisory_xact_lock(824737001)"); err != nil {
+				return 0, fmt.Errorf("failed to take backfill lock: %w", err)
+			}
+		}
+
+		marker, err := lit.SelectSingleNamed[models.CountResult](
+			tx,
+			"SELECT COUNT(*) as count FROM backfill_runs WHERE name = :name",
+			lit.P{"name": dashboardsBackfillName},
+		)
+		if err != nil {
+			return 0, fmt.Errorf("failed to check backfill marker: %w", err)
+		}
+		if marker != nil && marker.Count > 0 {
+			return 0, nil
+		}
+
+		existing, err := lit.SelectSingleNamed[models.CountResult](tx, "SELECT COUNT(*) as count FROM dashboards", lit.P{})
+		if err != nil {
+			return 0, fmt.Errorf("failed to count dashboards: %w", err)
+		}
+
+		converted := 0
+		if existing == nil || existing.Count == 0 {
+			converted, err = convertWidgetGroups(tx)
+			if err != nil {
+				return 0, err
+			}
+		}
+
+		query, args, err := lit.ParseNamedQuery(
+			db.Driver,
+			"INSERT INTO backfill_runs (name, ran_at) VALUES (:name, :ran_at)",
+			lit.P{"name": dashboardsBackfillName, "ran_at": time.Now().UTC()},
+		)
+		if err != nil {
+			return 0, err
+		}
+		if err := lit.UpdateNative(tx, query, args...); err != nil {
+			return 0, fmt.Errorf("failed to record backfill marker: %w", err)
+		}
+
+		return converted, nil
+	})
+	if err != nil {
+		return err
+	}
+	if converted > 0 {
+		config.Logf("dashboards backfill: converted %d widget group(s) into dashboards", converted)
+	}
+	return nil
+}
+
+func convertWidgetGroups(tx *sql.Tx) (int, error) {
+	skipped, err := lit.SelectSingleNamed[models.CountResult](
+		tx,
+		"SELECT COUNT(*) as count FROM widget_groups wg JOIN projects p ON p.id = wg.project_id WHERE p.organization_id IS NULL",
+		lit.P{},
+	)
+	if err != nil {
+		return 0, fmt.Errorf("failed to count orphaned widget groups: %w", err)
+	}
+	if skipped != nil && skipped.Count > 0 {
+		config.Logf("dashboards backfill: skipping %d widget group(s) whose project has no organization; the widget_groups table keeps their data", skipped.Count)
+	}
+
+	groups, err := lit.SelectNamed[widgetGroupWithOrg](
+		tx,
+		`SELECT wg.id, wg.project_id, wg.name, COALESCE(wg.description, '') AS description, wg.is_default, wg.created_by, wg.created_at, wg.updated_at, p.organization_id, p.name AS project_name
+		FROM widget_groups wg
+		JOIN projects p ON p.id = wg.project_id
+		WHERE p.organization_id IS NOT NULL
+		ORDER BY wg.project_id ASC, wg.is_default DESC, wg.name ASC`,
+		lit.P{},
+	)
+	if err != nil {
+		return 0, fmt.Errorf("failed to list widget groups: %w", err)
+	}
+	if len(groups) == 0 {
+		return 0, nil
+	}
+
+	positions := map[uuid.UUID]int{}
+	usedNames := map[int]map[string]bool{}
+	for _, group := range groups {
+		widgets, err := lit.SelectNamed[models.WidgetGroupWidget](
+			tx,
+			"SELECT id, widget_group_id, title, widget_type, config, position, created_at, updated_at FROM widget_group_widgets WHERE widget_group_id = :wg_id ORDER BY position ASC",
+			lit.P{"wg_id": group.Id},
+		)
+		if err != nil {
+			return 0, fmt.Errorf("failed to list widgets for group %d: %w", group.Id, err)
+		}
+
+		def := &models.DashboardDefinition{SchemaVersion: dashboardsvc.SchemaVersion}
+		widgetIds := make(map[int]string, len(widgets))
+		usedIds := make(map[string]bool, len(widgets))
+		for _, w := range widgets {
+			uid := dashboardsvc.NewWidgetId()
+			for usedIds[uid] {
+				uid = dashboardsvc.NewWidgetId()
+			}
+			usedIds[uid] = true
+			widgetIds[w.Id] = uid
+			widgetConfig := json.RawMessage(w.Config)
+			if len(widgetConfig) == 0 {
+				widgetConfig = json.RawMessage(`{}`)
+			}
+			def.Widgets = append(def.Widgets, models.DashboardWidget{
+				Id:         uid,
+				Title:      w.Title,
+				WidgetType: w.WidgetType,
+				Config:     widgetConfig,
+			})
+		}
+		definition, err := dashboardsvc.MarshalDefinition(def)
+		if err != nil {
+			return 0, fmt.Errorf("failed to marshal definition for group %d: %w", group.Id, err)
+		}
+
+		orgNames := usedNames[group.OrganizationId]
+		if orgNames == nil {
+			orgNames = map[string]bool{}
+			usedNames[group.OrganizationId] = orgNames
+		}
+		name := uniqueDashboardName(orgNames, group.Name, group.ProjectName)
+		orgNames[strings.ToLower(name)] = true
+
+		dashboard := &models.Dashboard{
+			OrganizationId: group.OrganizationId,
+			Name:           name,
+			Description:    group.Description,
+			Definition:     definition,
+			CreatedBy:      group.CreatedBy,
+			CreatedAt:      group.CreatedAt,
+			UpdatedAt:      group.UpdatedAt,
+		}
+		dashboardId, err := transactional.DashboardRepository.Create(tx, dashboard)
+		if err != nil {
+			return 0, fmt.Errorf("failed to create dashboard for group %d: %w", group.Id, err)
+		}
+
+		position := positions[group.ProjectId]
+		positions[group.ProjectId] = position + 1
+		if err := transactional.DashboardRepository.CreateAssignment(tx, &models.ProjectDashboard{
+			ProjectId:   group.ProjectId,
+			DashboardId: dashboardId,
+			Position:    position,
+			CreatedAt:   group.CreatedAt,
+		}); err != nil {
+			return 0, fmt.Errorf("failed to assign dashboard for group %d: %w", group.Id, err)
+		}
+
+		starred, err := lit.SelectNamed[models.StarredWidget](
+			tx,
+			`SELECT sw.id, sw.widget_id, sw.position, sw.col_span, sw.size, sw.created_at
+			FROM starred_widgets sw
+			JOIN widget_group_widgets wgw ON wgw.id = sw.widget_id
+			WHERE wgw.widget_group_id = :wg_id
+			ORDER BY sw.position ASC, sw.id ASC`,
+			lit.P{"wg_id": group.Id},
+		)
+		if err != nil {
+			return 0, fmt.Errorf("failed to list starred widgets for group %d: %w", group.Id, err)
+		}
+		for _, s := range starred {
+			uid := widgetIds[s.WidgetId]
+			if uid == "" {
+				continue
+			}
+			if err := transactional.DashboardRepository.CreateStarred(tx, &models.StarredDashboardWidget{
+				ProjectId:   group.ProjectId,
+				DashboardId: dashboardId,
+				WidgetId:    uid,
+				Position:    s.Position,
+				ColSpan:     s.ColSpan,
+				Size:        s.Size,
+				CreatedAt:   s.CreatedAt,
+			}); err != nil {
+				return 0, fmt.Errorf("failed to migrate starred widget %d for group %d: %w", s.Id, group.Id, err)
+			}
+		}
+	}
+
+	return len(groups), nil
+}
+
+func uniqueDashboardName(used map[string]bool, groupName string, projectName string) string {
+	base := dashboardsvc.TruncateName(groupName, maxDashboardNameRunes)
+	if !used[strings.ToLower(base)] {
+		return base
+	}
+	candidate := suffixDashboardName(base, projectName, 1)
+	if !used[strings.ToLower(candidate)] {
+		return candidate
+	}
+	for n := 2; ; n++ {
+		candidate = suffixDashboardName(base, projectName, n)
+		if !used[strings.ToLower(candidate)] {
+			return candidate
+		}
+	}
+}
+
+func suffixDashboardName(base string, projectName string, n int) string {
+	numeral := ""
+	if n > 1 {
+		numeral = fmt.Sprintf(" %d", n)
+	}
+	maxProjectRunes := maxDashboardNameRunes - 4 - utf8.RuneCountInString(numeral)
+	suffix := " (" + dashboardsvc.TruncateName(projectName, maxProjectRunes) + ")" + numeral
+	limit := maxDashboardNameRunes - utf8.RuneCountInString(suffix)
+	if limit < 1 {
+		limit = 1
+	}
+	return dashboardsvc.TruncateName(base, limit) + suffix
+}