Skip to content

Commit e082c1f

Browse files
jhayniffyjhayniffy
authored
Merge pull request #264 from michaelvic123/feat/bull-board-dlq-monitoring
feat: Bull Board dashboard, DLQ monitoring, coverage gaps + CI fix
2 parents ab2661b + 128a109 commit e082c1f

14 files changed

Lines changed: 1266 additions & 298 deletions

‎.github/workflows/ci.yml‎

Lines changed: 39 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ permissions:
1414
jobs:
1515
# ── Frontend quality gate ─────────────────────────────────────────────────
1616
frontend:
17-
name: Frontend (lint → typecheck → build)
17+
name: Frontend (lint → typecheck → build → test)
1818
runs-on: ubuntu-latest
1919
defaults:
2020
run:
@@ -40,11 +40,13 @@ jobs:
4040
- name: Build
4141
run: npm run build
4242

43+
- name: Test
44+
run: npm test
45+
4346
# ── Soroban ABI golden-vector drift guard ────────────────────────────────
4447
golden-vectors:
4548
name: Soroban ABI golden vectors
4649
runs-on: ubuntu-latest
47-
# Run whenever contracts or backend builder code changes
4850
if: |
4951
github.event_name == 'push' ||
5052
contains(toJson(github.event.pull_request.changed_files), 'contracts/') ||
@@ -73,77 +75,37 @@ jobs:
7375
exit 1
7476
fi
7577
76-
# ── Smart contract ────────────────────────────────────────────────────────
78+
# ── Smart contract (Rust / Soroban) ──────────────────────────────────────
7779
contract:
7880
name: Contract (Rust / Soroban)
7981
runs-on: ubuntu-latest
8082
steps:
8183
- uses: actions/checkout@v4
82-
- uses: actions/setup-node@v4
83-
with:
84-
node-version: '20'
85-
- name: Determine package manager
86-
id: pkgmgr
87-
run: |
88-
if [ -f pnpm-lock.yaml ]; then
89-
echo "manager=pnpm" >> "$GITHUB_OUTPUT"
90-
elif [ -f package-lock.json ]; then
91-
echo "manager=npm" >> "$GITHUB_OUTPUT"
92-
elif [ -f yarn.lock ]; then
93-
echo "manager=yarn" >> "$GITHUB_OUTPUT"
94-
else
95-
echo "manager=npm" >> "$GITHUB_OUTPUT"
96-
fi
97-
- name: Setup pnpm when needed
98-
if: steps.pkgmgr.outputs.manager == 'pnpm'
99-
uses: pnpm/action-setup@v2
84+
85+
- name: Install Rust toolchain
86+
uses: dtolnay/rust-toolchain@stable
10087
with:
101-
version: 8
102-
- name: Cache node modules
88+
targets: wasm32-unknown-unknown
89+
90+
- name: Cache Rust build artifacts
10391
uses: actions/cache@v4
10492
with:
10593
path: |
106-
node_modules
107-
~/.pnpm-store
108-
key: ${{ runner.os }}-node-${{ steps.pkgmgr.outputs.manager }}-${{ hashFiles('**/package-lock.json', '**/pnpm-lock.yaml', '**/yarn.lock') }}
109-
restore-keys: ${{ runner.os }}-node-${{ steps.pkgmgr.outputs.manager }}-
110-
- name: Cache .next cache
111-
uses: actions/cache@v4
112-
with:
113-
path: .next/cache
114-
key: ${{ runner.os }}-next-cache-${{ hashFiles('**/package-lock.json', '**/pnpm-lock.yaml', '**/yarn.lock') }}
115-
restore-keys: ${{ runner.os }}-next-cache-
116-
- name: Install dependencies
117-
run: |
118-
if [ "${{ steps.pkgmgr.outputs.manager }}" = "pnpm" ]; then
119-
pnpm install --frozen-lockfile
120-
else
121-
npm ci
122-
fi
123-
- name: Lint (fail on warnings)
124-
run: |
125-
if [ "${{ steps.pkgmgr.outputs.manager }}" = "pnpm" ]; then
126-
pnpm eslint --max-warnings=0 .
127-
else
128-
npm run lint -- --max-warnings=0
129-
fi
130-
- name: TypeScript compile
131-
run: |
132-
if [ "${{ steps.pkgmgr.outputs.manager }}" = "pnpm" ]; then
133-
pnpm tsc --noEmit
134-
else
135-
npm run build --if-present -- --noEmit
136-
fi
137-
- name: Build
138-
run: |
139-
if [ "${{ steps.pkgmgr.outputs.manager }}" = "pnpm" ]; then
140-
pnpm build
141-
else
142-
npm run build
143-
fi
94+
~/.cargo/registry
95+
~/.cargo/git
96+
target/
97+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
98+
restore-keys: ${{ runner.os }}-cargo-
99+
100+
- name: Run contract tests
101+
run: cargo test --workspace --features testutils
102+
103+
- name: Build WASM (release)
104+
run: cargo build --release --target wasm32-unknown-unknown --workspace
144105

106+
# ── Backend unit tests ────────────────────────────────────────────────────
145107
unit-tests:
146-
name: Unit tests
108+
name: Backend unit tests
147109
runs-on: ubuntu-latest
148110
defaults:
149111
run:
@@ -157,7 +119,6 @@ jobs:
157119
steps:
158120
- uses: actions/checkout@v4
159121

160-
# Install Redis directly on the runner — avoids Docker Hub rate limits entirely
161122
- name: Start Redis
162123
run: |
163124
sudo apt-get update -qq
@@ -168,59 +129,47 @@ jobs:
168129
- uses: actions/setup-node@v4
169130
with:
170131
node-version: '20'
132+
171133
- name: Install dependencies
172134
run: npm ci
135+
173136
- name: Run unit tests
174137
run: npm test
175138

139+
# ── Playwright E2E tests ──────────────────────────────────────────────────
176140
e2e-tests:
177141
name: Playwright E2E tests
178142
runs-on: ubuntu-latest
179143
needs: frontend
144+
defaults:
145+
run:
146+
working-directory: frontend
180147
steps:
181148
- uses: actions/checkout@v4
149+
182150
- uses: actions/setup-node@v4
183151
with:
184152
node-version: '20'
153+
185154
- name: Install dependencies
186155
run: npm ci
156+
187157
- name: Install Playwright browsers
188158
run: npx playwright install --with-deps
159+
189160
- name: Run Playwright tests
190161
run: npx playwright test --reporter=html
191162
continue-on-error: true
163+
192164
- name: Upload Playwright artifacts on failure
193165
if: failure()
194-
uses: actions/upload-artifact@v3
166+
uses: actions/upload-artifact@v4
195167
with:
196168
name: playwright-failure-${{ github.run_id }}
197169
path: |
198-
feat/accessibility-audit
199-
sbom-backend.json
200-
sbom-frontend.json
201-
retention-days: 90
202-
203-
# ── Frontend ──────────────────────────────────────────────────────────────
204-
frontend:
205-
name: Frontend (Next.js / TypeScript)
206-
runs-on: ubuntu-latest
207-
defaults:
208-
run:
209-
working-directory: frontend
210-
steps:
211-
- uses: actions/checkout@v4
212-
213-
- uses: actions/setup-node@v4
214-
with:
215-
node-version: 22
216-
cache: npm
217-
cache-dependency-path: frontend/package-lock.json
218-
219-
- run: npm ci
220-
- run: npm run lint
221-
- run: npm run check-docs
222-
- run: npm run build
223-
- run: npm test
170+
test-results/
171+
playwright-report/
172+
retention-days: 14
224173

225174
# ── Accessibility (axe) ───────────────────────────────────────────────────
226175
accessibility:
@@ -255,9 +204,3 @@ jobs:
255204
name: axe-report-${{ github.sha }}
256205
path: frontend/playwright-report/
257206
retention-days: 14
258-
259-
test-results
260-
playwright-report
261-
traces
262-
.playwright/traces
263-

‎.github/workflows/coverage.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Coverage
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
11+
permissions:
12+
contents: read
13+
14+
jobs:
15+
contract-coverage:
16+
name: Contract coverage (≥ 95%)
17+
runs-on: ubuntu-latest
18+
steps:
19+
- uses: actions/checkout@v4
20+
21+
- name: Install Rust toolchain
22+
uses: dtolnay/rust-toolchain@stable
23+
with:
24+
components: llvm-tools-preview
25+
26+
- name: Cache Rust build artifacts
27+
uses: actions/cache@v4
28+
with:
29+
path: |
30+
~/.cargo/registry
31+
~/.cargo/git
32+
target/
33+
key: ${{ runner.os }}-cargo-cov-${{ hashFiles('**/Cargo.lock') }}
34+
restore-keys: ${{ runner.os }}-cargo-cov-
35+
36+
- name: Install cargo-llvm-cov
37+
uses: taiki-e/install-action@cargo-llvm-cov
38+
39+
- name: Run coverage
40+
run: |
41+
cargo llvm-cov \
42+
--workspace \
43+
--features testutils \
44+
--lcov \
45+
--output-path lcov.info
46+
47+
- name: Enforce 95% line coverage threshold
48+
run: |
49+
cargo llvm-cov \
50+
--workspace \
51+
--features testutils \
52+
--summary-only 2>&1 | tee coverage-summary.txt
53+
54+
LINE_COV=$(grep -oP 'Lines\s+\K[\d.]+(?=%)' coverage-summary.txt | head -1)
55+
echo "Line coverage: ${LINE_COV}%"
56+
57+
# Use awk for float comparison (bash can't do floats)
58+
PASS=$(awk -v cov="$LINE_COV" 'BEGIN { print (cov >= 95.0) ? "yes" : "no" }')
59+
if [ "$PASS" != "yes" ]; then
60+
echo "::error::Coverage ${LINE_COV}% is below the required 95% threshold."
61+
exit 1
62+
fi
63+
echo "Coverage check passed: ${LINE_COV}%"
64+
65+
- name: Upload coverage report
66+
uses: actions/upload-artifact@v4
67+
with:
68+
name: lcov-${{ github.sha }}
69+
path: lcov.info
70+
retention-days: 30

‎backend/docs/observability.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,48 @@ this.winston.log(level, message, { ...fields, traceId, spanId });
129129
Similarly, `MetricsService.recordHttpRequest` / `recordRpcCall` map directly
130130
to OTel `Meter` histogram/counter calls — swap the prom-client calls for OTel
131131
Meter API calls when you're ready to migrate.
132+
133+
## Queue Dashboard — `/admin/queues`
134+
135+
Bull Board is mounted at `/admin/queues`. It requires a valid admin JWT in the
136+
`Authorization: Bearer <token>` header. No token or a non-admin token returns 401/403.
137+
138+
## Dead-Letter Queue (DLQ)
139+
140+
Jobs that fail `DLQ_MAX_ATTEMPTS` (5) times are moved to BullMQ's **failed** set.
141+
Two metrics track this:
142+
143+
| Metric | Type | Labels | Description |
144+
|---|---|---|---|
145+
| `bullmq_dlq_depth` | Gauge | `queue` | Current failed-job count per queue |
146+
| `bullmq_dlq_jobs_total` | Counter | `queue`, `job_name`, `failure_reason` | Cumulative jobs exhausted |
147+
148+
Alert `DlqDepthHigh` fires when `bullmq_dlq_depth > 10` for 5 minutes.
149+
The alert annotation includes the queue name and links to the replay endpoint.
150+
151+
### Manual Job Replay
152+
153+
1. Open Bull Board at `https://<host>/admin/queues` (admin JWT required) and
154+
identify the failed job id from the UI.
155+
2. Or query the API:
156+
```
157+
GET /api/admin/queues # via Bull Board UI
158+
```
159+
3. Replay a single job:
160+
```
161+
POST /api/admin/queues/:queue/jobs/:jobId/retry
162+
Authorization: Bearer <admin-jwt>
163+
```
164+
The job is moved back to `waiting` and retried from scratch.
165+
An audit row is written with actor, queue, and jobId.
166+
4. To bulk-replay all failed jobs on a queue, use the Bull Board UI
167+
"Retry all" button — it is equivalent to calling retry on each job.
168+
169+
### Queues monitored
170+
171+
| Queue | Max attempts | Purpose |
172+
|---|---|---|
173+
| `indexer` | 5 | Soroban ledger event indexing |
174+
| `notifications` | 5 | Claim-finalized email/Discord/Telegram |
175+
| `claim-events` | 5 | Raw claim event DB writes |
176+
| `reindex` | 5 | Admin-triggered ledger reindex |

‎backend/docs/prometheus-alerts.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,19 @@ groups:
6262
description: >
6363
RPC method {{ $labels.rpc_method }} p95 latency exceeded 10 s.
6464
Soroban network may be congested.
65+
66+
# ── Dead-letter queue depth alert ─────────────────────────────────────
67+
- alert: DlqDepthHigh
68+
expr: |
69+
bullmq_dlq_depth{app="niffyinsure-api"} > 10
70+
for: 5m
71+
labels:
72+
severity: critical
73+
team: backend
74+
annotations:
75+
summary: "Dead-letter queue depth above threshold on {{ $labels.queue }}"
76+
description: >
77+
Queue "{{ $labels.queue }}" has more than 10 failed jobs sitting in
78+
the dead-letter set. Check bullmq_dlq_jobs_total for job_name and
79+
failure_reason labels to triage. Replay via:
80+
POST /api/admin/queues/{{ $labels.queue }}/jobs/:jobId/retry

‎backend/jest.config.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
module.exports = {
33
testEnvironment: "node",
44
roots: ["<rootDir>/src", "<rootDir>/tests"],
5-
testMatch: ["**/__tests__/**/*.test.ts", "**/*.test.ts"],
5+
testMatch: ["**/__tests__/**/*.test.ts", "**/*.test.ts", "**/*.spec.ts"],
66
transform: {
77
"^.+\\.tsx?$": ["ts-jest", { tsconfig: "tsconfig.test.json" }],
88
},

0 commit comments

Comments
 (0)