Skip to content

Commit 18c2df2

Browse files
committed
Merge branch 'main' into oss
2 parents 6d96149 + 44ceb9a commit 18c2df2

13 files changed

Lines changed: 423 additions & 6 deletions

File tree

.github/workflows/_test.yml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
run-name: CI test (reusable)
2+
on:
3+
workflow_call:
4+
inputs:
5+
node-version:
6+
type: string
7+
default: '24.14.1'
8+
description: 'Node.js version to use.'
9+
10+
jobs:
11+
test:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
contents: read
15+
steps:
16+
- uses: actions/checkout@v4
17+
with:
18+
lfs: true
19+
- name: Set up Node.js
20+
uses: actions/setup-node@v4
21+
with:
22+
node-version: ${{ inputs.node-version }}
23+
cache: npm
24+
25+
- name: npm install
26+
run: npm ci
27+
- name: get non-redistributable assets
28+
run: npm run assets:download
29+
30+
- name: build application
31+
run: npm run build
32+
33+
- name: tests with coverage
34+
run: npm run test:coverage
35+
36+
- name: upload coverage to Codecov
37+
uses: codecov/codecov-action@v6
38+
with:
39+
files: coverage/lcov.info
40+
token: ${{ secrets.CODECOV_TOKEN }}
41+
fail_ci_if_error: true

.github/workflows/ci.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
run-name: CI (tests)
2+
on:
3+
pull_request:
4+
push:
5+
branches-ignore:
6+
- main
7+
- ops
8+
- ci-debug
9+
- dev
10+
11+
concurrency:
12+
group: ci-${{ github.ref }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
test:
17+
uses: ./.github/workflows/_test.yml
18+
permissions:
19+
contents: read
20+
secrets: inherit

.github/workflows/oss-image.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,19 @@ run-name: Build push container image to GH
22
on:
33
push:
44
branches:
5-
- oss
5+
- main
6+
tags:
7+
- '*'
68

79
jobs:
10+
test:
11+
uses: ./.github/workflows/_test.yml
12+
permissions:
13+
contents: read
14+
secrets: inherit
15+
816
build-and-push-to-ghcr:
17+
needs: test
918
runs-on: ubuntu-latest
1019
concurrency:
1120
group: ${{ github.ref_type == 'branch' && github.ref }}

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Reader
22

3+
[![codecov](https://codecov.io/gh/jina-ai/reader/branch/main/graph/badge.svg)](https://codecov.io/gh/jina-ai/reader)
4+
35
Your LLMs deserve better input.
46

57
Reader does two things:

src/api/crawler.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,9 @@ export class CrawlerHost extends RPCHost {
847847
if (!finalAutoSnapshot?.html) {
848848
throw new AssertionFailureError(`Unexpected non HTML content for ReaderLM: ${urlToCrawl}`);
849849
}
850+
if (finalAutoSnapshot?.status && crawlerOpts?.assertStatusCode && crawlerOpts.assertStatusCode !== finalAutoSnapshot.status) {
851+
throw new AssertionFailureError(`Expected status code ${crawlerOpts.assertStatusCode} but got ${finalAutoSnapshot.status}`);
852+
}
850853

851854
if (crawlerOpts?.instruction || crawlerOpts?.jsonSchema) {
852855
const jsonSchema = crawlerOpts.jsonSchema ? JSON.stringify(crawlerOpts.jsonSchema, undefined, 2) : undefined;
@@ -882,6 +885,9 @@ export class CrawlerHost extends RPCHost {
882885
this.logger.warn(`Failed to crawl blob URL ${snapshot.blobs[0].url} referenced from ${urlToCrawl}`, { err });
883886
}
884887
}
888+
if (snapshot?.status && crawlerOpts?.assertStatusCode && crawlerOpts.assertStatusCode !== snapshot.status) {
889+
throw new AssertionFailureError(`Expected status code ${crawlerOpts.assertStatusCode} but got ${snapshot.status}`);
890+
}
885891
yield snapshot;
886892
}
887893
} catch (err) {

src/dto/crawler-options.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,11 @@ class Viewport extends Coercible {
256256
in: 'header',
257257
schema: { type: 'string' }
258258
},
259+
'X-Assert-Status-Code': {
260+
description: 'Assert the HTTP status code of the crawled page. If the actual status code does not match the asserted one, the request is rejected with 422 Unprocessable Entity.',
261+
in: 'header',
262+
schema: { type: 'string' }
263+
},
259264
'X-Respond-Timing': {
260265
description: `Explicitly specify the respond timing. One of the following:\n\n` +
261266
`- html: directly return unrendered HTML\n` +
@@ -478,6 +483,9 @@ export class CrawlerOptions extends Coercible {
478483
@Prop()
479484
tokenBudget?: number;
480485

486+
@Prop()
487+
assertStatusCode?: number;
488+
481489
@Prop()
482490
viewport?: Viewport;
483491

@@ -679,6 +687,9 @@ export class CrawlerOptions extends Coercible {
679687
const maxTokens = ctx?.get('x-max-tokens');
680688
instance.maxTokens ??= parseInt(maxTokens || '') || undefined;
681689

690+
const assertStatusCode = ctx?.get('x-assert-status-code');
691+
instance.assertStatusCode ??= parseInt(assertStatusCode || '') || undefined;
692+
682693
const markdownChunking = ctx?.get('x-markdown-chunking');
683694
instance.markdownChunking ??= markdownChunking || undefined;
684695

src/services/markify.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -585,7 +585,7 @@ export class MarkifyService {
585585
this.tableStack = this.tableStack.slice(tableIndex + 1);
586586
const items: string[] = [];
587587
if (!markdown.startsWith('\n')) {
588-
items.push('\n');
588+
items.unshift('\n');
589589
}
590590
items.push(markdown);
591591
if (!markdown.endsWith('\n')) {
@@ -595,6 +595,10 @@ export class MarkifyService {
595595
}
596596

597597
protected processThead(element: Element): string {
598+
if (element.previousSibling && element.previousSibling.textContent) {
599+
return '\n' + this.processChildren(element);
600+
}
601+
598602
return this.processChildren(element);
599603
}
600604

src/services/puppeteer.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ export class PuppeteerControl extends AsyncService {
677677
'--disable-dev-shm-usage',
678678
'--disable-blink-features=AutomationControlled'
679679
];
680-
if (process.env.NODE_ENV === undefined) {
680+
if (process.env.NODE_ENV === undefined || process.env.NODE_ENV === 'test') {
681681
args.push('--no-sandbox', '--disable-setuid-sandbox');
682682
}
683683
this.browser = await puppeteer.launch({
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
/**
2+
* E2E tests for the assertStatusCode option using a real HTTP fixture server.
3+
*
4+
* The html-input path produces snapshots with no `status` field, so the
5+
* assertion in CrawlerHost.iterSnapshots is silently skipped there. To
6+
* exercise the assertion proper, these tests drive the crawler against a
7+
* local fixture server with `engine: 'curl'`.
8+
*
9+
* URL note: `assertNormalizedUrl` in src/services/misc.ts rejects bare IP
10+
* literals in non-public ranges unconditionally, so the fixture URL uses
11+
* the `localhost` hostname (server binds dual-stack).
12+
*/
13+
import { describe, it, before, after } from 'node:test';
14+
import assert from 'node:assert';
15+
import { getAgent } from '../helpers/client';
16+
import { startFixtureServer, FixtureServer } from '../helpers/fixture-server';
17+
18+
describe('assertStatusCode against real network fixture', () => {
19+
let fixture: FixtureServer;
20+
21+
before(async () => {
22+
fixture = await startFixtureServer();
23+
});
24+
25+
after(async () => {
26+
await fixture.close();
27+
});
28+
29+
async function crawlUrl(body: Record<string, unknown>, headers: Record<string, string> = {}) {
30+
let req = getAgent()
31+
.post('/')
32+
.set('Accept', 'application/json')
33+
.set('Content-Type', 'application/json');
34+
for (const [k, v] of Object.entries(headers)) {
35+
req = req.set(k, v);
36+
}
37+
return req.send(body);
38+
}
39+
40+
// ── matching status: request succeeds ──────────────────────────────────
41+
42+
it('200 fixture passes when assertStatusCode matches', async () => {
43+
const res = await crawlUrl({
44+
url: fixture.url('/status/200'),
45+
engine: 'curl',
46+
assertStatusCode: 200,
47+
});
48+
assert.strictEqual(res.status, 200);
49+
assert.match(res.body.data.content, /Status 200/);
50+
});
51+
52+
it('404 fixture passes when assertStatusCode: 404 matches the real status', async () => {
53+
const res = await crawlUrl({
54+
url: fixture.url('/status/404'),
55+
engine: 'curl',
56+
assertStatusCode: 404,
57+
});
58+
assert.strictEqual(res.status, 200);
59+
assert.match(res.body.data.content, /Status 404/);
60+
});
61+
62+
it('200 fixture without assertStatusCode still works (default behavior)', async () => {
63+
const res = await crawlUrl({
64+
url: fixture.url('/status/200'),
65+
engine: 'curl',
66+
});
67+
assert.strictEqual(res.status, 200);
68+
assert.match(res.body.data.content, /Status 200/);
69+
});
70+
71+
it('404 fixture without assertStatusCode does not throw (status check is opt-in)', async () => {
72+
const res = await crawlUrl({
73+
url: fixture.url('/status/404'),
74+
engine: 'curl',
75+
});
76+
assert.strictEqual(res.status, 200);
77+
assert.match(res.body.data.content, /Status 404/);
78+
});
79+
80+
// ── mismatching status: request rejected with AssertionFailureError ────
81+
82+
it('200 expected vs 404 actual throws AssertionFailureError (body form)', async () => {
83+
const res = await crawlUrl({
84+
url: fixture.url('/status/404'),
85+
engine: 'curl',
86+
assertStatusCode: 200,
87+
});
88+
assert.ok(res.status >= 400, `Expected error status, got ${res.status}`);
89+
assert.strictEqual(res.body.name, 'AssertionFailureError');
90+
assert.match(
91+
res.body.message || res.body.readableMessage,
92+
/Expected status code 200 but got 404/
93+
);
94+
});
95+
96+
it('200 expected vs 500 actual throws AssertionFailureError', async () => {
97+
const res = await crawlUrl({
98+
url: fixture.url('/status/500'),
99+
engine: 'curl',
100+
assertStatusCode: 200,
101+
});
102+
assert.ok(res.status >= 400);
103+
assert.strictEqual(res.body.name, 'AssertionFailureError');
104+
assert.match(
105+
res.body.message || res.body.readableMessage,
106+
/Expected status code 200 but got 500/
107+
);
108+
});
109+
110+
it('404 expected vs 200 actual throws AssertionFailureError', async () => {
111+
const res = await crawlUrl({
112+
url: fixture.url('/status/200'),
113+
engine: 'curl',
114+
assertStatusCode: 404,
115+
});
116+
assert.ok(res.status >= 400);
117+
assert.strictEqual(res.body.name, 'AssertionFailureError');
118+
assert.match(
119+
res.body.message || res.body.readableMessage,
120+
/Expected status code 404 but got 200/
121+
);
122+
});
123+
124+
// ── header form behaves the same ────────────────────────────────────────
125+
126+
it('X-Assert-Status-Code header: match passes', async () => {
127+
const res = await crawlUrl(
128+
{ url: fixture.url('/status/200'), engine: 'curl' },
129+
{ 'X-Assert-Status-Code': '200' }
130+
);
131+
assert.strictEqual(res.status, 200);
132+
});
133+
134+
it('X-Assert-Status-Code header: mismatch throws AssertionFailureError', async () => {
135+
const res = await crawlUrl(
136+
{ url: fixture.url('/status/500'), engine: 'curl' },
137+
{ 'X-Assert-Status-Code': '200' }
138+
);
139+
assert.ok(res.status >= 400);
140+
assert.strictEqual(res.body.name, 'AssertionFailureError');
141+
assert.match(
142+
res.body.message || res.body.readableMessage,
143+
/Expected status code 200 but got 500/
144+
);
145+
});
146+
147+
it('X-Assert-Status-Code: foo (non-numeric) is ignored — request succeeds despite 404', async () => {
148+
const res = await crawlUrl(
149+
{ url: fixture.url('/status/404'), engine: 'curl' },
150+
{ 'X-Assert-Status-Code': 'foo' }
151+
);
152+
assert.strictEqual(res.status, 200);
153+
assert.match(res.body.data.content, /Status 404/);
154+
});
155+
});

tests/e2e/error-handling.test.ts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,8 @@ describe('invalid option values', () => {
5454
assert.strictEqual(res.body.name, 'ParamValidationError');
5555
});
5656

57-
it('invalid engine value returns 400', async () => {
57+
it('invalid engine value omitted and returns 200', async () => {
5858
const res = await crawl({ engine: 'invalid-engine' });
59-
assert.strictEqual(res.status, 400);
60-
assert.strictEqual(res.body.name, 'ParamValidationError');
59+
assert.strictEqual(res.status, 200);
6160
});
6261
});

0 commit comments

Comments
 (0)