Skip to content

Commit 01da4e1

Browse files
ShramkowebShramkoweb
committed
feat: block chatgpt agent via signature-agent
ChatGPT agent browses with a plain browser User-Agent and identifies itself only via the Web Bot Auth Signature-Agent header, so the UA token matcher cannot see it. Add a signature-agent block-list checked in middleware as a second channel; the robots.txt-allowed OpenAI fetchers (ChatGPT-User, OAI-SearchBot) keep winning via a UA override. Matching the unverified header without validating the RFC 9421 signature is sound for a block-list: forging it only gets a request blocked. Also add UA tokens atlassian-bot (training) and google-agent (agent) to tokens.ts and robots.txt, and extract the shared 403 into policyResponse(). Middleware-level tests pin the header wiring in both directions.
1 parent 22e230f commit 01da4e1

7 files changed

Lines changed: 197 additions & 20 deletions

File tree

‎__tests__/middleware.test.ts‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,22 @@ import { NextRequest } from 'next/server';
55

66
import { middleware } from '../middleware';
77

8-
function makeRequest(userAgent: string | null, path = '/'): NextRequest {
9-
const headers = new Headers();
8+
function makeRequest(
9+
userAgent: string | null,
10+
extraHeaders: Record<string, string> = {},
11+
): NextRequest {
12+
const headers = new Headers(extraHeaders);
1013
if (userAgent !== null) headers.set('user-agent', userAgent);
11-
return new NextRequest(new URL(path, 'http://localhost'), { headers });
14+
return new NextRequest(new URL('/', 'http://localhost'), { headers });
1215
}
1316

1417
const BLOCK_BODY =
1518
'Automated AI training and scraping crawlers are not permitted on this site.\n';
1619

20+
const DESKTOP_UA =
21+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
22+
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
23+
1724
describe('middleware', () => {
1825
it('returns 403 with policy body for a blocked UA', async () => {
1926
const res = middleware(makeRequest('GPTBot/1.2'));
@@ -25,12 +32,7 @@ describe('middleware', () => {
2532
});
2633

2734
it('passes through (no 403) for a typical desktop UA', async () => {
28-
const res = middleware(
29-
makeRequest(
30-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
31-
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
32-
),
33-
);
35+
const res = middleware(makeRequest(DESKTOP_UA));
3436
expect(res.status).not.toBe(403);
3537
expect(await res.text()).not.toBe(BLOCK_BODY);
3638
});
@@ -39,4 +41,25 @@ describe('middleware', () => {
3941
const res = middleware(makeRequest(null));
4042
expect(res.status).not.toBe(403);
4143
});
44+
45+
// ChatGPT agent browses with a plain browser UA; only the Signature-Agent
46+
// header identifies it (see lib/bot-blocklist/signature-agent.ts).
47+
it('returns 403 for a browser UA signed by a blocked Signature-Agent', async () => {
48+
const res = middleware(
49+
makeRequest(DESKTOP_UA, { 'Signature-Agent': '"https://chatgpt.com"' }),
50+
);
51+
expect(res.status).toBe(403);
52+
expect(await res.text()).toBe(BLOCK_BODY);
53+
});
54+
55+
it('passes through a signed request from a robots.txt-allowed fetcher UA', async () => {
56+
const res = middleware(
57+
makeRequest(
58+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ' +
59+
'ChatGPT-User/1.0; +https://openai.com/bot',
60+
{ 'Signature-Agent': '"https://chatgpt.com"' },
61+
),
62+
);
63+
expect(res.status).not.toBe(403);
64+
});
4265
});

‎lib/bot-blocklist/matcher.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,18 @@ describe('isBlockedUserAgent', () => {
6565
});
6666
});
6767

68+
it('blocks atlassian-bot (Rovo crawler)', () => {
69+
expect(
70+
isBlockedUserAgent(
71+
'Mozilla/5.0 (compatible; atlassian-bot/1.0; +https://www.atlassian.com)',
72+
),
73+
).toEqual({
74+
blocked: true,
75+
token: 'atlassian-bot',
76+
group: 'training',
77+
});
78+
});
79+
6880
it('is case-insensitive (GPTBOT uppercase)', () => {
6981
expect(isBlockedUserAgent('GPTBOT/2.0')).toEqual({
7082
blocked: true,
@@ -98,6 +110,29 @@ describe('isBlockedUserAgent', () => {
98110
group: 'agent',
99111
});
100112
});
113+
114+
it('blocks Google-Agent (documented desktop UA)', () => {
115+
const ua =
116+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like ' +
117+
'Gecko; compatible; Google-Agent; +https://developers.google.com/' +
118+
'crawling/docs/crawlers-fetchers/google-agent) Chrome/125.0.0.0 ' +
119+
'Safari/537.36';
120+
expect(isBlockedUserAgent(ua)).toEqual({
121+
blocked: true,
122+
token: 'google-agent',
123+
group: 'agent',
124+
});
125+
});
126+
127+
it('blocks the legacy GoogleAgent-Mariner UA shape', () => {
128+
expect(
129+
isBlockedUserAgent('Mozilla/5.0 (compatible; GoogleAgent-Mariner)'),
130+
).toEqual({
131+
blocked: true,
132+
token: 'googleagent-mariner',
133+
group: 'agent',
134+
});
135+
});
101136
});
102137

103138
// robots.txt explicitly allows OpenAI's user-triggered + search bots. Every
@@ -141,6 +176,10 @@ describe('hygiene — short/generic tokens must not collide with real UAs', () =
141176
{ ua: 'Mozilla/5.0 YakDocReader/1.0', mentions: 'yak' },
142177
{ ua: 'Mozilla/5.0 DevinDocViewer/1.0', mentions: 'devin' },
143178
{ ua: 'Mozilla/5.0 openaitest/1.0 — fictional', mentions: 'openai' },
179+
{
180+
ua: 'Googlebot/2.1 (+http://www.google.com/bot.html)',
181+
mentions: 'google-agent',
182+
},
144183
];
145184

146185
for (const { ua, mentions } of cases) {

‎lib/bot-blocklist/signature-agent.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import { isBlockedSignatureAgent } from '@/lib/bot-blocklist/signature-agent';
2+
3+
// ChatGPT agent presents a plain browser UA — the header is the only signal.
4+
const BROWSER_UA =
5+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
6+
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
7+
8+
describe('isBlockedSignatureAgent', () => {
9+
it('blocks ChatGPT agent by its quoted sf-string header value', () => {
10+
expect(isBlockedSignatureAgent('"https://chatgpt.com"', BROWSER_UA)).toBe(
11+
true,
12+
);
13+
});
14+
15+
it('blocks unquoted and trailing-slash variants', () => {
16+
expect(isBlockedSignatureAgent('https://chatgpt.com', BROWSER_UA)).toBe(
17+
true,
18+
);
19+
expect(isBlockedSignatureAgent('"https://chatgpt.com/"', BROWSER_UA)).toBe(
20+
true,
21+
);
22+
});
23+
24+
it('is case-insensitive', () => {
25+
expect(isBlockedSignatureAgent('"HTTPS://ChatGPT.com"', BROWSER_UA)).toBe(
26+
true,
27+
);
28+
});
29+
30+
it('blocks when the UA is missing but the signature matches', () => {
31+
expect(isBlockedSignatureAgent('"https://chatgpt.com"', null)).toBe(true);
32+
});
33+
34+
it('ignores requests without the header', () => {
35+
expect(isBlockedSignatureAgent(null, BROWSER_UA)).toBe(false);
36+
expect(isBlockedSignatureAgent('', BROWSER_UA)).toBe(false);
37+
});
38+
39+
it('ignores signature agents that are not block-listed', () => {
40+
expect(
41+
isBlockedSignatureAgent('"https://www.browserbase.com"', BROWSER_UA),
42+
).toBe(false);
43+
});
44+
45+
it('lets robots.txt-allowed OpenAI fetchers through even when signed', () => {
46+
const ua =
47+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ' +
48+
'ChatGPT-User/1.0; +https://openai.com/bot';
49+
expect(isBlockedSignatureAgent('"https://chatgpt.com"', ua)).toBe(false);
50+
});
51+
});

‎lib/bot-blocklist/signature-agent.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// lib/bot-blocklist/signature-agent.ts
2+
//
3+
// Agentic browsers increasingly identify via Web Bot Auth (RFC 9421) instead
4+
// of a distinctive User-Agent token: ChatGPT agent browses with a plain
5+
// browser UA and is only recognizable by its `Signature-Agent` header
6+
// (https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting).
7+
// This complements matcher.ts, which cannot see such clients.
8+
//
9+
// Matching the unverified header value (without validating the RFC 9421
10+
// signature) is sound for a block-list: forging the header only gets a
11+
// request blocked, so there is no incentive to spoof it.
12+
13+
const BLOCKED_SIGNATURE_AGENTS: ReadonlySet<string> = new Set([
14+
// ChatGPT agent (ex-Operator) — blocked in public/robots.txt.
15+
'https://chatgpt.com',
16+
]);
17+
18+
// OpenAI fetchers allowed in public/robots.txt self-identify in the UA. If
19+
// they ever start signing with the agent's origin, the UA token must keep
20+
// winning.
21+
const ALLOWED_UA_OVERRIDES: readonly string[] = [
22+
'chatgpt-user',
23+
'oai-searchbot',
24+
];
25+
26+
export function isBlockedSignatureAgent(
27+
signatureAgent: string | null,
28+
userAgent: string | null,
29+
): boolean {
30+
if (!signatureAgent) return false;
31+
if (!BLOCKED_SIGNATURE_AGENTS.has(normalizeOrigin(signatureAgent))) {
32+
return false;
33+
}
34+
35+
const ua = userAgent?.toLowerCase() ?? '';
36+
return !ALLOWED_UA_OVERRIDES.some((token) => ua.includes(token));
37+
}
38+
39+
// The header value is an RFC 8941 sf-string, e.g. `"https://chatgpt.com"`.
40+
function normalizeOrigin(value: string): string {
41+
return value
42+
.trim()
43+
.replace(/^"+|"+$/g, '')
44+
.replace(/\/+$/, '')
45+
.toLowerCase();
46+
}

‎lib/bot-blocklist/tokens.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export const TRAINING_CRAWLER_TOKENS: readonly BlockToken[] = [
2323
{ value: 'amazon-kendra', kind: 'substring' },
2424
{ value: 'anthropic-ai', kind: 'substring' },
2525
{ value: 'applebot-extended', kind: 'substring' },
26+
{ value: 'atlassian-bot', kind: 'substring' },
2627
{ value: 'bedrockbot', kind: 'substring' },
2728
{ value: 'brightbot', kind: 'substring' },
2829
{ value: 'bytespider', kind: 'substring' },
@@ -116,6 +117,9 @@ export const AUTONOMOUS_AGENT_TOKENS: readonly BlockToken[] = [
116117
{ value: 'channel3bot', kind: 'substring' },
117118
{ value: 'chatgpt agent', kind: 'substring' },
118119
{ value: 'devin', kind: 'word-boundary' },
120+
// Documented token for agents on Google infrastructure (Project Mariner
121+
// et al.); `googleagent-mariner` kept for the legacy UA shape.
122+
{ value: 'google-agent', kind: 'substring' },
119123
{ value: 'googleagent-mariner', kind: 'substring' },
120124
{ value: 'novaact', kind: 'substring' },
121125
{ value: 'operator', kind: 'word-boundary' },
@@ -138,6 +142,7 @@ export const BLOCKED_BOT_GROUPS: Readonly<Record<string, BlockGroup>> = {
138142
'amazon-kendra': 'training',
139143
'anthropic-ai': 'training',
140144
'applebot-extended': 'training',
145+
'atlassian-bot': 'training',
141146
bedrockbot: 'training',
142147
brightbot: 'training',
143148
bytespider: 'training',
@@ -224,6 +229,7 @@ export const BLOCKED_BOT_GROUPS: Readonly<Record<string, BlockGroup>> = {
224229
channel3bot: 'agent',
225230
'chatgpt agent': 'agent',
226231
devin: 'agent',
232+
'google-agent': 'agent',
227233
'googleagent-mariner': 'agent',
228234
novaact: 'agent',
229235
operator: 'agent',

‎middleware.ts‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,33 @@
77
import { NextResponse, type NextRequest } from 'next/server';
88

99
import { isBlockedUserAgent } from '@/lib/bot-blocklist/matcher';
10+
import { isBlockedSignatureAgent } from '@/lib/bot-blocklist/signature-agent';
1011

1112
const POLICY_BODY =
1213
'Automated AI training and scraping crawlers are not permitted on this site.\n';
1314

15+
function policyResponse(): NextResponse {
16+
return new NextResponse(POLICY_BODY, {
17+
status: 403,
18+
headers: {
19+
'Content-Type': 'text/plain; charset=utf-8',
20+
'X-Robots-Tag': 'noindex, noai, noimageai',
21+
'Cache-Control': 'no-store',
22+
},
23+
});
24+
}
25+
1426
export function middleware(request: NextRequest) {
1527
const ua = request.headers.get('user-agent');
16-
const result = isBlockedUserAgent(ua);
17-
18-
if (result.blocked) {
19-
return new NextResponse(POLICY_BODY, {
20-
status: 403,
21-
headers: {
22-
'Content-Type': 'text/plain; charset=utf-8',
23-
'X-Robots-Tag': 'noindex, noai, noimageai',
24-
'Cache-Control': 'no-store',
25-
},
26-
});
28+
29+
if (isBlockedUserAgent(ua).blocked) {
30+
return policyResponse();
31+
}
32+
33+
// Agentic browsers with plain browser UAs (ChatGPT agent) identify only
34+
// via Web Bot Auth — see lib/bot-blocklist/signature-agent.ts.
35+
if (isBlockedSignatureAgent(request.headers.get('signature-agent'), ua)) {
36+
return policyResponse();
2737
}
2838

2939
return NextResponse.next();

‎public/robots.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ User-agent: Amazonbot
1212
User-agent: amazon-kendra
1313
User-agent: anthropic-ai
1414
User-agent: Applebot-Extended
15+
User-agent: atlassian-bot
1516
User-agent: bedrockbot
1617
User-agent: Brightbot
1718
User-agent: Bytespider
@@ -105,6 +106,7 @@ User-agent: BuddyBot
105106
User-agent: Channel3Bot
106107
User-agent: ChatGPT Agent
107108
User-agent: Devin
109+
User-agent: Google-Agent
108110
User-agent: GoogleAgent-Mariner
109111
User-agent: NovaAct
110112
User-agent: Operator

0 commit comments

Comments
 (0)