Skip to content

Commit a983092

Browse files
lfoppianolfoppiano
committed
fix: switch to tutorial, keep grobid.readthedocs as the master for AI
Signed-off-by: Luca Foppiano <luca@foppiano.org>
1 parent 02d8f89 commit a983092

10 files changed

Lines changed: 1120 additions & 98 deletions

File tree

‎.github/workflows/deploy.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,36 @@ jobs:
3232
- name: Install dependencies
3333
run: npm ci
3434

35+
# Restore the upstream-docs cache from the most recent run.
36+
# The cache is keyed by github.run_id so each run writes a unique key,
37+
# and restore-keys always restores the most recent prior cache. Combined
38+
# with the SHA-keyed cache check inside fetch-upstream-docs.cjs, repeat
39+
# builds skip downloads entirely unless upstream actually changed.
40+
- name: Restore upstream docs cache
41+
uses: actions/cache@v4
42+
with:
43+
path: .cache/upstream-grobid-doc
44+
key: upstream-grobid-doc-${{ github.run_id }}
45+
restore-keys: |
46+
upstream-grobid-doc-
47+
48+
# Fetch reference docs from kermitt2/grobid before building. The script
49+
# fails gracefully (exits 0 with a warning) on network errors, so a
50+
# GitHub API outage cannot block deploys. GITHUB_TOKEN is auto-provided
51+
# by Actions and raises the API rate limit from 60/hr to 5000/hr.
52+
- name: Fetch upstream reference docs
53+
run: node scripts/fetch-upstream-docs.cjs
54+
env:
55+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
56+
57+
# Regenerate the LLM index against the freshly fetched cache. We can't
58+
# rely on `npm run build`'s `prebuild` hook because that also runs
59+
# generate-docker-configs.cjs which needs a sibling ../grobid-home/
60+
# checkout that doesn't exist in CI. Calling the LLM index generator
61+
# directly is the surgical option.
62+
- name: Regenerate LLM doc index
63+
run: node scripts/generate-llm-docs-index.cjs
64+
3565
# Call docusaurus directly rather than `npm run build`: the `prebuild`
3666
# hook tries to regenerate src/generated/dockerBaseConfigs.ts from a
3767
# sibling ../grobid-home/ checkout that only exists in local dev

‎docs/intro.md‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
---
2-
title: "GROBID Documentation"
3-
description: "Extract structured data from scholarly PDFs with the fastest safe path to a working GROBID setup"
2+
title: "GROBID Tutorial"
3+
description: "The fast, beginner-friendly path to your first GROBID extraction — a hands-on tutorial that complements the GROBID reference docs"
44
sidebar_position: 1
55
slug: /
66
---
77

8-
# GROBID Documentation
8+
# GROBID Tutorial
9+
10+
:::tip This is the tutorial site — not the reference docs
11+
You're on the **GROBID Tutorial** — a hands-on, beginner-friendly on-ramp that gets you from zero to your first successful PDF extraction as quickly as possible.
12+
13+
For the full reference documentation (every endpoint, every config knob, every training parameter), see **[grobid.readthedocs.io](https://grobid.readthedocs.io/)**.
14+
15+
Not sure which you need? If you've never run GROBID before, start here. If you already run GROBID and are looking up a specific flag or endpoint, you probably want the reference docs.
16+
:::
917

1018
GROBID extracts structured data from scholarly PDFs: titles, authors, affiliations, references, citations, section structure, full text, and TEI XML.
1119

@@ -98,8 +106,8 @@ This documentation is organized to get you past those blockers early.
98106

99107
### Reference
100108

101-
For detailed reference documentation (API endpoints, configuration parameters, TEI encoding, training guidelines), see the [GROBID reference docs](https://github.com/kermitt2/grobid/blob/master/doc/).
109+
For detailed reference documentation (API endpoints, configuration parameters, TEI encoding, training guidelines), see **[grobid.readthedocs.io](https://grobid.readthedocs.io/)**.
102110

103111
:::info About this tutorial
104-
This tutorial site provides practical, task-oriented guides. For detailed reference documentation on API endpoints, configuration parameters, TEI encoding, and training annotation rules, see the [GROBID reference docs](https://github.com/kermitt2/grobid/tree/master/doc).
112+
This tutorial site provides practical, task-oriented guides designed to get newcomers productive fast. For the full reference documentation — API endpoints, configuration parameters, TEI encoding, and training annotation rules — see **[grobid.readthedocs.io](https://grobid.readthedocs.io/)**.
105113
:::

‎docusaurus.config.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ const giscusThemeDark = process.env.DOCS_GISCUS_THEME_DARK ?? 'noborder_dark';
2020
const giscusThemeLight = process.env.DOCS_GISCUS_THEME_LIGHT ?? 'light';
2121

2222
const config: Config = {
23-
title: 'GROBID Documentation',
24-
tagline: 'Machine learning library for extracting structured data from scholarly PDFs',
23+
title: 'GROBID Tutorial',
24+
tagline: 'The fast, beginner-friendly path to your first GROBID extraction',
2525
favicon: 'img/favicon.ico',
2626

2727
future: {
@@ -141,7 +141,7 @@ const config: Config = {
141141
},
142142
},
143143
navbar: {
144-
title: 'GROBID',
144+
title: 'GROBID Tutorial',
145145
logo: {
146146
alt: 'GROBID Logo',
147147
src: 'img/logo.svg',
@@ -155,7 +155,10 @@ const config: Config = {
155155
label: 'Tutorial',
156156
},
157157
{
158-
href: `${githubRepoUrl}/tree/master/doc`,
158+
// Reference docs live at the upstream-hosted ReadTheDocs site, not
159+
// in this tutorial repo. Hardcoded because `githubRepoUrl` points
160+
// at grobidOrg/grobid-tutorial after the deployment config change.
161+
href: 'https://grobid.readthedocs.io/',
159162
position: 'left',
160163
label: 'Reference Docs',
161164
},

‎plugins/llms-txt-plugin.js‎

Lines changed: 176 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,93 +6,203 @@ module.exports = function llmsTxtPlugin(context) {
66
return {
77
name: 'llms-txt-plugin',
88

9-
async postBuild({outDir, content}) {
9+
async postBuild({outDir}) {
1010
const docsDir = path.join(context.siteDir, 'docs');
11+
const upstreamCacheDir = path.join(context.siteDir, '.cache', 'upstream-grobid-doc');
1112
const baseUrl = context.siteConfig.baseUrl;
1213
const siteUrl = context.siteConfig.url;
1314

1415
const pages = [];
15-
collectDocs(docsDir, docsDir, pages);
16-
pages.sort((a, b) => a.path.localeCompare(b.path));
1716

18-
const llmsTxt = generateLlmsTxt(pages, siteUrl, baseUrl);
17+
// Tutorial content from this repo's docs/.
18+
collectTutorialDocs(docsDir, docsDir, pages, siteUrl, baseUrl);
19+
20+
// Reference content from the upstream cache, populated by
21+
// scripts/fetch-upstream-docs.cjs. May be absent on offline builds.
22+
const freshness = collectReferenceDocs(upstreamCacheDir, pages);
23+
24+
// Stable order: tutorial pages first (sorted by path), then reference.
25+
// This keeps the llms.txt narrative reading "tutorial first, reference
26+
// second" which mirrors how the corpus should be consumed.
27+
pages.sort((a, b) => {
28+
if (a.category !== b.category) {
29+
return a.category === 'tutorial' ? -1 : 1;
30+
}
31+
return a.path.localeCompare(b.path);
32+
});
33+
34+
const llmsTxt = generateLlmsTxt(pages, freshness);
1935
fs.writeFileSync(path.join(outDir, 'llms.txt'), llmsTxt, 'utf-8');
2036

21-
const llmsFullTxt = generateLlmsFullTxt(pages);
37+
const llmsFullTxt = generateLlmsFullTxt(pages, freshness);
2238
fs.writeFileSync(path.join(outDir, 'llms-full.txt'), llmsFullTxt, 'utf-8');
2339

24-
console.log(`[llms-txt] Generated llms.txt (${pages.length} pages) and llms-full.txt`);
40+
const tutorialCount = pages.filter((p) => p.category === 'tutorial').length;
41+
const referenceCount = pages.filter((p) => p.category === 'reference').length;
42+
console.log(
43+
`[llms-txt] Generated llms.txt and llms-full.txt ` +
44+
`(${tutorialCount} tutorial + ${referenceCount} reference = ${pages.length} pages)`,
45+
);
2546
},
2647
};
2748
};
2849

29-
function collectDocs(dir, rootDir, pages) {
50+
function collectTutorialDocs(dir, rootDir, pages, siteUrl, baseUrl) {
3051
const entries = fs.readdirSync(dir, {withFileTypes: true});
3152
for (const entry of entries) {
3253
const fullPath = path.join(dir, entry.name);
3354
if (entry.isDirectory()) {
34-
collectDocs(fullPath, rootDir, pages);
35-
} else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
36-
const raw = fs.readFileSync(fullPath, 'utf-8');
37-
const {data: frontmatter, content} = matter(raw);
38-
const relPath = path.relative(rootDir, fullPath).replace(/\\/g, '/');
39-
const slug = frontmatter.slug ||
40-
'/' + relPath
41-
.replace(/\/index\.md(x)?$/, '')
42-
.replace(/\.mdx?$/, '');
43-
44-
pages.push({
45-
path: slug,
46-
title: frontmatter.title || path.basename(fullPath, path.extname(fullPath)),
47-
description: frontmatter.description || '',
48-
content: content.trim(),
49-
});
55+
collectTutorialDocs(fullPath, rootDir, pages, siteUrl, baseUrl);
56+
continue;
5057
}
58+
if (!entry.name.endsWith('.md') && !entry.name.endsWith('.mdx')) continue;
59+
60+
const raw = fs.readFileSync(fullPath, 'utf-8');
61+
const {data: frontmatter, content} = matter(raw);
62+
const relPath = path.relative(rootDir, fullPath).replace(/\\/g, '/');
63+
const slug = frontmatter.slug ||
64+
'/' + relPath
65+
.replace(/\/index\.md(x)?$/, '')
66+
.replace(/\.mdx?$/, '');
67+
68+
pages.push({
69+
category: 'tutorial',
70+
path: slug,
71+
title: frontmatter.title || path.basename(fullPath, path.extname(fullPath)),
72+
description: frontmatter.description || '',
73+
content: content.trim(),
74+
url: `${siteUrl}${baseUrl}${slug.replace(/^\//, '')}`,
75+
});
76+
}
77+
}
78+
79+
function collectReferenceDocs(upstreamCacheDir, pages) {
80+
const manifestPath = path.join(upstreamCacheDir, 'manifest.json');
81+
if (!fs.existsSync(manifestPath)) {
82+
return null;
83+
}
84+
85+
let manifest;
86+
try {
87+
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
88+
} catch (err) {
89+
console.warn(`[llms-txt] WARN: failed to parse upstream manifest: ${err.message}`);
90+
return null;
91+
}
92+
93+
if (!manifest || !Array.isArray(manifest.files)) {
94+
return null;
95+
}
96+
97+
const filesRoot = path.join(upstreamCacheDir, 'files');
98+
const {repo, sha} = manifest;
99+
100+
for (const fileEntry of manifest.files) {
101+
const filePath = path.join(filesRoot, fileEntry.path);
102+
if (!fs.existsSync(filePath)) continue;
103+
104+
const content = fs.readFileSync(filePath, 'utf-8');
105+
pages.push({
106+
category: 'reference',
107+
// Synthesize a path-like key for sorting/classification using the
108+
// upstream-relative file path. Always starts with "/reference/" so
109+
// section classification can detect it.
110+
path: `/reference/${fileEntry.path.replace(/\.md$/, '')}`,
111+
title: fileEntry.title,
112+
description: '',
113+
content: content.trim(),
114+
url: `https://github.com/${repo}/blob/${sha}/doc/${fileEntry.path}`,
115+
// Carry the upstream-relative path so section classification can split
116+
// by leading subdirectory (training/, benchmarks/, etc.) without parsing.
117+
referencePath: fileEntry.path,
118+
});
119+
}
120+
121+
return {
122+
repo: manifest.repo,
123+
ref: manifest.ref,
124+
sha: manifest.sha,
125+
shortSha: manifest.shortSha,
126+
commitDate: manifest.commitDate,
127+
fetchedAt: manifest.fetchedAt,
128+
fileCount: manifest.fileCount,
129+
};
130+
}
131+
132+
function classifyPage(page) {
133+
if (page.category === 'reference') {
134+
const refPath = page.referencePath || '';
135+
if (refPath.startsWith('training/')) return 'Reference: Training Models';
136+
if (refPath.startsWith('benchmarks/')) return 'Reference: Benchmarks';
137+
return 'Reference Documentation';
51138
}
139+
140+
// Tutorial classification — preserves the original path-prefix logic.
141+
const p = page.path;
142+
if (p.startsWith('/getting-started')) return 'Getting Started';
143+
if (p.startsWith('/guides/docker')) return 'Docker Guides';
144+
if (p.startsWith('/guides/api')) return 'API Guides';
145+
if (p.startsWith('/guides/training')) return 'Training Guides';
146+
if (p.startsWith('/guides/')) return 'Configuration & Troubleshooting';
147+
if (p.startsWith('/community')) return 'Community';
148+
return 'Getting Started';
52149
}
53150

54-
function generateLlmsTxt(pages, siteUrl, baseUrl) {
151+
function generateLlmsTxt(pages, freshness) {
55152
const lines = [
56-
'# GROBID',
57-
'> Machine learning library for extracting structured data from scholarly PDFs.',
153+
'# GROBID Tutorial',
154+
'> The fast, beginner-friendly path to your first GROBID extraction.',
58155
'',
59156
'## About',
60157
'GROBID (GeneRation Of BIbliographic Data) is a machine learning tool for',
61158
'extracting, parsing, and restructuring raw documents (PDF) into structured',
62159
'TEI-encoded XML. It is designed for technical and scientific publications.',
63160
'',
64-
'## Documentation',
161+
'This is the **GROBID Tutorial** site — a hands-on, task-oriented on-ramp',
162+
'that gets newcomers from zero to a working GROBID setup as quickly as',
163+
'possible. For the full reference documentation (every endpoint, every',
164+
'config knob, every training parameter), see https://grobid.readthedocs.io/.',
65165
'',
66166
];
67167

68-
const sections = {
69-
'Getting Started': [],
70-
'Docker Guides': [],
71-
'API Guides': [],
72-
'Training Guides': [],
73-
'Configuration & Troubleshooting': [],
74-
'Reference': [],
75-
'Explanation': [],
76-
'Community': [],
77-
};
168+
if (freshness) {
169+
lines.push('## Reference content provenance');
170+
lines.push(`Reference documentation in this corpus is synced from`);
171+
lines.push(`${freshness.repo}@${freshness.shortSha} (commit ${freshness.commitDate}),`);
172+
lines.push(`fetched ${freshness.fetchedAt}.`);
173+
lines.push(`Always-current upstream: https://github.com/${freshness.repo}/tree/master/doc`);
174+
lines.push('');
175+
}
176+
177+
lines.push('## Documentation');
178+
lines.push('');
179+
180+
// Section order: tutorial groups first, then reference groups.
181+
const sectionOrder = [
182+
'Getting Started',
183+
'Docker Guides',
184+
'API Guides',
185+
'Training Guides',
186+
'Configuration & Troubleshooting',
187+
'Community',
188+
'Reference Documentation',
189+
'Reference: Training Models',
190+
'Reference: Benchmarks',
191+
];
192+
193+
const sections = {};
194+
for (const heading of sectionOrder) sections[heading] = [];
78195

79196
for (const page of pages) {
80-
const url = `${siteUrl}${baseUrl}${page.path.replace(/^\//, '')}`;
81-
const entry = `- [${page.title}](${url}): ${page.description}`;
82-
83-
if (page.path.startsWith('/getting-started')) sections['Getting Started'].push(entry);
84-
else if (page.path.startsWith('/guides/docker')) sections['Docker Guides'].push(entry);
85-
else if (page.path.startsWith('/guides/api')) sections['API Guides'].push(entry);
86-
else if (page.path.startsWith('/guides/training')) sections['Training Guides'].push(entry);
87-
else if (page.path.startsWith('/guides/')) sections['Configuration & Troubleshooting'].push(entry);
88-
else if (page.path.startsWith('/reference')) sections['Reference'].push(entry);
89-
else if (page.path.startsWith('/explanation')) sections['Explanation'].push(entry);
90-
else if (page.path.startsWith('/community')) sections['Community'].push(entry);
91-
else sections['Getting Started'].unshift(entry);
197+
const heading = classifyPage(page);
198+
if (!sections[heading]) sections[heading] = [];
199+
const entry = `- [${page.title}](${page.url})${page.description ? `: ${page.description}` : ''}`;
200+
sections[heading].push(entry);
92201
}
93202

94-
for (const [heading, entries] of Object.entries(sections)) {
95-
if (entries.length > 0) {
203+
for (const heading of sectionOrder) {
204+
const entries = sections[heading];
205+
if (entries && entries.length > 0) {
96206
lines.push(`### ${heading}`);
97207
lines.push(...entries);
98208
lines.push('');
@@ -102,18 +212,29 @@ function generateLlmsTxt(pages, siteUrl, baseUrl) {
102212
return lines.join('\n');
103213
}
104214

105-
function generateLlmsFullTxt(pages) {
215+
function generateLlmsFullTxt(pages, freshness) {
106216
const lines = [
107-
'# GROBID - Complete Documentation',
217+
'# GROBID Tutorial - Complete Documentation',
108218
'',
109-
'Machine learning library for extracting structured data from scholarly PDFs.',
219+
'The fast, beginner-friendly path to your first GROBID extraction.',
220+
'For the full reference docs, see https://grobid.readthedocs.io/.',
110221
'',
111222
];
112223

224+
if (freshness) {
225+
lines.push(
226+
`Reference content synced from ${freshness.repo}@${freshness.shortSha} ` +
227+
`on ${freshness.fetchedAt}.`,
228+
);
229+
lines.push('');
230+
}
231+
113232
for (const page of pages) {
114-
lines.push(`---`);
233+
lines.push('---');
115234
lines.push(`## ${page.title}`);
116235
lines.push(`Path: ${page.path}`);
236+
lines.push(`URL: ${page.url}`);
237+
lines.push(`Category: ${page.category}`);
117238
if (page.description) {
118239
lines.push(`Description: ${page.description}`);
119240
}

0 commit comments

Comments
 (0)