fix: read content encoding hints from <meta> tags (#3780)

Utilizes the byte-stream prescan
(https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding)
to determine the HTML document encoding, if not specified in HTTP
headers (or explicitly forced by user).

Closes #2317

Author: barjin

packages/http-crawler/src/internals/http-crawler.ts

@@ -34,7 +34,7 @@ import type { JsonValue } from 'type-fest';
 
 import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
 
-import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
+import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 
 /**
  * Default mime types, which HttpScraper supports.
@@ -640,6 +640,15 @@ export class HttpCrawler<
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
             throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         } else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            if (!charset && !this.forceResponseEncoding) {
+                const rawBytes = Buffer.from(await response.arrayBuffer());
+                const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+                const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
+                const body = iconv.encodingExists(charsetToUse)
+                    ? iconv.decode(rawBytes, charsetToUse)
+                    : rawBytes.toString('utf8');
+                return { response, contentType: { type, encoding: 'utf-8' as BufferEncoding }, body };
+            }
             return { response, contentType, body: await reencodedResponse.text() };
         } else {
             const body = Buffer.from(await reencodedResponse.bytes());

packages/http-crawler/src/internals/utils.ts

@@ -59,6 +59,18 @@ export function processHttpRequestOptions({
     return { ...request, body, url, headers };
 }
 
+/**
+ * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
+ * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
+ * This implements a simplified version of the HTML spec's byte-stream prescan algorithm.
+ */
+export function extractCharsetFromHtmlBytes(bytes: Buffer): string | undefined {
+    // latin1 preserves byte values for ASCII-compatible encodings, making the meta tags readable
+    const prescan = bytes.subarray(0, 1024).toString('latin1');
+    const match = /<meta[^>]+\bcharset\s*=\s*["']?\s*([^"'\s;>]+)/i.exec(prescan);
+    return match?.[1];
+}
+
 /**
  * Gets parsed content type from response object
  * @param response HTTP response object

test/core/crawlers/cheerio_crawler.test.ts

@@ -695,6 +695,22 @@ describe('CheerioCrawler', () => {
             expect(await response.text()).toBe(html);
         });
 
+        test('via http-equiv meta tag when no charset in HTTP header', async () => {
+            let context: CheerioCrawlingContext | null = null;
+
+            const crawler = new CheerioCrawler({
+                requestHandler: (ctx) => {
+                    context = ctx;
+                },
+            });
+
+            await crawler.run([`${serverAddress}/special/meta-charset`]);
+
+            context = context as unknown as CheerioCrawlingContext;
+            expect(context?.body).toContain('Žluťoučký kůň');
+            expect(context?.$('body').text()).toContain('Žluťoučký kůň');
+        });
+
         test('Cheerio decodes html entities', async () => {
             let context: CheerioCrawlingContext | null = null;
 

test/core/crawlers/http_crawler.test.ts

@@ -4,6 +4,7 @@ import { Readable } from 'node:stream';
 
 import { HttpCrawler, SessionPool } from '@crawlee/http';
 import { ResponseWithUrl } from '@crawlee/http-client';
+import iconv from 'iconv-lite';
 import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';
 
 const router = new Map<string, http.RequestListener>();
@@ -60,6 +61,20 @@ router.set('/403-with-octet-stream', (req, res) => {
     res.end();
 });
 
+router.set('/meta-charset', (req, res) => {
+    const text = 'Žluťoučký kůň';
+    const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
+    res.setHeader('content-type', 'text/html'); // no charset in HTTP header
+    res.end(iconv.encode(html, 'windows-1250'));
+});
+
+router.set('/meta-charset-html5', (req, res) => {
+    const text = 'Žluťoučký kůň';
+    const html = `<html><head><meta charset="windows-1250"></head><body>${text}</body></html>`;
+    res.setHeader('content-type', 'text/html'); // no charset in HTTP header
+    res.end(iconv.encode(html, 'windows-1250'));
+});
+
 let server: http.Server;
 let url: string;
 
@@ -208,6 +223,36 @@ test('invalid content type defaults to octet-stream', async () => {
     ]);
 });
 
+test('decodes charset from http-equiv meta tag when absent in HTTP header', async () => {
+    const results: string[] = [];
+
+    const crawler = new HttpCrawler({
+        maxRequestRetries: 0,
+        requestHandler: ({ body }) => {
+            results.push(body as string);
+        },
+    });
+
+    await crawler.run([`${url}/meta-charset`]);
+
+    expect(results[0]).toContain('Žluťoučký kůň');
+});
+
+test('decodes charset from HTML5 meta charset attribute when absent in HTTP header', async () => {
+    const results: string[] = [];
+
+    const crawler = new HttpCrawler({
+        maxRequestRetries: 0,
+        requestHandler: ({ body }) => {
+            results.push(body as string);
+        },
+    });
+
+    await crawler.run([`${url}/meta-charset-html5`]);
+
+    expect(results[0]).toContain('Žluťoučký kůň');
+});
+
 test('handles cookies from redirects', async () => {
     const results: string[] = [];
 

test/shared/_helper.ts

@@ -7,6 +7,7 @@ import bodyParser from 'body-parser';
 import { entries } from 'crawlee';
 import type { Application } from 'express';
 import express from 'express';
+import iconv from 'iconv-lite';
 
 export const startExpressAppPromise = async (app: Application, port: number) => {
     return new Promise<Server>((resolve) => {
@@ -332,6 +333,13 @@ export async function runExampleComServer(): Promise<[Server, number]> {
             res.type('html').send('&quot;&lt;&gt;"<>');
         });
 
+        special.get('/meta-charset', (_req, res) => {
+            const text = 'Žluťoučký kůň';
+            const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
+            res.setHeader('content-type', 'text/html');
+            res.end(iconv.encode(html, 'windows-1250'));
+        });
+
         special.get('/set-cookie', (req, res) => {
             const cookieName = (req.query.name as string) || 'testCookie';
             const cookieValue = (req.query.value as string) || 'testValue';