Skip to content

Commit 8cf173e

Browse files
authored
chore: drop got-scraping as the direct crawlee dependency (#3358)
Removes `got-scraping` from the dependency lists of `@crawlee`-scoped packages (except for `@crawlee/got-scraping`). `HttpClient` implementations now throw Node's `TimeoutError` as a result of `AbortSignal.timeout()` firing. Closes #3275
1 parent a7a6eb7 commit 8cf173e

3 files changed

Lines changed: 1 addition & 26 deletions

File tree

packages/http-crawler/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@
5656
"@types/content-type": "^1.1.8",
5757
"cheerio": "^1.0.0",
5858
"content-type": "^1.0.5",
59-
"got-scraping": "^4.1.1",
6059
"iconv-lite": "^0.7.0",
6160
"mime-types": "^3.0.1",
6261
"ow": "^2.0.0",

packages/http-crawler/src/internals/http-crawler.ts

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils';
3030
import * as cheerio from 'cheerio';
3131
import type { RequestLike, ResponseLike } from 'content-type';
3232
import contentTypeParser from 'content-type';
33-
import type { TimeoutError as TimeoutErrorClass } from 'got-scraping';
3433
import iconv from 'iconv-lite';
3534
import ow from 'ow';
3635
import type { JsonValue } from 'type-fest';
@@ -39,8 +38,6 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
3938

4039
import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
4140

42-
let TimeoutError: typeof TimeoutErrorClass;
43-
4441
/**
4542
* Default mime types, which HttpScraper supports.
4643
*/
@@ -608,17 +605,12 @@ export class HttpCrawler<
608605
proxyUrl,
609606
cookieString,
610607
}: RequestFunctionOptions): Promise<Response> {
611-
if (!TimeoutError) {
612-
// @ts-ignore
613-
({ TimeoutError } = await import('got-scraping'));
614-
}
615-
616608
const opts = this._getRequestOptions(request, session, proxyUrl);
617609

618610
try {
619611
return await this._requestAsBrowser(opts, session, cookieString);
620612
} catch (e) {
621-
if (e instanceof TimeoutError) {
613+
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
622614
this._handleRequestTimeout(session);
623615
return new Response(); // this will never happen, as _handleRequestTimeout always throws
624616
}

yarn.lock

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,6 @@ __metadata:
686686
"@types/content-type": "npm:^1.1.8"
687687
cheerio: "npm:^1.0.0"
688688
content-type: "npm:^1.0.5"
689-
got-scraping: "npm:^4.1.1"
690689
iconv-lite: "npm:^0.7.0"
691690
mime-types: "npm:^3.0.1"
692691
ow: "npm:^2.0.0"
@@ -7603,21 +7602,6 @@ __metadata:
76037602
languageName: node
76047603
linkType: hard
76057604

7606-
"got-scraping@npm:^4.1.1":
7607-
version: 4.1.1
7608-
resolution: "got-scraping@npm:4.1.1"
7609-
dependencies:
7610-
got: "npm:^14.2.1"
7611-
header-generator: "npm:^2.1.41"
7612-
http2-wrapper: "npm:^2.2.0"
7613-
mimic-response: "npm:^4.0.0"
7614-
ow: "npm:^1.1.1"
7615-
quick-lru: "npm:^7.0.0"
7616-
tslib: "npm:^2.6.2"
7617-
checksum: 10c0/66b9bd88fea1c7a1248fec6e9c9757300b70e6039d2b2e0cf1c70e44e88be80f02a26e2e36d5f9c3acb4ec963558d72b0d236a7f11a7a6c87b39b5615afcf7db
7618-
languageName: node
7619-
linkType: hard
7620-
76217605
"got-scraping@npm:^4.1.3":
76227606
version: 4.1.3
76237607
resolution: "got-scraping@npm:4.1.3"

0 commit comments

Comments
 (0)