fix: patch GotScrapingHttpClient header handling and proxies (#3308)

Fixes the response header handling in `GotScrapingHttpClient`
(`got-scraping` headers contain unexpected `Symbol`s and HTTP2
pseudoheaders).

Fixes omission from one of the previous commits -
`GotScrapingHttpClient.stream` now uses proxy correctly again.

Closes #2917

Author: barjin

packages/core/src/http_clients/got-scraping-http-client.ts

@@ -20,6 +20,29 @@ export class GotScrapingHttpClient implements BaseHttpClient {
         return !['CONNECT', 'connect'].includes(request.method!);
     }
 
+    private *iterateHeaders(
+        headers: Record<string, string | string[] | undefined>,
+    ): Generator<[string, string], void, unknown> {
+        for (const [key, value] of Object.entries(headers)) {
+            // Filter out pseudo-headers
+            if (key.startsWith(':') || value === undefined) {
+                continue;
+            }
+
+            if (Array.isArray(value)) {
+                for (const v of value) {
+                    yield [key, v];
+                }
+            } else {
+                yield [key, value];
+            }
+        }
+    }
+
+    private parseHeaders(headers: Record<string, string | string[] | undefined>): Headers {
+        return new Headers([...this.iterateHeaders(headers)]);
+    }
+
     /**
      * @inheritDoc
      */
@@ -43,20 +66,10 @@ export class GotScrapingHttpClient implements BaseHttpClient {
             cookieJar: undefined,
         });
 
-        const parsedHeaders = Object.entries(gotResult.headers)
-            .map(([key, value]) => {
-                if (value === undefined) return [];
-
-                if (Array.isArray(value)) {
-                    return value.map((v) => [key, v]);
-                }
-
-                return [[key, value]];
-            })
-            .flat() as [string, string][];
+        const responseHeaders = this.parseHeaders(gotResult.headers);
 
         return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), {
-            headers: new Headers(parsedHeaders),
+            headers: responseHeaders,
             status: gotResult.statusCode,
             statusText: gotResult.statusMessage ?? '',
             url: gotResult.url,
@@ -67,6 +80,8 @@ export class GotScrapingHttpClient implements BaseHttpClient {
      * @inheritDoc
      */
     async stream(request: Request, options?: StreamOptions): Promise<Response> {
+        const { session, timeout } = options ?? {};
+
         if (!this.validateRequest(request)) {
             throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
         }
@@ -78,6 +93,8 @@ export class GotScrapingHttpClient implements BaseHttpClient {
                 headers: Object.fromEntries(request.headers.entries()),
                 body: request.body ? Readable.fromWeb(request.body as any) : undefined,
                 isStream: true,
+                proxyUrl: session?.proxyInfo?.url,
+                timeout: { request: timeout },
                 cookieJar: undefined,
             });
 
@@ -111,12 +128,13 @@ export class GotScrapingHttpClient implements BaseHttpClient {
             stream.on('error', reject);
 
             stream.on('response', (response: PlainResponse) => {
+                const headers = this.parseHeaders(response.headers);
                 // Cast shouldn't be needed here, undici might have a different `ReadableStream` type
                 resolve(
                     new ResponseWithUrl(Readable.toWeb(stream) as any, {
                         status: response.statusCode,
                         statusText: response.statusMessage ?? '',
-                        headers: response.headers as HeadersInit,
+                        headers,
                         url: response.url,
                     }),
                 );

packages/http-crawler/src/internals/http-crawler.ts

@@ -686,7 +686,7 @@ export class HttpCrawler<
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: { request: this.navigationTimeoutMillis },
+            timeout: this.navigationTimeoutMillis,
             cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
             sessionToken: session,
             headers: request.headers,