feat: rename KVS objects from SDK_ to CRAWLEE_ (#3783)

Drops the `SDK_` prefix (replaces w/ `CRAWLEE_`) on multiple spots in
the codebase.

Closes #1729

Author: barjin

docs/guides/configuration.mdx

@@ -57,7 +57,7 @@ crawler.router.addDefaultHandler(async ({ request }) => {
 await crawler.run(['https://www.example.com/1']);
 ```
 
-If you run this example (assuming you placed the `crawlee.json` file with `persistStateIntervalMillis` and `logLevel` specified there in the root of your project), you will find the `SDK_CRAWLER_STATISTICS` file in default Key-Value store,
+If you run this example (assuming you placed the `crawlee.json` file with `persistStateIntervalMillis` and `logLevel` specified there in the root of your project), you will find the `CRAWLEE_CRAWLER_STATISTICS` file in default Key-Value store,
 which would show, that there's 1 finished request and crawler runtime was ~10 seconds.
 This confirms that the state was persisted after 10 seconds, as it was set in `crawlee.json`.
 Besides, you should see `DEBUG` logs in addition to `INFO` ones in your terminal, as `logLevel` was set to `DEBUG` in the `crawlee.json`, meaning Crawlee picked both provided options correctly.
@@ -174,14 +174,14 @@ crawler.router.addDefaultHandler(async ({ request }) => {
 await crawler.run(['https://www.example.com/1']);
 ```
 
-If you run this example - you will find the `SDK_CRAWLER_STATISTICS` file in default Key-Value store,
+If you run this example - you will find the `CRAWLEE_CRAWLER_STATISTICS` file in default Key-Value store,
 which would show the same number of finished requests (one) and the same crawler runtime (~10 seconds).
 This confirms that provided parameters worked: the state was persisted after 10 seconds, as it was set in the configuration.
 
 :::note
 
 After running the same example without the custom configuration, there will be
-no `SDK_CRAWLER_STATISTICS` file stored in the default Key-Value store:
+no `CRAWLEE_CRAWLER_STATISTICS` file stored in the default Key-Value store:
 as we did not change the `persistStateIntervalMillis`, Crawlee used the default value of 60 seconds,
 and the crawler was forcefully aborted after ~15 seconds of run time before it persisted the state for the first time.
 
@@ -222,13 +222,13 @@ await crawler.run(['https://www.example.com/1']);
 ```
 
 If you run this example - it would work exactly the same as before,
-with the same `SDK_CRAWLER_STATISTICS` file in default Key-Value store after the run,
+with the same `CRAWLEE_CRAWLER_STATISTICS` file in default Key-Value store after the run,
 showing the same number of finished requests and the same crawler run time.
 
 :::note
 
 If you would not pass the configuration to the crawler, there again will be
-no `SDK_CRAWLER_STATISTICS` file stored in the default Key-Value store, this time for a different reason though.
+no `CRAWLEE_CRAWLER_STATISTICS` file stored in the default Key-Value store, this time for a different reason though.
 Since we did not pass the configuration to the crawler,
 the crawler will use the global configuration, which is using the default `persistStateIntervalMillis`.
 So again, the run was aborted before the state was persisted for the first time.

docs/upgrading/upgrading_v4.md

@@ -504,6 +504,10 @@ The protected `HttpCrawler._applyCookies` method is removed. If you were overrid
 
 The `persistCookiesPerSession` crawler option has been renamed to `saveResponseCookies` on both `HttpCrawler` (and its subclasses like `CheerioCrawler`, `JSDOMCrawler`, etc.) and `BrowserCrawler`. The behavior is unchanged - when enabled (the default), response `Set-Cookie` headers are stored in the session's cookie jar so they're sent on subsequent requests using the same session. Rename the option in your crawler constructor options to migrate.
 
+## Internal KVS keys renamed
+
+Several internal Crawlee keys were prefixed with the `SDK_` prefix for legacy reasons - these keys now start with `CRAWLEE_` instead. These are, e.g., `CRAWLEE_SESSION_POOL_STATE` or `CRAWLEE_CRAWLER_STATISTICS_{n}`.
+
 ## `StorageClient` interface simplified
 
 The `StorageClient` interface (from `@crawlee/types`) has been redesigned to match the simplified architecture from Crawlee for Python. A new storage backend now needs **4 classes** instead of the previous 7.

packages/core/src/crawlers/statistics.ts

@@ -49,7 +49,7 @@ export interface PersistenceOptions {
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
@@ -130,7 +130,7 @@ export class Statistics {
         } = options;
 
         this.id = id ?? String(Statistics.id++);
-        this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
+        this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
 
         this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });

packages/core/src/session_pool/consts.ts

@@ -1,3 +1,3 @@
 export const BLOCKED_STATUS_CODES = [401, 403, 429];
-export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';
+export const PERSIST_STATE_KEY = 'CRAWLEE_SESSION_POOL_STATE';
 export const MAX_POOL_SIZE = 1000;

packages/core/src/session_pool/session_pool.ts

@@ -48,7 +48,7 @@ export interface SessionPoolOptions {
 
     /**
      * Session pool persists its state under this key in Key value store.
-     * @default SDK_SESSION_POOL_STATE_{id}
+     * @default CRAWLEE_SESSION_POOL_STATE_{id}
      */
     persistStateKey?: string;
 

packages/core/src/storages/request_list.ts

@@ -332,8 +332,8 @@ export class RequestList implements IRequestLoader {
             }),
         );
 
-        this.persistStateKey = persistStateKey ? `SDK_${persistStateKey}` : persistStateKey;
-        this.persistRequestsKey = persistRequestsKey ? `SDK_${persistRequestsKey}` : persistRequestsKey;
+        this.persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
+        this.persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
         this.initialState = state;
         this.httpClient = httpClient;
 

test/core/crawlers/statistics.test.ts

@@ -36,7 +36,7 @@ describe('Statistics', () => {
             // @ts-expect-error Accessing private prop
             expect(Statistics.id).toEqual(1);
             // @ts-expect-error Accessing private prop
-            expect(stats.persistStateKey).toEqual('SDK_CRAWLER_STATISTICS_0');
+            expect(stats.persistStateKey).toEqual('CRAWLEE_CRAWLER_STATISTICS_0');
             const [n1, n2] = [new Statistics(), new Statistics()];
             expect(n1.id).toEqual('1');
             expect(n2.id).toEqual('2');

test/core/request_list.test.ts

@@ -566,15 +566,15 @@ describe('RequestList', () => {
             const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
 
             const name = 'xxx';
-            const SDK_KEY = `SDK_${name}`;
+            const CRAWLEE_KEY = `CRAWLEE_${name}`;
             const sources = [{ url: 'https://example.com' }];
 
             const rl = await RequestList.open(name, sources);
             expect(rl).toBeInstanceOf(RequestList);
             // @ts-expect-error accessing private var
-            expect(rl.persistStateKey.startsWith(SDK_KEY)).toBe(true);
+            expect(rl.persistStateKey.startsWith(CRAWLEE_KEY)).toBe(true);
             // @ts-expect-error accessing private var
-            expect(rl.persistRequestsKey.startsWith(SDK_KEY)).toBe(true);
+            expect(rl.persistRequestsKey.startsWith(CRAWLEE_KEY)).toBe(true);
             // @ts-expect-error accessing private var
             expect(rl.sources).toEqual([]);
             // @ts-expect-error accessing private var
@@ -589,16 +589,16 @@ describe('RequestList', () => {
             const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
 
             const name = 'xxx';
-            const SDK_KEY = `SDK_${name}`;
+            const CRAWLEE_KEY = `CRAWLEE_${name}`;
             const sources = ['https://example.com'];
             const requests = sources.map((url) => ({ url, uniqueKey: url }));
 
             const rl = await RequestList.open(name, sources);
             expect(rl).toBeInstanceOf(RequestList);
             // @ts-expect-error accessing private var
-            expect(rl.persistStateKey.startsWith(SDK_KEY)).toBe(true);
+            expect(rl.persistStateKey.startsWith(CRAWLEE_KEY)).toBe(true);
             // @ts-expect-error accessing private var
-            expect(rl.persistRequestsKey.startsWith(SDK_KEY)).toBe(true);
+            expect(rl.persistRequestsKey.startsWith(CRAWLEE_KEY)).toBe(true);
             expect(rl.requests).toEqual(requests);
             // @ts-expect-error accessing private var
             expect(rl.isInitialized).toBe(true);
@@ -612,7 +612,7 @@ describe('RequestList', () => {
             const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
 
             const name = 'xxx';
-            const SDK_KEY = `SDK_${name}`;
+            const CRAWLEE_KEY = `CRAWLEE_${name}`;
             let counter = 0;
             const sources = [{ url: 'https://example.com' }];
             const requests = sources.map(({ url }) => ({ url, uniqueKey: `${url}-${counter++}` }));
@@ -624,9 +624,9 @@ describe('RequestList', () => {
             const rl = await RequestList.open(name, sources, options);
             expect(rl).toBeInstanceOf(RequestList);
             // @ts-expect-error accessing private var
-            expect(rl.persistStateKey.startsWith(SDK_KEY)).toBe(true);
+            expect(rl.persistStateKey.startsWith(CRAWLEE_KEY)).toBe(true);
             // @ts-expect-error accessing private var
-            expect(rl.persistRequestsKey.startsWith(SDK_KEY)).toBe(true);
+            expect(rl.persistRequestsKey.startsWith(CRAWLEE_KEY)).toBe(true);
             expect(rl.requests).toEqual(requests);
             // @ts-expect-error accessing private var
             expect(rl.isInitialized).toBe(true);

test/e2e/tools.mjs

@@ -22,7 +22,7 @@ function execSync(command, options) {
 /**
  * @param {string} name
  */
-const isPrivateEntry = (name) => name === 'SDK_CRAWLER_STATISTICS_0' || name === 'SDK_SESSION_POOL_STATE';
+const isPrivateEntry = (name) => name === 'CRAWLEE_CRAWLER_STATISTICS_0' || name === 'CRAWLEE_SESSION_POOL_STATE';
 
 export const SKIPPED_TEST_CLOSE_CODE = 404;
 
@@ -49,7 +49,7 @@ export function getStorage(dirName) {
  */
 export async function getStats(dirName) {
     const dir = getStorage(dirName);
-    const path = join(dir, `key_value_stores/default/SDK_CRAWLER_STATISTICS_0.json`);
+    const path = join(dir, `key_value_stores/default/CRAWLEE_CRAWLER_STATISTICS_0.json`);
 
     if (!existsSync(path)) {
         return false;
@@ -208,7 +208,9 @@ export async function runActor(dirName, memory = 4096) {
         const runTook = (runFinishedAt.getTime() - runStartedAt.getTime()) / 1000;
         console.log(`[run] View run: https://console.apify.com/view/runs/${runId} [run took ${runTook}s]`);
 
-        const statsRecord = await client.keyValueStore(defaultKeyValueStoreId).getRecord('SDK_CRAWLER_STATISTICS_0');
+        const statsRecord = await client
+            .keyValueStore(defaultKeyValueStoreId)
+            .getRecord('CRAWLEE_CRAWLER_STATISTICS_0');
         stats = statsRecord?.value;
 
         const { items } = await client.dataset(defaultDatasetId).listItems();