import { readFile, rm } from 'node:fs/promises';
import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';

import type {
    CrawlingContext,
    EnqueueLinksOptions,
    ErrorHandler,
    RequestHandler,
    RequestOptions,
    Source,
} from '@crawlee/basic';
import {
    BasicCrawler,
    Configuration,
    CriticalError,
    EventType,
    KeyValueStore,
    MissingRouteError,
    NonRetryableError,
    Request,
    RequestList,
    RequestQueue,
} from '@crawlee/basic';
import { RequestState } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
import express from 'express';
import type { SetRequired } from 'type-fest';
import type { Mock } from 'vitest';
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';

import log from '@apify/log';

import { startExpressAppPromise } from '../../shared/_helper';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';

describe('BasicCrawler', () => {
    let logLevel: number;
    const localStorageEmulator = new MemoryStorageEmulator();
    const events = Configuration.getEventManager();

    const HOSTNAME = '127.0.0.1';
    let port: number;
    let server: Server;
    beforeAll(async () => {
        const app = express();

        app.get('/', (req, res) => {
            res.send(`<html><head><title>Example Domain</title></head></html>`);
        });

        server = await startExpressAppPromise(app, 0);
        port = (server.address() as AddressInfo).port;
    });

    beforeAll(async () => {
        logLevel = log.getLevel();
        log.setLevel(log.LEVELS.OFF);
    });

    beforeEach(async () => {
        vitest.clearAllMocks();
        await localStorageEmulator.init();
    });

    afterAll(async () => {
        await localStorageEmulator.destroy();
    });

    afterAll(async () => {
        log.setLevel(logLevel);
    });

    afterAll(() => {
        server.close();
    });

    test('does not leak sigint events', async () => {
        let count = 0;

        const crawler = new BasicCrawler({
            requestHandler: () => {
                count = process.listenerCount('SIGINT');
            },
        });

        await crawler.run(['https://example.com']);

        expect(process.listenerCount('SIGINT')).toBe(count - 1);
    });

    test('should run in parallel thru all the requests', async () => {
        const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` }));
        const sourcesCopy = JSON.parse(JSON.stringify(sources));

        const processed: { url: string }[] = [];
        const requestList = await RequestList.open(null, sources);
        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed.push({ url: request.url });
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            minConcurrency: 25,
            maxConcurrency: 25,
            requestHandler,
        });

        await basicCrawler.run();

        expect(basicCrawler.autoscaledPool!.minConcurrency).toBe(25);
        expect(processed).toEqual(sourcesCopy);
        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
    });

    test('should allow using run method multiple times', async () => {
        const sources = [...Array(100).keys()].map((index) => `https://example.com/${index}`);
        const sourcesCopy = JSON.parse(JSON.stringify(sources));

        const processed: { url: string }[] = [];
        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed.push({ url: request.url });
        };

        const basicCrawler = new BasicCrawler({
            minConcurrency: 25,
            maxConcurrency: 25,
            requestHandler,
        });

        await basicCrawler.run(sources);
        await basicCrawler.run(sources);
        await basicCrawler.run(sources);

        expect(processed).toHaveLength(sourcesCopy.length * 3);
    });

    test('should process 4 requests total when calling run() twice with maxRequestsPerCrawl: 2', async () => {
        const processed: { url: string }[] = [];

        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed.push({ url: request.url });
        };

        const crawler = new BasicCrawler({
            maxRequestsPerCrawl: 2,
            minConcurrency: 1,
            maxConcurrency: 1,
            requestHandler,
        });

        // First run should process 2 requests
        await crawler.run([...Array(5).keys()].map((index) => `https://example.com/first/${index}`));
        expect(processed).toHaveLength(2);

        // Make sure no extra requests were enqueued
        expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);

        // Second run should process 2 more requests
        await crawler.run([...Array(5).keys()].map((index) => `https://example.com/second/${index}`));
        expect(processed).toHaveLength(4);

        // Make sure no extra requests were enqueued
        expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);

        const processedUrls = processed.map((p) => p.url);

        expect(processedUrls).toEqual([
            'https://example.com/first/0',
            'https://example.com/first/1',
            'https://example.com/second/0',
            'https://example.com/second/1',
        ]);
    });

    test('addRequests should respect maxCrawlDepth', async () => {
        const processedUrls: string[] = [];

        const requestHandler: RequestHandler = async ({ request, addRequests }) => {
            processedUrls.push(request.url);
            const url = new URL(request.url);
            url.pathname = `${url.pathname}deep/`;

            await addRequests([url.toString()]);
        };

        const crawler = new BasicCrawler({
            maxCrawlDepth: 2,
            maxRequestsPerCrawl: 10, // safeguard against infinite loops
            requestHandler,
        });

        await crawler.run(['https://example.com/']);

        expect(processedUrls).toEqual([
            'https://example.com/',
            'https://example.com/deep/',
            'https://example.com/deep/deep/',
        ]);
    });

    test('enqueueLinks should respect maxCrawlDepth', async () => {
        const processedUrls: string[] = [];

        const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {
            processedUrls.push(request.url);
            const url = new URL(request.url);
            url.pathname = `${url.pathname}deep/`;

            await enqueueLinks({
                urls: [url.toString()],
            });
        };

        const crawler = new BasicCrawler({
            maxCrawlDepth: 2,
            maxRequestsPerCrawl: 10, // safeguard against infinite loops
            requestHandler,
        });

        await crawler.run(['https://example.com/']);

        expect(processedUrls).toEqual([
            'https://example.com/',
            'https://example.com/deep/',
            'https://example.com/deep/deep/',
        ]);
    });

    describe('enqueueLinksWithCrawlDepth()', () => {
        let onSkippedRequestMock: Mock;
        let addRequestsBatchedMock: Mock;
        let options: SetRequired<EnqueueLinksOptions, 'urls'>;
        let request: Request;
        let requestQueue: RequestQueue;

        type EnqueueLinksWrapperOptions = Parameters<BasicCrawler['enqueueLinksWithCrawlDepth']>;
        class TestCrawler extends BasicCrawler {
            public exposedEnqueueLinksWithCrawlDepth(...enqueueLinksOptions: EnqueueLinksWrapperOptions) {
                return this.enqueueLinksWithCrawlDepth(...enqueueLinksOptions);
            }
        }

        const crawler = new TestCrawler({ maxCrawlDepth: 3 });

        beforeEach(() => {
            addRequestsBatchedMock = vi.fn().mockImplementation(async () => ({
                addedRequests: [],
                waitForAllRequestsToBeAdded: Promise.resolve([]),
                requestsOverLimit: [],
            }));
            onSkippedRequestMock = vi.fn();

            options = {
                urls: ['https://example.com/1/', 'https://example.com/2/'],
                onSkippedRequest: onSkippedRequestMock,
            };
            request = new Request({ url: 'https://example.com/', crawlDepth: 2 });
            requestQueue = {
                addRequestsBatched: addRequestsBatchedMock as RequestQueue['addRequestsBatched'],
            } as RequestQueue;
        });

        it('should generate requests with maxCrawlDepth', async () => {
            await crawler.exposedEnqueueLinksWithCrawlDepth(options, request, requestQueue);

            const requests = addRequestsBatchedMock.mock.calls[0][0];
            expect(requests).toHaveLength(2);
            expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
            expect(requests[1]).toMatchObject({ url: 'https://example.com/2/', crawlDepth: 3 });

            expect(onSkippedRequestMock).not.toBeCalled();
        });

        it('should skip requests with crawlDepth exceeding maxCrawlDepth', async () => {
            const requestWithMaxDepth = new Request({ url: 'https://example.com/', crawlDepth: 3 });
            await crawler.exposedEnqueueLinksWithCrawlDepth(options, requestWithMaxDepth, requestQueue);

            const requests = addRequestsBatchedMock.mock.calls[0][0];
            expect(requests).toHaveLength(0);

            const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);
            expect(skippedRequests).toHaveLength(2);
            expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });
            expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });
        });

        it('should respect user provided transformRequestFunction', async () => {
            const transformRequestFunction = vi.fn((req: RequestOptions) => req);
            const optionsWithTransform = { ...options, transformRequestFunction };

            await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);

            expect(transformRequestFunction).toHaveBeenCalled();

            const requests = addRequestsBatchedMock.mock.calls[0][0];
            expect(requests).toHaveLength(2);
            expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
        });
    });

    it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {
        type AddCrawlDepthWrapperOptions = Parameters<BasicCrawler['addCrawlDepthRequestGenerator']>;
        class TestCrawler extends BasicCrawler {
            public exposedAddCrawlDepthRequestGenerator(...enqueueLinksOptions: AddCrawlDepthWrapperOptions) {
                return this.addCrawlDepthRequestGenerator(...enqueueLinksOptions);
            }
        }

        const crawler = new TestCrawler();

        const requests = ['https://example.com/1/', { url: 'https://example.com/2/' }];
        const newCrawlDepth = 4;
        const addRequestsGenerator = crawler.exposedAddCrawlDepthRequestGenerator(requests, newCrawlDepth);

        const generatedRequests: Source[] = [];
        for await (const generatedRequest of addRequestsGenerator) {
            generatedRequests.push(generatedRequest);
        }

        expect(generatedRequests).toHaveLength(2);
        expect(generatedRequests[0].url).toBe('https://example.com/1/');
        expect(generatedRequests[0].crawlDepth).toBe(4);

        expect(generatedRequests[1].url).toBe('https://example.com/2/');
        expect(generatedRequests[1].crawlDepth).toBe(4);
    });

    test('should correctly combine shorthand and full length options', async () => {
        const shorthandOptions = {
            minConcurrency: 123,
            maxConcurrency: 456,
            maxRequestsPerMinute: 789,
        };

        const autoscaledPoolOptions = {
            minConcurrency: 16,
            maxConcurrency: 32,
            maxTasksPerMinute: 64,
        };

        const collectResults = (crawler: BasicCrawler): typeof shorthandOptions | typeof autoscaledPoolOptions => {
            return {
                minConcurrency: crawler.autoscaledPool!.minConcurrency,
                maxConcurrency: crawler.autoscaledPool!.maxConcurrency,
                // eslint-disable-next-line dot-notation -- accessing a private member
                maxRequestsPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'],
                // eslint-disable-next-line dot-notation
                maxTasksPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'],
            };
        };

        const requestList = await RequestList.open(null, []);
        const requestHandler = async () => {};

        const results = await Promise.all(
            [
                new BasicCrawler({
                    requestList,
                    requestHandler,
                    ...shorthandOptions,
                }),
                new BasicCrawler({
                    requestList,
                    requestHandler,
                    autoscaledPoolOptions,
                }),
                new BasicCrawler({
                    requestList,
                    requestHandler,
                    ...shorthandOptions,
                    autoscaledPoolOptions,
                }),
            ].map(async (c) => {
                await c.run();
                return collectResults(c);
            }),
        );

        expect(results[0]).toEqual(expect.objectContaining(shorthandOptions));

        expect(results[1]).toEqual(expect.objectContaining(autoscaledPoolOptions));

        expect(results[2]).toEqual(expect.objectContaining(shorthandOptions));
    });

    test('auto-saved state object', async () => {
        const sources = [...Array(50).keys()].map((index) => ({ url: `https://example.com/${index}` }));
        const sourcesCopy = JSON.parse(JSON.stringify(sources));

        const processed: { url: string }[] = [];
        const requestList = await RequestList.open(null, sources);
        const requestHandler: RequestHandler = async ({ request, crawler }) => {
            await sleep(10);
            const state = await crawler.useState({ processed });
            state.processed.push({ url: request.url });
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            requestHandler,
        });

        await basicCrawler.run();
        const state = await basicCrawler.useState();

        expect(processed).toEqual(sourcesCopy);
        expect(state.processed).toEqual(sourcesCopy);
        expect(state.processed).toBe(processed);
        expect(state.processed).toEqual(sourcesCopy);
        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
    });

    test.each([
        EventType.MIGRATING,
        EventType.ABORTING,
    ])('should pause on %s event and persist RequestList state', async (event) => {
        const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` }));

        let persistResolve!: (value?: unknown) => void;
        const persistPromise = new Promise((res) => {
            persistResolve = res;
        });

        // Mock the calls to persist sources.
        const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
        const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
        getValueSpy.mockResolvedValue(null);

        const processed: { url: string }[] = [];
        const requestList = await RequestList.open('reqList', sources);
        const requestHandler: RequestHandler = async ({ request }) => {
            if (request.url.endsWith('200')) events.emit(event);
            processed.push({ url: request.url });
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            minConcurrency: 25,
            maxConcurrency: 25,
            requestHandler,
        });

        let finished = false;
        // Mock the call to persist state.
        setValueSpy.mockImplementationOnce(persistResolve as any);
        // The crawler will pause after 200 requests
        const runPromise = basicCrawler.run();
        void runPromise.then(() => {
            finished = true;
        });

        // need to monkeypatch the stats class, otherwise it will never finish
        basicCrawler.stats.persistState = async () => Promise.resolve();
        await persistPromise;

        expect(finished).toBe(false);
        expect(await requestList.isFinished()).toBe(false);
        expect(await requestList.isEmpty()).toBe(false);
        expect(processed.length).toBe(200);

        expect(getValueSpy).toBeCalled();
        expect(setValueSpy).toBeCalled();

        // clean up
        // @ts-expect-error Accessing private method
        await basicCrawler.autoscaledPool!._destroy();
    });

    test('should retry failed requests', async () => {
        const sources = [
            { url: 'http://example.com/1' },
            { url: 'http://example.com/2' },
            { url: 'http://example.com/3' },
        ];
        const processed: Dictionary<Request> = {};
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed[request.url] = request;

            if (request.url === 'http://example.com/2') {
                throw Error(`This is ${request.retryCount}th error!`);
            }

            request.userData.foo = 'bar';
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            maxRequestRetries: 10,
            minConcurrency: 3,
            maxConcurrency: 3,
            requestHandler,
        });

        await basicCrawler.run();

        expect(processed['http://example.com/1'].userData.foo).toBe('bar');
        expect(processed['http://example.com/1'].errorMessages).toEqual([]);
        expect(processed['http://example.com/1'].retryCount).toBe(0);
        expect(processed['http://example.com/3'].userData.foo).toBe('bar');
        expect(processed['http://example.com/3'].errorMessages).toEqual([]);
        expect(processed['http://example.com/3'].retryCount).toBe(0);

        expect(processed['http://example.com/2'].userData.foo).toBeUndefined();
        expect(processed['http://example.com/2'].errorMessages).toHaveLength(11);
        expect(processed['http://example.com/2'].retryCount).toBe(10);

        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
    });

    test('should retry failed requests based on `request.maxRetries`', async () => {
        const sources = [
            { url: 'http://example.com/1', maxRetries: 10 },
            { url: 'http://example.com/2', maxRetries: 5 },
            { url: 'http://example.com/3', maxRetries: 1 },
        ];
        const processed: Dictionary<Request> = {};
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed[request.url] = request;
            throw Error(`This is ${request.retryCount}th error!`);
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            minConcurrency: 3,
            maxConcurrency: 3,
            requestHandler,
        });

        await basicCrawler.run();

        expect(processed['http://example.com/1'].errorMessages).toHaveLength(11);
        expect(processed['http://example.com/1'].retryCount).toBe(10);
        expect(processed['http://example.com/2'].errorMessages).toHaveLength(6);
        expect(processed['http://example.com/2'].retryCount).toBe(5);
        expect(processed['http://example.com/3'].errorMessages).toHaveLength(2);
        expect(processed['http://example.com/3'].retryCount).toBe(1);

        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
    });

    test('should not retry requests with noRetry set to true ', async () => {
        const noRetryRequest = new Request({ url: 'http://example.com/3' });
        noRetryRequest.noRetry = true;

        const sources = [
            { url: 'http://example.com/1', noRetry: true },
            { url: 'http://example.com/2' },
            noRetryRequest,
        ];
        const processed: Dictionary<Request> = {};
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed[request.url] = request;
            request.userData.foo = 'bar';
            throw Error(`This is ${request.retryCount}th error!`);
        };

        let failedRequestHandlerCalls = 0;
        const failedRequestHandler = async () => {
            failedRequestHandlerCalls++;
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            maxRequestRetries: 10,
            minConcurrency: 3,
            maxConcurrency: 3,
            requestHandler,
            failedRequestHandler,
        });

        await basicCrawler.run();

        expect(processed['http://example.com/1'].userData.foo).toBe('bar');
        expect(processed['http://example.com/1'].errorMessages).toHaveLength(1);
        expect(processed['http://example.com/1'].retryCount).toBe(0);
        expect(processed['http://example.com/3'].userData.foo).toBe('bar');
        expect(processed['http://example.com/3'].errorMessages).toHaveLength(1);
        expect(processed['http://example.com/3'].retryCount).toBe(0);

        expect(processed['http://example.com/2'].userData.foo).toBe('bar');
        expect(processed['http://example.com/2'].errorMessages).toHaveLength(11);
        expect(processed['http://example.com/2'].retryCount).toBe(10);

        expect(failedRequestHandlerCalls).toBe(3);

        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
    });

    test('should correctly track request.state', async () => {
        const sources = [{ url: 'http://example.com/1' }];
        const requestList = await RequestList.open(null, sources);
        const requestStates: RequestState[] = [];

        const requestHandler: RequestHandler = async ({ request }) => {
            requestStates.push(request.state);
            throw new Error('Error');
        };

        const errorHandler: RequestHandler = async ({ request }) => {
            requestStates.push(request.state);
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            maxConcurrency: 1,
            maxRequestRetries: 1,
            requestHandler,
            errorHandler,
        });

        await basicCrawler.run();

        expect(requestStates).toEqual([
            RequestState.REQUEST_HANDLER,
            RequestState.ERROR_HANDLER,
            RequestState.REQUEST_HANDLER,
        ]);
    });

    test('should use errorHandler', async () => {
        const sources = [{ url: 'http://example.com/', label: 'start' }];

        let errorHandlerCalls = 0;
        let failedRequestHandlerCalls = 0;

        const failed: Dictionary<{ request: Request; error: Error }> = {};
        const requestList = await RequestList.open({ sources });

        const requestHandler: RequestHandler = async ({ request }) => {
            expect(request.label).toBe(errorHandlerCalls === 0 ? 'start' : `error_${errorHandlerCalls}`);
            throw new Error(`This is an error ${errorHandlerCalls}`);
        };

        const errorHandler: ErrorHandler = async ({ request }, error) => {
            expect(error.message).toBe(`This is an error ${errorHandlerCalls}`);
            errorHandlerCalls++;
            request.label = `error_${errorHandlerCalls}`;
        };

        const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
            failed[request.url] = { request, error };
            failedRequestHandlerCalls++;
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            requestHandler,
            errorHandler,
            failedRequestHandler,
        });

        await basicCrawler.run();

        expect(errorHandlerCalls).toBe(3);
        expect(failedRequestHandlerCalls).toBe(1);
        expect(Object.values(failed)).toHaveLength(1);
        expect(failed['http://example.com/'].request.label).not.toBe('start');
        expect(failed['http://example.com/'].request.label).toBe('error_3');
        expect(failed['http://example.com/'].error.message).toEqual('This is an error 3');
    });

    test('should allow to handle failed requests', async () => {
        const sources = [
            { url: 'http://example.com/1' },
            { url: 'http://example.com/2' },
            { url: 'http://example.com/3' },
        ];
        const processed: Dictionary<Request> = {};
        const failed: Dictionary<Request> = {};
        const errors: Error[] = [];
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async ({ request }) => {
            await Promise.reject(new Error('some-error'));
            processed[request.url] = request;
        };

        const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
            failed[request.url] = request;
            errors.push(error);
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            requestHandler,
            failedRequestHandler,
        });

        await basicCrawler.run();

        expect(failed['http://example.com/1'].errorMessages).toHaveLength(4);
        expect(failed['http://example.com/1'].retryCount).toBe(3);
        expect(failed['http://example.com/2'].errorMessages).toHaveLength(4);
        expect(failed['http://example.com/2'].retryCount).toBe(3);
        expect(failed['http://example.com/3'].errorMessages).toHaveLength(4);
        expect(failed['http://example.com/3'].retryCount).toBe(3);
        expect(Object.values(failed)).toHaveLength(3);
        expect(Object.values(processed)).toHaveLength(0);
        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
        errors.forEach((error) => expect(error).toBeInstanceOf(Error));
    });

    test('should not retry on NonRetryableError', async () => {
        const sources = [
            { url: 'http://example.com/1' },
            { url: 'http://example.com/2' },
            { url: 'http://example.com/3' },
        ];
        const failed: Dictionary<Request> = {};
        const errors: Error[] = [];
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async () => {
            throw new NonRetryableError('some-error');
        };

        const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
            failed[request.url] = request;
            errors.push(error);
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            requestHandler,
            failedRequestHandler,
        });

        await basicCrawler.run();

        expect(failed['http://example.com/1'].errorMessages).toHaveLength(1);
        expect(failed['http://example.com/1'].retryCount).toBe(0);
        expect(failed['http://example.com/2'].errorMessages).toHaveLength(1);
        expect(failed['http://example.com/2'].retryCount).toBe(0);
        expect(failed['http://example.com/3'].errorMessages).toHaveLength(1);
        expect(failed['http://example.com/3'].retryCount).toBe(0);
        expect(Object.values(failed)).toHaveLength(3);
        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);
        errors.forEach((error) => expect(error).toBeInstanceOf(NonRetryableError));
    });

    test('noRetry after calling errorHandler', async () => {
        const sources = [{ url: `http://example.com` }];
        const requestList = await RequestList.open(null, sources);

        let request: Request<Dictionary<any>>;

        const crawler = new BasicCrawler({
            requestList,
            errorHandler: (context, error) => {
                request = context.request;
                context.request.noRetry = true;
            },
            maxRequestRetries: 3,
            requestHandler: () => {
                throw new Error('Failure');
            },
        });

        await crawler.run();

        expect(request!.retryCount).toBe(0);
    });

    test('should crash on CriticalError', async () => {
        const sources = [
            { url: 'http://example.com/1' },
            { url: 'http://example.com/2' },
            { url: 'http://example.com/3' },
        ];
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async () => {
            throw new CriticalError('some-error');
        };

        const failedRequestHandler = vitest.fn() as ErrorHandler;

        const basicCrawler = new BasicCrawler({
            requestList,
            requestHandler,
            failedRequestHandler,
        });

        await expect(basicCrawler.run()).rejects.toThrow(CriticalError);

        expect(failedRequestHandler).not.toBeCalled();
        expect(await requestList.isFinished()).toBe(false);
    });

    test('should crash on MissingRouteError', async () => {
        const sources = [
            { url: 'http://example.com/1', label: 'TEST' }, // will match
            { url: 'http://example.com/2', label: 'FOO' }, // will fail as no FOO route or default route exists
            { url: 'http://example.com/3' }, // will fail as no default route exists
        ];
        const requestList = await RequestList.open(null, sources);

        const failedRequestHandler = vitest.fn() as ErrorHandler;

        const basicCrawler = new BasicCrawler({
            requestList,
            failedRequestHandler,
        });
        const testRoute = vitest.fn();
        basicCrawler.router.addHandler('TEST', testRoute);

        await expect(basicCrawler.run()).rejects.toThrow(MissingRouteError);

        expect(failedRequestHandler).not.toBeCalled();
        expect(testRoute).toBeCalled();
        expect(await requestList.isFinished()).toBe(false);
    });

    test('should correctly combine RequestList and RequestQueue', async () => {
        const sources = [
            { url: 'http://example.com/0' },
            { url: 'http://example.com/1' },
            { url: 'http://example.com/2' },
        ];
        const processed: Dictionary<Request> = {};
        const requestList = await RequestList.open(null, sources);
        const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });

        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed[request.url] = request;

            if (request.url === 'http://example.com/1') {
                throw Error(`This is ${request.retryCount}th error!`);
            }

            request.userData.foo = 'bar';
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            requestQueue,
            maxRequestRetries: 3,
            minConcurrency: 1,
            maxConcurrency: 1,
            requestHandler,
        });

        vitest.spyOn(requestQueue, 'handledCount').mockResolvedValueOnce(0);

        vitest
            .spyOn(requestQueue, 'addRequest')
            .mockResolvedValueOnce({ requestId: 'id-0' } as any)
            .mockResolvedValueOnce({ requestId: 'id-1' } as any)
            .mockResolvedValueOnce({ requestId: 'id-2' } as any);

        const request0 = new Request({ id: 'id-0', ...sources[0] });
        const request1 = new Request({ id: 'id-1', ...sources[1] });
        const request2 = new Request({ id: 'id-2', ...sources[2] });

        const queueContent = [request0, request1, request2, request1, request1, request1];

        vitest.spyOn(requestQueue, 'fetchNextRequest').mockImplementation(async () => queueContent.shift() ?? null);

        const markReqHandled = vitest
            .spyOn(requestQueue, 'markRequestHandled')
            .mockReturnValue(Promise.resolve() as any);
        const reclaimReq = vitest.spyOn(requestQueue, 'reclaimRequest').mockReturnValue(Promise.resolve() as any);

        vitest.spyOn(requestQueue, 'isEmpty').mockImplementation(async () => queueContent.length <= 0);
        vitest.spyOn(requestQueue, 'isFinished').mockResolvedValueOnce(true);

        await basicCrawler.run();

        // 1st try

        expect(reclaimReq).toBeCalledWith(request1, expect.objectContaining({}));
        expect(reclaimReq).toBeCalledTimes(3);

        expect(processed['http://example.com/0'].userData.foo).toBe('bar');
        expect(processed['http://example.com/0'].errorMessages).toEqual([]);
        expect(processed['http://example.com/0'].retryCount).toBe(0);
        expect(processed['http://example.com/2'].userData.foo).toBe('bar');
        expect(processed['http://example.com/2'].errorMessages).toEqual([]);
        expect(processed['http://example.com/2'].retryCount).toBe(0);

        expect(processed['http://example.com/1'].userData.foo).toBeUndefined();
        expect(processed['http://example.com/1'].errorMessages).toHaveLength(4);
        expect(processed['http://example.com/1'].retryCount).toBe(3);

        expect(await requestList.isFinished()).toBe(true);
        expect(await requestList.isEmpty()).toBe(true);

        vitest.restoreAllMocks();
    });

    test('should say that task is not ready requestList is not set and requestQueue is empty', async () => {
        const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
        requestQueue.isEmpty = async () => Promise.resolve(true);

        const crawler = new BasicCrawler({
            requestQueue,
            requestHandler: async () => {},
        });

        // @ts-expect-error Accessing private prop
        expect(await crawler._isTaskReadyFunction()).toBe(false);
    });

    test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {
        const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
        const processed: Request[] = [];
        const queue: Request[] = [];
        let isFinished = false;
        let isFinishedFunctionCalled = false;
        let isTaskReadyFunctionCalled = false;

        const basicCrawler = new BasicCrawler({
            requestQueue,
            autoscaledPoolOptions: {
                minConcurrency: 1,
                maxConcurrency: 1,
                isFinishedFunction: async () => {
                    isFinishedFunctionCalled = true;
                    return Promise.resolve(isFinished);
                },
                isTaskReadyFunction: async () => {
                    isTaskReadyFunctionCalled = true;
                    return Promise.resolve(!isFinished);
                },
            },
            requestHandler: async ({ request }) => {
                await sleep(10);
                processed.push(request);
            },
        });

        // Speed up the test
        // @ts-expect-error Accessing private prop
        basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;

        const request0 = new Request({ url: 'http://example.com/0' });
        const request1 = new Request({ url: 'http://example.com/1' });

        vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);

        let handledCount = 0;
        const markRequestHandled = vitest.spyOn(requestQueue, 'markRequestHandled').mockImplementation(async () => {
            handledCount++;
            // Only set isFinished after both requests have been handled
            if (handledCount >= 2) {
                // Small delay to ensure the test can verify everything
                setTimeout(() => {
                    isFinished = true;
                }, 50);
            }
            return Promise.resolve() as any;
        });

        const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');

        requestQueue.fetchNextRequest = async () => queue.pop()!;
        requestQueue.isEmpty = async () => Promise.resolve(!queue.length);

        // Add requests with buffer time for crawler startup.
        // Use longer delays to avoid flakiness under CPU load from parallel tests.
        setTimeout(() => queue.push(request0), 500);
        setTimeout(() => queue.push(request1), 1000);

        await basicCrawler.run();

        expect(markRequestHandled).toBeCalledWith(request0);
        expect(markRequestHandled).toBeCalledWith(request1);
        expect(isFinishedOrig).not.toBeCalled();
        expect(isFinishedFunctionCalled).toBe(true);
        expect(isTaskReadyFunctionCalled).toBe(true);

        // TODO: see why the request1 was passed as a second parameter to includes
        expect(processed.includes(request0)).toBe(true);

        vitest.restoreAllMocks();
    });

    test('keepAlive', async () => {
        const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
        const processed: Request[] = [];
        const queue: Request[] = [];

        const basicCrawler = new BasicCrawler({
            requestQueue,
            keepAlive: true,
            requestHandler: async ({ request }) => {
                await sleep(10);
                processed.push(request);
            },
        });

        // Speed up the test
        // @ts-expect-error Accessing private prop
        basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;

        const request0 = new Request({ url: 'http://example.com/0' });
        const request1 = new Request({ url: 'http://example.com/1' });

        vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);
        const markRequestHandled = vitest
            .spyOn(requestQueue, 'markRequestHandled')
            .mockReturnValue(Promise.resolve() as any);

        const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');

        requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()!);
        requestQueue.isEmpty = async () => Promise.resolve(!queue.length);

        // Use longer delays to avoid flakiness under CPU load from parallel tests.
        setTimeout(() => queue.push(request0), 500);
        setTimeout(() => queue.push(request1), 1000);
        setTimeout(() => {
            void basicCrawler.teardown();
        }, 3000);

        await basicCrawler.run();

        expect(markRequestHandled).toBeCalledWith(request0);
        expect(markRequestHandled).toBeCalledWith(request1);
        expect(isFinishedOrig).not.toBeCalled();

        // TODO: see why the request1 was passed as a second parameter to includes
        expect(processed.includes(request0)).toBe(true);

        vitest.restoreAllMocks();
    });

    test('should support maxRequestsPerCrawl parameter', async () => {
        const sources = [
            { url: 'http://example.com/1' },
            { url: 'http://example.com/2' },
            { url: 'http://example.com/3' },
            { url: 'http://example.com/4' },
            { url: 'http://example.com/5' },
        ];
        const processed: Dictionary<Request> = {};
        const requestList = await RequestList.open(null, sources);

        const requestHandler: RequestHandler = async ({ request }) => {
            await sleep(10);
            processed[request.url] = request;
            if (request.url === 'http://example.com/2') throw Error();
            request.userData.foo = 'bar';
        };

        let failedRequestHandlerCalls = 0;
        const failedRequestHandler = async () => {
            failedRequestHandlerCalls++;
        };

        const basicCrawler = new BasicCrawler({
            requestList,
            maxRequestRetries: 3,
            maxRequestsPerCrawl: 3,
            maxConcurrency: 1,
            requestHandler,
            failedRequestHandler,
        });

        await basicCrawler.run();

        expect(processed['http://example.com/1'].userData.foo).toBe('bar');
        expect(processed['http://example.com/1'].errorMessages).toEqual([]);
        expect(processed['http://example.com/1'].retryCount).toBe(0);
        expect(processed['http://example.com/3'].userData.foo).toBe('bar');
        expect(processed['http://example.com/3'].errorMessages).toEqual([]);
        expect(processed['http://example.com/3'].retryCount).toBe(0);

        expect(processed['http://example.com/2'].userData.foo).toEqual(undefined);
        expect(processed['http://example.com/2'].errorMessages).toHaveLength(4);
        expect(processed['http://example.com/2'].retryCount).toBe(3);

        expect(failedRequestHandlerCalls).toBe(1);

        expect(await requestList.isFinished()).toBe(false);
        expect(await requestList.isEmpty()).toBe(false);
    });

    test('should derive handledRequestCount from Statistics', async () => {
        const requestQueue = new RequestQueue({ id: 'id', client: Configuration.getStorageClient() });
        requestQueue.isEmpty = async () => false;
        requestQueue.isFinished = async () => false;

        requestQueue.fetchNextRequest = async () => new Request({ id: 'id', url: 'http://example.com' });
        // @ts-expect-error Overriding the method for testing purposes
        requestQueue.markRequestHandled = async () => {};

        // Even though the request queue reports 33 handled requests (e.g. from a previous crawler run),
        // the crawler should use its own Statistics to track the count and process all 40 requests.
        vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);

        let count = 0;
        const crawler = new BasicCrawler({
            requestQueue,
            maxConcurrency: 1,
            requestHandler: async () => {
                await sleep(1);
                count++;
            },
            maxRequestsPerCrawl: 40,
        });

        await crawler.run();
        // The crawler should have processed 40 requests (its own limit), not 7 (40 - 33).
        expect(count).toBe(40);
        vitest.restoreAllMocks();
    });

    test('should timeout after handleRequestTimeoutSecs', async () => {
        const url = 'https://example.com';
        const requestList = await RequestList.open({ sources: [{ url }] });

        const results: Request[] = [];
        const crawler = new BasicCrawler({
            requestList,
            handleRequestTimeoutSecs: 0.01,
            maxRequestRetries: 1,
            requestHandler: async () => sleep(1000),
            failedRequestHandler: async ({ request }) => {
                results.push(request);
            },
        });

        await crawler.run();
        expect(results).toHaveLength(1);
        expect(results[0].url).toEqual(url);
        results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));
    });

    test('limits handleRequestTimeoutSecs and derived vars to a valid value', async () => {
        const url = 'https://example.com';
        const requestList = await RequestList.open({ sources: [{ url }] });

        const results = [];
        const crawler = new BasicCrawler({
            requestList,
            requestHandlerTimeoutSecs: Infinity,
            maxRequestRetries: 1,
            requestHandler: async () => sleep(1000),
            failedRequestHandler: async ({ request }) => {
                results.push(request);
            },
        });

        const maxSignedInteger = 2 ** 31 - 1;
        // @ts-expect-error Accessing private prop
        expect(crawler.requestHandlerTimeoutMillis).toBe(maxSignedInteger);
        // @ts-expect-error Accessing private prop
        expect(crawler.internalTimeoutMillis).toBe(maxSignedInteger);
    });

    test('should not log stack trace for timeout errors by default', async () => {
        const sources = [{ url: `http://${HOSTNAME}:${port}` }];
        const requestList = await RequestList.open(null, sources);

        const crawler = new BasicCrawler({
            requestList,
            requestHandlerTimeoutSecs: 0.1,
            maxRequestRetries: 3,
            requestHandler: async () => sleep(1e3),
        });

        const warningSpy = vitest.spyOn(crawler.log, 'warning');
        const errorSpy = vitest.spyOn(crawler.log, 'error');

        await crawler.run();

        expect(warningSpy.mock.calls.length).toBe(3);
        for (const args of warningSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
            expect(/requestHandler timed out after/.test(args[0])).toBe(true);
            expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);
            expect(args[1]).toBeDefined();
        }

        expect(errorSpy.mock.calls.length).toBe(1);
        for (const args of errorSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
            expect(/requestHandler timed out after/.test(args[0])).toBe(true);
            expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);
            expect(args[1]).toBeDefined();
        }
    });

    test('should log stack trace for non-timeout errors only when request will no longer be retried by default', async () => {
        const sources = [{ url: `http://${HOSTNAME}:${port}` }];
        const requestList = await RequestList.open(null, sources);

        const crawler = new BasicCrawler({
            requestList,
            maxRequestRetries: 3,
            requestHandler: () => {
                throw new Error('Other non-timeout error');
            },
        });

        const warningSpy = vitest.spyOn(crawler.log, 'warning');
        const errorSpy = vitest.spyOn(crawler.log, 'error');

        await crawler.run();

        expect(warningSpy.mock.calls.length).toBe(3);
        for (const args of warningSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
            expect(/Other non-timeout error/.test(args[0])).toBe(true);
            expect(args[0].split('\n').length).toBeLessThanOrEqual(2);
            expect(args[1]).toBeDefined();
        }

        expect(errorSpy.mock.calls.length).toBe(1);
        for (const args of errorSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
            expect(/Other non-timeout error/.test(args[0])).toBe(true);
            expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
            expect(args[1]).toBeDefined();
        }
    });

    test('should log stack trace for timeout errors when verbose log is enabled', async () => {
        log.setLevel(log.LEVELS.INFO);
        process.env.CRAWLEE_VERBOSE_LOG = 'true';
        const sources = [{ url: `http://${HOSTNAME}:${port}` }];
        const requestList = await RequestList.open(null, sources);

        const crawler = new BasicCrawler({
            requestList,
            requestHandlerTimeoutSecs: 0.1,
            maxRequestRetries: 3,
            requestHandler: async () => sleep(1e3),
        });

        const warningSpy = vitest.spyOn(crawler.log, 'warning');
        const errorSpy = vitest.spyOn(crawler.log, 'error');

        await crawler.run();

        expect(warningSpy.mock.calls.length).toBe(3);
        for (const args of warningSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
            expect(/requestHandler timed out after/.test(args[0])).toBe(true);
            expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);
            expect(args[1]).toBeDefined();
        }

        expect(errorSpy.mock.calls.length).toBe(1);
        for (const args of errorSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
            expect(/requestHandler timed out after/.test(args[0])).toBe(true);
            expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);
            expect(args[1]).toBeDefined();
        }

        log.setLevel(log.LEVELS.OFF);
        process.env.CRAWLEE_VERBOSE_LOG = undefined;
    });

    test('should log stack trace for non-timeout errors when verbose log is enabled', async () => {
        log.setLevel(log.LEVELS.INFO);
        process.env.CRAWLEE_VERBOSE_LOG = 'true';
        const sources = [{ url: `http://${HOSTNAME}:${port}` }];
        const requestList = await RequestList.open(null, sources);

        const crawler = new BasicCrawler({
            requestList,
            maxRequestRetries: 3,
            requestHandler: () => {
                throw new Error('Other non-timeout error');
            },
        });

        const warningSpy = vitest.spyOn(crawler.log, 'warning');
        const errorSpy = vitest.spyOn(crawler.log, 'error');

        await crawler.run();

        expect(warningSpy.mock.calls.length).toBe(3);
        for (const args of warningSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
            expect(/Other non-timeout error/.test(args[0])).toBe(true);
            expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
            expect(args[1]).toBeDefined();
        }

        expect(errorSpy.mock.calls.length).toBe(1);
        for (const args of errorSpy.mock.calls) {
            expect(args.length).toBe(2);
            expect(typeof args[0]).toBe('string');
            expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
            expect(/Other non-timeout error/.test(args[0])).toBe(true);
            expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
            expect(args[1]).toBeDefined();
        }

        log.setLevel(log.LEVELS.OFF);
        process.env.CRAWLEE_VERBOSE_LOG = undefined;
    });

    describe('Uses SessionPool', () => {
        it('should use SessionPool when useSessionPool is true ', async () => {
            const url = 'https://example.com';
            const requestList = await RequestList.open({ sources: [{ url }] });
            const results: Request[] = [];

            const crawler = new BasicCrawler({
                requestList,
                handleRequestTimeoutSecs: 0.01,
                maxRequestRetries: 1,
                useSessionPool: true,
                sessionPoolOptions: {
                    maxPoolSize: 10,
                    persistStateKey: 'POOL',
                },
                requestHandler: async ({ session }) => {
                    expect(session!.constructor.name).toEqual('Session');
                    expect(session!.id).toBeDefined();
                },
                failedRequestHandler: async ({ request }) => {
                    results.push(request);
                },
            });

            await crawler.run();
            expect(crawler.sessionPool).toBeDefined();
            expect(results).toHaveLength(0);
        });

        it('should use pass options to sessionPool', async () => {
            const url = 'https://example.com';
            const requestList = await RequestList.open({ sources: [{ url }] });

            const crawler = new BasicCrawler({
                requestList,
                handleRequestTimeoutSecs: 0.01,
                maxRequestRetries: 1,
                useSessionPool: true,
                sessionPoolOptions: {
                    maxPoolSize: 10,
                    persistStateKey: 'POOL',
                },
                requestHandler: async () => {},
                failedRequestHandler: async () => {},
            });
            await crawler.run();

            // @ts-expect-error private symbol
            expect(crawler.sessionPool.maxPoolSize).toEqual(10);
        });

        it('should destroy Session pool after it is finished', async () => {
            const url = 'https://example.com';
            const requestList = await RequestList.open({ sources: [{ url }] });
            events.off(EventType.PERSIST_STATE);

            const crawler = new BasicCrawler({
                requestList,
                handleRequestTimeoutSecs: 0.01,
                maxRequestRetries: 1,
                useSessionPool: true,
                sessionPoolOptions: {
                    maxPoolSize: 10,
                },
                requestHandler: async () => {},
                failedRequestHandler: async () => {},
            });

            await crawler.run();
            expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);
            // @ts-expect-error private symbol
            expect(crawler.sessionPool.maxPoolSize).toEqual(10);
        });
    });

    describe('CrawlingContext', () => {
        test('should be kept and later deleted', async () => {
            const urls = [
                'https://example.com/0',
                'https://example.com/1',
                'https://example.com/2',
                'https://example.com/3',
            ];
            const requestList = await RequestList.open(null, urls);
            let counter = 0;
            let finish: (value?: unknown) => void;
            const allFinishedPromise = new Promise((resolve) => {
                finish = resolve;
            });
            const mainContexts: CrawlingContext[] = [];
            const otherContexts: CrawlingContext[][] = [];
            const crawler = new BasicCrawler({
                requestList,
                minConcurrency: 4,
                async requestHandler(crawlingContext) {
                    // @ts-expect-error Accessing private prop
                    mainContexts[counter] = crawler.crawlingContexts.get(crawlingContext.id);
                    // @ts-expect-error Accessing private prop
                    otherContexts[counter] = Array.from(crawler.crawlingContexts).map(([, v]) => v);
                    counter++;
                    if (counter === 4) finish();
                    await allFinishedPromise;
                },
            });
            await crawler.run();

            expect(counter).toBe(4);
            expect(mainContexts).toHaveLength(4);
            expect(otherContexts).toHaveLength(4);
            // @ts-expect-error Accessing private prop
            expect(crawler.crawlingContexts.size).toBe(0);
            mainContexts.forEach((ctx, idx) => {
                expect(typeof ctx.id).toBe('string');
                expect(otherContexts[idx]).toContain(ctx);
            });
            otherContexts.forEach((list, idx) => {
                expect(list).toHaveLength(idx + 1);
            });
        });
    });

    describe('sendRequest', () => {
        const html = `<!DOCTYPE html><html><head><title>foobar</title></head><body><p>Hello, world!</p></body></html>`;

        const httpServer = http.createServer((request, response) => {
            response.setHeader('content-type', 'text/html');
            response.end(html);
        });

        let url: string;

        beforeAll(async () => {
            await new Promise<void>((resolve) => {
                httpServer.listen(0, () => {
                    url = `http://127.0.0.1:${(httpServer.address() as AddressInfo).port}/`;

                    resolve();
                });
            });
        });

        afterAll(async () => {
            await new Promise((resolve) => httpServer.close(resolve));
        });

        test('works', async () => {
            const responses: { statusCode: number; body: string }[] = [];

            const requestList = await RequestList.open(null, [url]);

            const crawler = new BasicCrawler({
                useSessionPool: true,
                requestList,
                async requestHandler({ sendRequest }) {
                    const response = await sendRequest();

                    responses.push({
                        statusCode: response.statusCode,
                        body: response.body,
                    });
                },
            });

            await crawler.run();

            expect(responses).toStrictEqual([
                {
                    statusCode: 200,
                    body: html,
                },
            ]);
        });

        test('works without session', async () => {
            const requestList = await RequestList.open(null, [url]);

            const responses: { statusCode: number; body: string }[] = [];

            const crawler = new BasicCrawler({
                useSessionPool: false,
                requestList,
                async requestHandler({ sendRequest }) {
                    const response = await sendRequest();

                    responses.push({
                        statusCode: response.statusCode,
                        body: response.body,
                    });
                },
            });

            await crawler.run();

            expect(responses).toStrictEqual([
                {
                    statusCode: 200,
                    body: html,
                },
            ]);
        });

        test('proxyUrl TypeScript support', async () => {
            const crawler = new BasicCrawler({
                useSessionPool: true,
                async requestHandler({ sendRequest }) {
                    await sendRequest({
                        proxyUrl: 'http://example.com',
                    });
                },
            });

            expect(crawler).toBeTruthy();
        });
    });

    describe('Request enqueuing limits with maxRequestsPerCrawl', () => {
        test('should not enqueue more requests than maxRequestsPerCrawl allows', async () => {
            const requestQueue = await RequestQueue.open();
            const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');

            const crawler = new BasicCrawler({
                requestQueue,
                maxRequestsPerCrawl: 5,
                requestHandler: async () => {},
            });

            crawler.stats.state.requestsFinished = 2;

            // Try to add 6 requests - should only add 3 due to limit
            const requestsToAdd = [
                'http://example.com/1',
                'http://example.com/2',
                'http://example.com/3',
                'http://example.com/4',
                'http://example.com/5',
                'http://example.com/6',
            ];

            await crawler.addRequests(requestsToAdd);

            // Should only have added the first 3 requests (since 2 were already processed, limit allows 3 more)
            expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
                { url: 'http://example.com/1' },
                { url: 'http://example.com/2' },
                { url: 'http://example.com/3' },
            ]);
        });

        test('should not enqueue more requests than maxRequestsPerCrawl allows across multiple addRequests calls', async () => {
            const requestQueue = await RequestQueue.open();
            const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');

            const crawler = new BasicCrawler({
                requestQueue,
                maxRequestsPerCrawl: 5,
                requestHandler: async () => {},
            });

            crawler.stats.state.requestsFinished = 1;

            // First call - should add 2 requests (2 more slots to go)
            await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);

            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
                { url: 'http://example.com/1' },
                { url: 'http://example.com/2' },
            ]);

            // Second call - should add only 2 more requests
            await crawler.addRequests([
                'http://example.com/3',
                'http://example.com/4',
                'http://example.com/5', // This should be ignored
                'http://example.com/6', // This should be ignored
            ]);

            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
                { url: 'http://example.com/1' },
                { url: 'http://example.com/2' },
                { url: 'http://example.com/3' },
                { url: 'http://example.com/4' },
            ]);

            // Third call - should add no requests (limit already reached)
            await crawler.addRequests(['http://example.com/7', 'http://example.com/8']);

            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
                { url: 'http://example.com/1' },
                { url: 'http://example.com/2' },
                { url: 'http://example.com/3' },
                { url: 'http://example.com/4' },
            ]);

            expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(3);
        });

        test('should respect robots.txt when limiting requests', async () => {
            const requestQueue = await RequestQueue.open();
            const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');

            const crawler = new BasicCrawler({
                requestQueue,
                maxRequestsPerCrawl: 2,
                respectRobotsTxtFile: true,
                requestHandler: async () => {},
            });

            crawler.stats.state.requestsFinished = 0;

            // Mock robots.txt checking to disallow some URLs
            vitest.spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile').mockImplementation(async (url) => {
                return url !== 'http://example.com/2';
            });

            await crawler.addRequests([
                'http://example.com/1', // Allowed by robots.txt
                'http://example.com/2', // Blocked by robots.txt
                'http://example.com/3', // Allowed by robots.txt && within limit
                'http://example.com/4', // Would exceed limit
            ]);

            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
                { url: 'http://example.com/1' },
                { url: 'http://example.com/3' },
            ]);

            // Should only have added the first request (allowed by robots.txt and within limit)
            expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
        });

        test.each([
            {
                testName: 'custom user-agent robots.txt rules',
                userAgent: 'MyCrawler',
                visitedUrls: [
                    {
                        url: 'http://example.com/yes',
                    },
                    {
                        url: 'http://example.com/my-crawler/anything',
                    },
                ],
            },
            {
                testName: 'catch-all robots.txt rules with custom user-agent',
                userAgent: 'RandomCrawler',
                visitedUrls: [
                    {
                        url: 'http://example.com/yes',
                    },
                ],
            },
        ])('should respect $testName', async ({ userAgent, visitedUrls }) => {
            const requestQueue = await RequestQueue.open();
            const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');

            const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
                override async getRobotsTxtFileForUrl(_: string) {
                    return RobotsTxtFile.from(
                        'http://example.com/robots.txt',
                        `User-agent: *
                         Disallow: /
                         Allow: /yes

                         User-agent: MyCrawler
                         Disallow: /no
                         Allow: /my-crawler
                        `,
                    );
                }
            })({
                requestQueue,
                respectRobotsTxtFile: { userAgent },
                requestHandler: async () => {},
            });

            await crawler.addRequests([
                'http://example.com/yes', // Allowed by robots.txt
                'http://example.com/no', // Blocked by robots.txt for "MyCrawler"
                'http://example.com/no-globally', // Blocked by robots.txt, "*" rule
                'http://example.com/my-crawler/anything', // Blocked by robots.txt for all user-agents, but allowed for "MyCrawler"
            ]);

            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject(visitedUrls);

            // Should only have added the first request (allowed by robots.txt and within limit)
            expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
        });

        test('enqueueLinks should respect custom user-agent robots.txt rules', async () => {
            const requestQueue = await RequestQueue.open();
            const visitedUrls: string[] = [];

            const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
                override async getRobotsTxtFileForUrl(_: string) {
                    return RobotsTxtFile.from(
                        'http://example.com/robots.txt',
                        `User-agent: *
                         Disallow: /
                         Allow: /yes

                         User-agent: MyCrawler
                         Disallow: /no
                         Allow: /my-crawler
                        `,
                    );
                }
            })({
                requestQueue,
                maxConcurrency: 1,
                respectRobotsTxtFile: { userAgent: 'MyCrawler' },
                requestHandler: async (context) => {
                    visitedUrls.push(context.request.url);

                    if (context.request.label) {
                        return;
                    }

                    await context.enqueueLinks({
                        urls: [
                            'http://example.com/yes',
                            'http://example.com/no',
                            'http://example.com/no-globally',
                            'http://example.com/my-crawler/anything',
                        ],
                        label: 'child',
                    });
                },
            });

            await crawler.run(['http://example.com/start']);

            expect(visitedUrls).toEqual([
                'http://example.com/start',
                'http://example.com/yes',
                'http://example.com/my-crawler/anything',
            ]);
        });

        test('enqueueLinks forwards respectRobotsTxtFile.userAgent to the robots.txt check', async () => {
            const requestQueue = await RequestQueue.open();
            const isAllowedSpy = vitest.fn(() => true);

            const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
                override async getRobotsTxtFileForUrl(_: string) {
                    return { isAllowed: isAllowedSpy } as unknown as RobotsTxtFile;
                }
            })({
                requestQueue,
                maxConcurrency: 1,
                respectRobotsTxtFile: { userAgent: 'MyCrawler' },
                requestHandler: async (context) => {
                    if (context.request.label) return;
                    await context.enqueueLinks({ urls: ['http://example.com/child'], label: 'child' });
                },
            });

            await crawler.run(['http://example.com/start']);

            expect(isAllowedSpy).toHaveBeenCalledWith('http://example.com/child', 'MyCrawler');
        });

        test('enqueueLinks should respect maxRequestsPerCrawl', async () => {
            const requestQueue = await RequestQueue.open();
            const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');

            // Will try to add 6 requests - should only add 3 due to limit
            const requestsToAdd = [
                'http://example.com/1',
                'http://example.com/2',
                'http://example.com/3',
                'http://example.com/4',
                'http://example.com/5',
                'http://example.com/6',
            ];
            const visitedUrls: string[] = [];

            const crawler = new BasicCrawler({
                requestQueue,
                maxRequestsPerCrawl: 5,
                requestHandler: async (context) => {
                    visitedUrls.push(context.request.url);

                    if (context.request.label) {
                        return;
                    }

                    crawler.stats.state.requestsFinished = 2;

                    await context.enqueueLinks({ urls: requestsToAdd, label: 'not-undefined' });
                },
            });

            await crawler.run(['http://example.com']);

            expect(visitedUrls).toEqual([
                'http://example.com', // added by crawler.run()
                'http://example.com/1',
                'http://example.com/2',
            ]);

            // Should only have added the first 2 requests (since 2 were already processed and 1 is in progress, limit allows 2 more)
            expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2);
        });

        test('enqueueLinks limit log message should only be logged once', async () => {
            const requestQueue = await RequestQueue.open();

            // Will try to add 10 requests with a limit of 2
            const requestsToAdd = Array.from({ length: 10 }, (_, i) => `http://example.com/${i}`);

            const crawler = new BasicCrawler({
                requestQueue,
                requestHandler: async (context) => {
                    if (context.request.label) {
                        return;
                    }

                    await context.enqueueLinks({ urls: requestsToAdd, limit: 2, label: 'child' });
                },
            });

            const infoSpy = vitest.spyOn(crawler.log, 'info');

            await crawler.run(['http://example.com']);

            // The enqueueLinks limit message should only appear once, not 8 times (for each skipped request)
            const enqueueLimitMessages = infoSpy.mock.calls.filter(
                (call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
            );
            expect(enqueueLimitMessages).toHaveLength(1);
        });

        test('enqueueLinks limit log message should be logged again on subsequent runs', async () => {
            const requestQueue = await RequestQueue.open();

            const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/${i}`);

            const crawler = new BasicCrawler({
                requestQueue,
                requestHandler: async (context) => {
                    if (context.request.label) {
                        return;
                    }

                    await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
                },
            });

            const infoSpy = vitest.spyOn(crawler.log, 'info');

            // First run
            await crawler.run(['http://example.com/first']);

            // Second run with a new start URL
            await crawler.run(['http://example.com/second']);

            // The enqueueLinks limit message should appear twice (once per run)
            const enqueueLimitMessages = infoSpy.mock.calls.filter(
                (call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
            );
            expect(enqueueLimitMessages).toHaveLength(2);
        });

        test('enqueueLinks limit log message should be logged once per request handler, not once per run', async () => {
            const requestQueue = await RequestQueue.open();

            // Each handler will try to add 5 URLs with a limit of 1
            const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);

            const crawler = new BasicCrawler({
                requestQueue,
                requestHandler: async (context) => {
                    if (context.request.label === 'child') {
                        return;
                    }

                    await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
                },
            });

            const infoSpy = vitest.spyOn(crawler.log, 'info');

            // Single run with two initial requests - both will trigger the limit
            await crawler.run(['http://example.com/first', 'http://example.com/second']);

            // The enqueueLinks limit message should appear twice (once per request handler that triggered the limit)
            const enqueueLimitMessages = infoSpy.mock.calls.filter(
                (call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
            );
            expect(enqueueLimitMessages).toHaveLength(2);
        });

        test('maxCrawlDepth limit log message should only be logged once per run', async () => {
            const requestQueue = await RequestQueue.open();

            // Each handler will try to add URLs that exceed maxCrawlDepth
            const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);

            const crawler = new BasicCrawler({
                requestQueue,
                maxCrawlDepth: 1, // Only allow depth 0 (initial) and depth 1 (first level children)
                requestHandler: async (context) => {
                    // Stop processing at depth 2 to avoid infinite loops
                    if (context.request.crawlDepth >= 2) {
                        return;
                    }

                    // This will add requests at depth+1, so initial requests add at depth 1 (allowed)
                    // and depth 1 requests add at depth 2 (blocked by maxCrawlDepth)
                    await context.enqueueLinks({
                        urls: requestsToAdd,
                        label: `depth-${context.request.crawlDepth + 1}`,
                    });
                },
            });

            const infoSpy = vitest.spyOn(crawler.log, 'info');

            // Run with two initial requests
            // Each will enqueue children at depth 1, then those children will try to enqueue at depth 2 (blocked)
            await crawler.run(['http://example.com/first', 'http://example.com/second']);

            // The maxCrawlDepth limit message should only appear once per run, even though multiple requests triggered it
            const maxCrawlDepthMessages = infoSpy.mock.calls.filter(
                (call) => typeof call[0] === 'string' && call[0].includes('maxCrawlDepth'),
            );
            expect(maxCrawlDepthMessages).toHaveLength(1);
        });

        test('should not count duplicate URLs toward maxRequestsPerCrawl limit (addRequests)', async () => {
            const requestQueue = await RequestQueue.open();

            const crawler = new BasicCrawler({
                requestQueue,
                maxRequestsPerCrawl: 5,
                requestHandler: async () => {},
            });

            // 10 duplicate links to the same URL + 1 unique link at the end
            const requestsToAdd = [
                ...Array.from({ length: 10 }, () => 'http://example.com/same'),
                'http://example.com/new',
            ];

            await crawler.addRequests(requestsToAdd);

            // Both unique URLs should have been enqueued — duplicates should not consume the budget
            await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
                { url: 'http://example.com/same' },
                { url: 'http://example.com/new' },
            ]);
        });

        test('addRequestsBatched with maxNewRequests should correctly report requestsOverLimit for array input', async () => {
            const queue = await RequestQueue.open();

            const result = await queue.addRequestsBatched(
                [
                    { url: 'http://example.com/a' },
                    { url: 'http://example.com/b' },
                    { url: 'http://example.com/c' },
                    { url: 'http://example.com/d' },
                    { url: 'http://example.com/e' },
                ],
                { maxNewRequests: 2 },
            );

            const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);

            const overLimitUrls = (result.requestsOverLimit ?? []).map((r) => (typeof r === 'string' ? r : r.url));

            expect(addedUrls).toHaveLength(2);
            expect(overLimitUrls).toHaveLength(3);
        });

        test('addRequestsBatched with maxNewRequests should correctly report requestsOverLimit for generator input', async () => {
            const queue = await RequestQueue.open();

            async function* urls() {
                yield { url: 'http://example.com/a' };
                yield { url: 'http://example.com/b' };
                yield { url: 'http://example.com/c' };
                yield { url: 'http://example.com/d' };
                yield { url: 'http://example.com/e' };
            }

            const result = await queue.addRequestsBatched(urls(), { maxNewRequests: 2 });

            const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);

            const overLimitUrls = (result.requestsOverLimit ?? []).map((r) => (typeof r === 'string' ? r : r.url));

            expect(addedUrls).toHaveLength(2);
            expect(overLimitUrls).toHaveLength(3);
        });

        test('should not count duplicate URLs toward maxRequestsPerCrawl limit (enqueueLinks)', async () => {
            const requestQueue = await RequestQueue.open();

            const visitedUrls: string[] = [];

            const crawler = new BasicCrawler({
                requestQueue,
                maxRequestsPerCrawl: 5,
                requestHandler: async (context) => {
                    visitedUrls.push(context.request.url);

                    if (context.request.label) {
                        return;
                    }

                    // Enqueue 10 duplicate links + 1 new unique link
                    const urls = [...Array.from({ length: 10 }, () => 'http://example.com/'), 'http://example.com/new'];

                    await context.enqueueLinks({ urls, label: 'child' });
                },
            });

            await crawler.run(['http://example.com/']);

            // Both the start URL and the new URL should have been visited
            expect(visitedUrls).toContain('http://example.com/');
            expect(visitedUrls).toContain('http://example.com/new');
        });
    });

    describe('addRequests input validation', () => {
        test('should throw error when passed a non-iterable value', async () => {
            const crawler = new BasicCrawler({
                requestHandler: async () => {},
            });

            await expect(crawler.addRequests(new WeakSet() as any)).rejects.toThrow(
                'Expected an iterable or async iterable, got weakset',
            );
        });
    });

    describe('Dataset helpers, crawler parallelism', () => {
        const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }];
        const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }];

        const tmpDir = `${__dirname}/tmp/foo/bar`;

        beforeAll(async () => {
            await rm(tmpDir, { recursive: true, force: true });
        });

        test('should expose default Dataset methods', async () => {
            const crawler = new BasicCrawler();

            await crawler.pushData(payload);

            expect((await crawler.getData()).items).toEqual(payload);
        });

        test('export data', async () => {
            const row: Dictionary = { foo: 'bar', baz: 123 };
            const crawler = new BasicCrawler();

            await crawler.pushData(row);
            await crawler.pushData(row);
            await crawler.pushData(row);

            await crawler.exportData(`${tmpDir}/result.csv`);
            await crawler.exportData(`${tmpDir}/result.json`);

            const csv = await readFile(`${tmpDir}/result.csv`);
            expect(csv.toString()).toBe('foo,baz\nbar,123\nbar,123\nbar,123\n');
            const json = await readFile(`${tmpDir}/result.json`);
            expect(json.toString()).toBe(
                '[\n' +
                    '    {\n' +
                    '        "foo": "bar",\n' +
                    '        "baz": 123\n' +
                    '    },\n' +
                    '    {\n' +
                    '        "foo": "bar",\n' +
                    '        "baz": 123\n' +
                    '    },\n' +
                    '    {\n' +
                    '        "foo": "bar",\n' +
                    '        "baz": 123\n' +
                    '    }\n' +
                    ']\n',
            );

            await rm(`${tmpDir}/result.csv`);
            await rm(`${tmpDir}/result.json`);
        });

        test('exports do not fail on empty dataset', async () => {
            const crawler = new BasicCrawler();

            await crawler.exportData(`${tmpDir}/result.csv`);
            await crawler.exportData(`${tmpDir}/result.json`);

            const csv = await readFile(`${tmpDir}/result.csv`);
            expect(csv.toString()).toBe('');
            const json = await readFile(`${tmpDir}/result.json`);
            expect(json.toString()).toBe('[]\n');

            await rm(`${tmpDir}/result.csv`);
            await rm(`${tmpDir}/result.json`);
        });

        test('should expose pushData helper', async () => {
            const crawler = new BasicCrawler({
                requestHandler: async ({ pushData }) => pushData(payload),
            });

            await crawler.run([
                {
                    url: `http://${HOSTNAME}:${port}`,
                },
            ]);

            expect((await crawler.getData()).items).toEqual(payload);
        });

        test('crawler.exportData works with `collectAllKeys`', async () => {
            const crawler = new BasicCrawler();
            await crawler.pushData([{ foo: 'bar', baz: 123 }]);
            await crawler.pushData([{ foo: 'baz', qux: 456 }]);

            await crawler.exportData(`${tmpDir}/result.csv`, 'csv', { collectAllKeys: true });

            const csv = await readFile(`${tmpDir}/result.csv`);
            expect(csv.toString()).toBe('foo,baz,qux\nbar,123,\nbaz,,456\n');

            await rm(`${tmpDir}/result.csv`);
        });

        test("Crawlers with different Configurations don't share Datasets", async () => {
            const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false }));
            const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));

            await crawlerA.pushData(getPayload('A'));
            await crawlerB.pushData(getPayload('B'));

            expect((await crawlerA.getData()).items).toEqual(getPayload('A'));

            expect((await crawlerB.getData()).items).toEqual(getPayload('B'));
        });

        test('Crawlers with different Configurations run separately', async () => {
            const crawlerA = new BasicCrawler(
                { requestHandler: () => {} },
                new Configuration({ persistStorage: false }),
            );
            const crawlerB = new BasicCrawler(
                { requestHandler: () => {} },
                new Configuration({ persistStorage: false }),
            );

            await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
            await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);

            expect(crawlerA.stats.state.requestsFinished).toBe(1);
            expect(crawlerB.stats.state.requestsFinished).toBe(1);
        });

        test('Crawlers with different Configurations does not use global Configuration', async () => {
            const getGlobalConfigSpy = vitest.spyOn(Configuration, 'getGlobalConfig');

            const configA = new Configuration({ persistStorage: false });
            const crawlerA = new BasicCrawler({ requestHandler: () => {} }, configA);
            const configB = new Configuration({ persistStorage: false });
            const crawlerB = new BasicCrawler({ requestHandler: () => {} }, configB);

            await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
            await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);

            expect(getGlobalConfigSpy.mock.calls.length).toBe(0);
            expect(crawlerA.requestQueue?.config).toBe(configA);
            expect(crawlerB.requestQueue?.config).toBe(configB);
        });
    });
});
