import type { Server } from 'node:http';

import { BROWSER_POOL_EVENTS, BrowserPool, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';
import { BLOCKED_STATUS_CODES } from '@crawlee/core';
import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, PuppeteerRequestHandler } from '@crawlee/puppeteer';
import {
    AutoscaledPool,
    EnqueueStrategy,
    ProxyConfiguration,
    Request,
    RequestList,
    RequestState,
    Session,
} from '@crawlee/puppeteer';
import { sleep } from '@crawlee/utils';
import type { HTTPResponse } from 'puppeteer';
import puppeteer from 'puppeteer';

import { ENV_VARS } from '@apify/consts';
import log from '@apify/log';

import { runExampleComServer } from '../../shared/_helper';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';
import { BrowserCrawlerTest } from './basic_browser_crawler';

describe('BrowserCrawler', () => {
    let prevEnvHeadless: string;
    let logLevel: number;

    let serverAddress = 'http://localhost:';
    let port: number;
    let server: Server;

    beforeAll(async () => {
        prevEnvHeadless = process.env.CRAWLEE_HEADLESS!;
        process.env.CRAWLEE_HEADLESS = '1';
        logLevel = log.getLevel();
        log.setLevel(log.LEVELS.ERROR);

        [server, port] = await runExampleComServer();
        serverAddress += port;
    });

    afterAll(async () => {
        log.setLevel(logLevel);
        process.env.CRAWLEE_HEADLESS = prevEnvHeadless;
        server.close();
    });

    test.concurrent('should work', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const sources = [
                { url: `${serverAddress}/?q=1` },
                { url: `${serverAddress}/?q=2` },
                { url: `${serverAddress}/?q=3` },
                { url: `${serverAddress}/?q=4` },
                { url: `${serverAddress}/?q=5` },
                { url: `${serverAddress}/?q=6` },
            ];
            const sourcesCopy = JSON.parse(JSON.stringify(sources));
            const processed: Request[] = [];
            const failed: Request[] = [];
            const requestList = await RequestList.open(null, sources);
            const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => {
                await page.waitForSelector('title');

                expect(response!.status()).toBe(200);
                request.userData.title = await page.title();
                processed.push(request);
            };

            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                minConcurrency: 1,
                maxConcurrency: 1,
                requestHandler,
                failedRequestHandler: async ({ request }) => {
                    failed.push(request);
                },
            });

            await browserCrawler.run();

            expect(browserCrawler.autoscaledPool!.minConcurrency).toBe(1);
            expect(processed).toHaveLength(6);
            expect(failed).toHaveLength(0);

            processed.forEach((request, id) => {
                expect(request.url).toEqual(sourcesCopy[id].url);
                expect(request.userData.title).toBe('Example Domain');
            });
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should teardown browser pool', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: 'http://example.com/?q=1' }],
            });
            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                requestHandler: async () => {},
                maxRequestRetries: 1,
            });

            // Spy on destroy and track if it was called
            let destroyCalled = false;
            const originalDestroy = browserCrawler.browserPool.destroy.bind(browserCrawler.browserPool);
            browserCrawler.browserPool.destroy = async () => {
                destroyCalled = true;
                return originalDestroy();
            };

            await browserCrawler.run();
            expect(destroyCalled).toBe(true);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should retire session after TimeoutError', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: 'http://example.com/?q=1' }],
            });
            class TimeoutError extends Error {}
            let markBadCalled = false;
            let sessionGoto!: Session;
            const browserCrawler = new (class extends BrowserCrawlerTest {
                protected override async _navigationHandler(
                    ctx: PuppeteerCrawlingContext,
                ): Promise<HTTPResponse | null | undefined> {
                    sessionGoto = ctx.session!;
                    const originalMarkBad = sessionGoto.markBad.bind(sessionGoto);
                    sessionGoto.markBad = () => {
                        markBadCalled = true;
                        return originalMarkBad();
                    };
                    throw new TimeoutError();
                }
            })({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                requestHandler: async () => {},
                maxRequestRetries: 1,
            });

            await browserCrawler.run();
            expect(markBadCalled).toBe(true);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should evaluate preNavigationHooks', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: 'http://example.com/?q=1' }],
            });
            let isEvaluated = false;

            const browserCrawler = new (class extends BrowserCrawlerTest {
                protected override async _navigationHandler(
                    ctx: PuppeteerCrawlingContext,
                    gotoOptions: PuppeteerGoToOptions,
                ): Promise<HTTPResponse | null | undefined> {
                    isEvaluated = ctx.hookFinished as boolean;
                    return ctx.page.goto(ctx.request.url, gotoOptions);
                }
            })({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                requestHandler: async () => {},
                maxRequestRetries: 0,
                preNavigationHooks: [
                    async (crawlingContext) => {
                        await sleep(10);
                        crawlingContext.hookFinished = true;
                    },
                ],
            });

            await browserCrawler.run();

            expect(isEvaluated).toBeTruthy();
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should evaluate postNavigationHooks', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: `${serverAddress}/?q=1` }],
            });
            let isEvaluated = false;

            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                requestHandler: async ({ hookFinished }) => {
                    isEvaluated = hookFinished as boolean;
                },
                maxRequestRetries: 0,
                postNavigationHooks: [
                    async (crawlingContext) => {
                        await sleep(10);
                        crawlingContext.hookFinished = true;
                    },
                ],
            });

            await browserCrawler.run();

            expect(isEvaluated).toBeTruthy();
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('errorHandler has open page', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: `${serverAddress}/?q=1` }],
            });

            const result: string[] = [];

            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                requestHandler: async (ctx) => {
                    throw new Error('Test error');
                },
                maxRequestRetries: 1,
                errorHandler: async (ctx, error) => {
                    result.push(await ctx.page.evaluate(() => window.location.origin));
                },
            });

            await browserCrawler.run();

            expect(result.length).toBe(1);
            expect(result[0]).toBe(serverAddress);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('errorHandler has open page after non-timeout navigation error', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: `${serverAddress}/?q=1` }],
            });

            const pageClosedStates: boolean[] = [];

            const browserCrawler = new (class extends BrowserCrawlerTest {
                protected override async _navigationHandler(): Promise<HTTPResponse | null | undefined> {
                    throw new Error('net::ERR_NAME_NOT_RESOLVED');
                }
            })({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                requestHandler: async () => {},
                maxRequestRetries: 1,
                errorHandler: async (ctx) => {
                    pageClosedStates.push(ctx.page.isClosed());
                },
            });

            await browserCrawler.run();

            expect(pageClosedStates).toHaveLength(1);
            expect(pageClosedStates[0]).toBe(false);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should correctly track request.state', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const sources = [{ url: `${serverAddress}/?q=1` }];
            const requestList = await RequestList.open(null, sources);
            const requestStates: RequestState[] = [];

            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                preNavigationHooks: [
                    async ({ request }) => {
                        requestStates.push(request.state);
                    },
                ],
                postNavigationHooks: [
                    async ({ request }) => {
                        requestStates.push(request.state);
                    },
                ],
                requestHandler: async ({ request }) => {
                    requestStates.push(request.state);
                    throw new Error('Error');
                },
                maxRequestRetries: 1,
                errorHandler: async ({ request }) => {
                    requestStates.push(request.state);
                },
            });

            await browserCrawler.run();

            expect(requestStates).toEqual([
                RequestState.BEFORE_NAV,
                RequestState.AFTER_NAV,
                RequestState.REQUEST_HANDLER,
                RequestState.ERROR_HANDLER,
                RequestState.BEFORE_NAV,
                RequestState.AFTER_NAV,
                RequestState.REQUEST_HANDLER,
            ]);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should allow modifying gotoOptions by pre navigation hooks', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: `${serverAddress}/?q=1` }],
            });
            let optionsGoto: PuppeteerGoToOptions;
            const browserCrawler = new (class extends BrowserCrawlerTest {
                protected override async _navigationHandler(
                    ctx: PuppeteerCrawlingContext,
                    gotoOptions: PuppeteerGoToOptions,
                ): Promise<HTTPResponse | null | undefined> {
                    optionsGoto = gotoOptions;
                    return ctx.page.goto(ctx.request.url, gotoOptions);
                }
            })({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                requestHandler: async () => {},
                maxRequestRetries: 0,
                preNavigationHooks: [
                    async (_crawlingContext, gotoOptions) => {
                        gotoOptions!.timeout = 60000;
                    },
                ],
            });

            await browserCrawler.run();

            expect(optionsGoto!.timeout).toEqual(60000);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should ignore errors in Page.close()', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            for (let i = 0; i < 2; i++) {
                const requestList = await RequestList.open({
                    sources: [{ url: `${serverAddress}/?q=1` }],
                });
                let failedCalled = false;

                const browserCrawler = new BrowserCrawlerTest({
                    browserPoolOptions: {
                        browserPlugins: [puppeteerPlugin],
                    },
                    requestList,
                    requestHandler: async ({ page }) => {
                        page.close = async () => {
                            if (i === 0) {
                                throw new Error();
                            } else {
                                return Promise.reject(new Error());
                            }
                        };
                        return Promise.resolve();
                    },
                    failedRequestHandler: async () => {
                        failedCalled = true;
                    },
                });
                await browserCrawler.run();
                expect(failedCalled).toBe(false);
            }
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should respect the requestHandlerTimeoutSecs option', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: `${serverAddress}/?q=1` }],
            });

            const callSpy = vitest.fn();

            // Use a very long delay for "bad" so it can never fire during test execution.
            // The test verifies that the 500ms timeout aborts the handler before "bad" would fire.
            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                requestHandler: async () => {
                    setTimeout(() => callSpy('good'), 300);
                    setTimeout(() => callSpy('bad'), 60_000);
                    await new Promise(() => {});
                },
                requestHandlerTimeoutSecs: 0.5,
                maxRequestRetries: 0,
            });
            await browserCrawler.run();

            expect(callSpy).toBeCalledTimes(1);
            expect(callSpy).toBeCalledWith('good');
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should not throw without SessionPool', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: 'http://example.com/?q=1' }],
            });
            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: false,
                requestHandler: async () => {},
            });

            expect(browserCrawler).toBeDefined();
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should correctly set session pool options', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: 'http://example.com/?q=1' }],
            });

            const crawler = new BrowserCrawlerTest({
                requestList,
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                useSessionPool: true,
                persistCookiesPerSession: false,
                sessionPoolOptions: {
                    sessionOptions: {
                        maxUsageCount: 1,
                    },
                    persistStateKeyValueStoreId: 'abc',
                },
                requestHandler: async () => {},
            });

            // @ts-expect-error Accessing private prop
            expect(crawler.sessionPoolOptions.sessionOptions.maxUsageCount).toBe(1);
            // @ts-expect-error Accessing private prop
            expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.skip('should persist cookies per session', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
        const name = `list-${Math.random()}`;
        const requestList = await RequestList.open({
            persistStateKey: name,
            persistRequestsKey: name,
            sources: [
                { url: 'http://example.com/?q=1' },
                { url: 'http://example.com/?q=2' },
                { url: 'http://example.com/?q=3' },
                { url: 'http://example.com/?q=4' },
            ],
        });
        const goToPageSessions = [];
        const loadedCookies: string[] = [];
        const browserCrawler = new BrowserCrawlerTest({
            browserPoolOptions: {
                browserPlugins: [puppeteerPlugin],
            },
            requestList,
            useSessionPool: true,
            persistCookiesPerSession: true,
            requestHandler: async ({ session, request }) => {
                loadedCookies.push(session!.getCookieString(request.url));
                return Promise.resolve();
            },
            preNavigationHooks: [
                async ({ session, page }) => {
                    await page.setCookie({
                        name: 'TEST',
                        value: '12321312312',
                        domain: 'example.com',
                        expires: Date.now() + 100000,
                    });
                    goToPageSessions.push(session);
                },
            ],
        });

        await browserCrawler.run();
        expect(loadedCookies).toHaveLength(4);

        loadedCookies.forEach((cookie) => {
            // TODO this test is flaky in CI and we need some more info to debug why.
            if (cookie !== 'TEST=12321312312') {
                // for some reason, the CI failures report the first cookie to be just empty string

                console.log('loadedCookies:');

                console.dir(loadedCookies);
            }

            expect(cookie).toEqual('TEST=12321312312');
        });
    });

    test.concurrent('should throw on "blocked" status codes', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const baseUrl = 'https://example.com/';
            const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
                return {
                    url: baseUrl + statusCode,
                    userData: { statusCode },
                };
            });
            const requestList = await RequestList.open(null, sources);

            let called = false;
            const failedRequests: Request[] = [];
            const crawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                persistCookiesPerSession: false,
                maxRequestRetries: 0,
                requestHandler: async () => {
                    called = true;
                },
                failedRequestHandler: async ({ request }) => {
                    failedRequests.push(request);
                },
            });

            // @ts-expect-error Overriding protected method
            crawler._navigationHandler = async ({ request }) => {
                return { status: () => request.userData.statusCode };
            };

            await crawler.run();

            expect(failedRequests.length).toBe(3);
            failedRequests.forEach((fr) => {
                const [msg] = fr.errorMessages;
                expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
            });
            expect(called).toBe(false);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('retryOnBlocked should retry on Cloudflare challenge', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];
            const maxSessionRotations = 1;

            let processed = false;
            const errorMessages: string[] = [];

            const crawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                retryOnBlocked: true,
                maxSessionRotations,
                requestHandler: async ({ page, response }) => {
                    processed = true;
                },
                failedRequestHandler: async ({ request }) => {
                    errorMessages.push(...request.errorMessages);
                },
            });

            await crawler.run(urls);

            expect(errorMessages).toHaveLength(urls.length * (maxSessionRotations + 1));
            expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
            expect(processed).toBe(false);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('retryOnBlocked throws on "blocked" status codes', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const baseUrl = 'https://example.com/';
            const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
                return {
                    url: baseUrl + statusCode,
                    userData: { statusCode },
                };
            });
            const requestList = await RequestList.open(null, sources);
            const maxSessionRotations = 1;
            const errorMessages: string[] = [];

            let processed = false;
            const crawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                retryOnBlocked: true,
                maxSessionRotations,
                requestHandler: async () => {
                    processed = true;
                },
                failedRequestHandler: async ({ request }) => {
                    errorMessages.push(...request.errorMessages);
                },
            });

            // @ts-expect-error Overriding protected method
            crawler._navigationHandler = async ({ request }) => {
                return { status: () => request.userData.statusCode };
            };

            await crawler.run();

            expect(errorMessages.length).toBe(sources.length * (maxSessionRotations + 1));
            expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
            expect(processed).toBe(false);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should throw on "blocked" status codes (retire session)', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const baseUrl = 'https://example.com/';
            const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
                return {
                    url: baseUrl + statusCode,
                    userData: { statusCode },
                };
            });
            const requestList = await RequestList.open(null, sources);

            let called = false;
            const failedRequests: Request[] = [];
            const crawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                persistCookiesPerSession: false,
                maxRequestRetries: 0,
                requestHandler: async () => {
                    called = true;
                },
                failedRequestHandler: async ({ request }) => {
                    failedRequests.push(request);
                },
            });

            // @ts-expect-error Overriding protected method
            crawler._navigationHandler = async ({ request }) => {
                return { status: () => request.userData.statusCode };
            };

            await crawler.run();

            expect(failedRequests.length).toBe(3);
            failedRequests.forEach((fr) => {
                const [msg] = fr.errorMessages;
                expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
            });
            expect(called).toBe(false);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should retire browser with session', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: 'http://example.com/?q=1' }],
            });
            let resolve: (value?: unknown) => void;

            const retirementPromise = new Promise((r) => {
                resolve = r;
            });
            let called = false;
            const browserCrawler = new (class extends BrowserCrawlerTest {
                protected override async _navigationHandler(
                    ctx: PuppeteerCrawlingContext,
                ): Promise<HTTPResponse | null | undefined> {
                    ctx.crawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => {
                        resolve();
                        called = true;
                    });
                    ctx.session!.retire();
                    return ctx.page.goto(ctx.request.url);
                }
            })({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                requestList,
                useSessionPool: true,
                requestHandler: async () => {
                    await retirementPromise;
                },
                maxRequestRetries: 1,
            });

            await browserCrawler.run();

            expect(called).toBeTruthy();
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should increment session usage correctly', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const sessionUsageHistory: number[] = [];

            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                useSessionPool: true,
                sessionPoolOptions: {
                    maxPoolSize: 1,
                },
                requestHandler: async ({ session }) => {
                    sessionUsageHistory.push(session!.usageCount);
                },
            });

            await browserCrawler.run([
                { url: `${serverAddress}/?q=1` },
                { url: `${serverAddress}/?q=2` },
                { url: `${serverAddress}/?q=3` },
                { url: `${serverAddress}/?q=4` },
                { url: `${serverAddress}/?q=5` },
                { url: `${serverAddress}/?q=6` },
            ]);

            expect(sessionUsageHistory).toEqual([0, 1, 2, 3, 4, 5]);
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test.concurrent('should allow using fingerprints from browser pool', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const requestList = await RequestList.open({
                sources: [{ url: `${serverAddress}/?q=1` }],
            });
            const browserCrawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                    useFingerprints: true,
                    fingerprintOptions: {
                        fingerprintGeneratorOptions: {
                            operatingSystems: [OperatingSystemsName.windows],
                        },
                    },
                },
                requestList,
                useSessionPool: false,
                requestHandler: async ({ browserController }) => {
                    expect(browserController.launchContext.fingerprint).toBeDefined();
                },
            });

            await browserCrawler.run();
            expect.hasAssertions();
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    describe('proxy', () => {
        // TODO move to actor sdk tests before splitting the repos
        // test('browser should launch with correct proxyUrl', async () => {
        //     process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
        //     const status = { connected: true };
        //     const fakeCall = async () => {
        //         return { body: status } as never;
        //     };
        //
        //     // @ts-expect-error FIXME
        //     const stub = gotScrapingSpy.mockImplementation(fakeCall);
        //     const proxyConfiguration = await Actor.createProxyConfiguration();
        //     const generatedProxyUrl = new URL(await proxyConfiguration.newUrl()).href.slice(0, -1);
        //     let browserProxy;
        //
        //     const browserCrawler = new BrowserCrawlerTest({
        //         browserPoolOptions: {
        //             browserPlugins: [puppeteerPlugin],
        //             postLaunchHooks: [(pageId, browserController) => {
        //                 browserProxy = browserController.launchContext.proxyUrl;
        //             }],
        //         },
        //         useSessionPool: false,
        //         persistCookiesPerSession: false,
        //         navigationTimeoutSecs: 1,
        //         requestList,
        //         maxRequestsPerCrawl: 1,
        //         maxRequestRetries: 0,
        //         requestHandler: async () => {},
        //         proxyConfiguration,
        //     });
        //     await browserCrawler.run();
        //     delete process.env[ENV_VARS.PROXY_PASSWORD];
        //
        //     expect(browserProxy).toEqual(generatedProxyUrl);
        //
        //     stub.mockClear();
        // });

        // TODO move to actor sdk tests before splitting the repos
        // test('requestHandler should expose the proxyInfo object with sessions correctly', async () => {
        //     process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
        //     const status = { connected: true };
        //     const fakeCall = async () => {
        //         return { body: status } as never;
        //     };
        //
        //     // @ts-expect-error FIXME
        //     const stub = gotScrapingSpy.mockImplementation(fakeCall);
        //
        //     const proxyConfiguration = await Actor.createProxyConfiguration();
        //     const proxies: ProxyInfo[] = [];
        //     const sessions: Session[] = [];
        //     const requestHandler = async ({ session, proxyInfo }: BrowserCrawlingContext) => {
        //         proxies.push(proxyInfo);
        //         sessions.push(session);
        //     };
        //
        //     const browserCrawler = new BrowserCrawlerTest({
        //         browserPoolOptions: {
        //             browserPlugins: [puppeteerPlugin],
        //         },
        //         requestList,
        //         requestHandler,
        //
        //         proxyConfiguration,
        //         useSessionPool: true,
        //         sessionPoolOptions: {
        //             maxPoolSize: 1,
        //         },
        //     });
        //
        //     await browserCrawler.run();
        //
        //     expect(proxies[0].sessionId).toEqual(sessions[0].id);
        //     expect(proxies[1].sessionId).toEqual(sessions[1].id);
        //     expect(proxies[2].sessionId).toEqual(sessions[2].id);
        //     expect(proxies[3].sessionId).toEqual(sessions[3].id);
        //
        //     delete process.env[ENV_VARS.PROXY_PASSWORD];
        //     stub.mockClear();
        // });

        // This test manipulates environment variables, so it must NOT be run concurrently
        test('browser should launch with rotated custom proxy', async () => {
            const localStorageEmulator = new MemoryStorageEmulator();
            await localStorageEmulator.init();
            const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

            try {
                process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';

                const requestList = await RequestList.open({
                    sources: [
                        { url: `${serverAddress}/?q=1` },
                        { url: `${serverAddress}/?q=2` },
                        { url: `${serverAddress}/?q=3` },
                    ],
                });

                const proxyConfiguration = new ProxyConfiguration({
                    proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
                });

                const browserProxies: string[] = [];

                const browserCrawler = new BrowserCrawlerTest({
                    browserPoolOptions: {
                        browserPlugins: [puppeteerPlugin],
                        maxOpenPagesPerBrowser: 1,
                        retireBrowserAfterPageCount: 1,
                    },
                    requestList,
                    requestHandler: async () => {},
                    proxyConfiguration,
                    maxRequestRetries: 0,
                    maxConcurrency: 1,
                });

                browserCrawler.browserPool.postLaunchHooks.push((_pageId, browserController) => {
                    browserProxies.push(browserController.launchContext.proxyUrl!);
                });

                await browserCrawler.run();

                // @ts-expect-error Accessing private property
                const proxiesToUse = proxyConfiguration.proxyUrls!;
                for (const proxyUrl of proxiesToUse) {
                    expect(browserProxies.includes(new URL(proxyUrl!).href.slice(0, -1))).toBeTruthy();
                }

                delete process.env[ENV_VARS.PROXY_PASSWORD];
            } finally {
                await localStorageEmulator.destroy();
            }
        });

        test.concurrent('proxy rotation on error works as expected', async () => {
            const localStorageEmulator = new MemoryStorageEmulator();
            await localStorageEmulator.init();
            const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

            try {
                const requestList = await RequestList.open({
                    sources: [
                        { url: 'http://example.com/?q=1' },
                        { url: 'http://example.com/?q=2' },
                        { url: 'http://example.com/?q=3' },
                        { url: 'http://example.com/?q=4' },
                    ],
                });

                const goodProxyUrl = 'http://good.proxy';
                const proxyConfiguration = new ProxyConfiguration({
                    proxyUrls: ['http://localhost', 'http://localhost:1234', goodProxyUrl],
                });
                const requestHandler = vitest.fn();

                const browserCrawler = new (class extends BrowserCrawlerTest {
                    protected override async _navigationHandler(
                        ctx: PuppeteerCrawlingContext,
                    ): Promise<HTTPResponse | null | undefined> {
                        const { session } = ctx;
                        const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);

                        if (proxyInfo!.url !== goodProxyUrl) {
                            throw new Error('ERR_PROXY_CONNECTION_FAILED');
                        }

                        return null;
                    }
                })({
                    browserPoolOptions: {
                        browserPlugins: [puppeteerPlugin],
                    },
                    requestList,
                    maxRequestRetries: 0,
                    maxConcurrency: 1,
                    useSessionPool: true,
                    proxyConfiguration,
                    requestHandler,
                });

                await expect(browserCrawler.run()).resolves.not.toThrow();
                expect(requestHandler).toHaveBeenCalledTimes(requestList!.length());
            } finally {
                await localStorageEmulator.destroy();
            }
        });

        test.concurrent('proxy rotation on error respects maxSessionRotations, calls failedRequestHandler', async () => {
            const localStorageEmulator = new MemoryStorageEmulator();
            await localStorageEmulator.init();
            const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

            try {
                const requestList = await RequestList.open({
                    sources: [
                        { url: 'http://example.com/?q=1' },
                        { url: 'http://example.com/?q=2' },
                        { url: 'http://example.com/?q=3' },
                        { url: 'http://example.com/?q=4' },
                    ],
                });

                const proxyConfiguration = new ProxyConfiguration({
                    proxyUrls: ['http://localhost', 'http://localhost:1234'],
                });
                const failedRequestHandler = vitest.fn();

                /**
                 * The first increment is the base case when the proxy is retrieved for the first time.
                 */
                let numberOfRotations = -requestList!.length();
                const browserCrawler = new (class extends BrowserCrawlerTest {
                    protected override async _navigationHandler(
                        ctx: PuppeteerCrawlingContext,
                    ): Promise<HTTPResponse | null | undefined> {
                        const { session } = ctx;
                        const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);

                        numberOfRotations++;

                        if (proxyInfo!.url.includes('localhost')) {
                            throw new Error('ERR_PROXY_CONNECTION_FAILED');
                        }

                        return null;
                    }
                })({
                    browserPoolOptions: {
                        browserPlugins: [puppeteerPlugin],
                    },
                    requestList,
                    maxSessionRotations: 5,
                    maxConcurrency: 1,
                    proxyConfiguration,
                    requestHandler: async () => {},
                    failedRequestHandler,
                });

                await browserCrawler.run();
                expect(failedRequestHandler).toBeCalledTimes(requestList!.length());
                expect(numberOfRotations).toBe(requestList!.length() * 5);
            } finally {
                await localStorageEmulator.destroy();
            }
        });

        test.concurrent('proxy rotation logs the original proxy error', async () => {
            const localStorageEmulator = new MemoryStorageEmulator();
            await localStorageEmulator.init();
            const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

            try {
                const requestList = await RequestList.open({
                    sources: [
                        { url: 'http://example.com/?q=1' },
                        { url: 'http://example.com/?q=2' },
                        { url: 'http://example.com/?q=3' },
                        { url: 'http://example.com/?q=4' },
                    ],
                });

                const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });

                const proxyError =
                    'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';

                const crawler = new (class extends BrowserCrawlerTest {
                    protected override async _navigationHandler(
                        ctx: PuppeteerCrawlingContext,
                    ): Promise<HTTPResponse | null | undefined> {
                        const { session } = ctx;
                        const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);

                        if (proxyInfo!.url.includes('localhost')) {
                            throw new Error(proxyError);
                        }

                        return null;
                    }
                })({
                    browserPoolOptions: {
                        browserPlugins: [puppeteerPlugin],
                    },
                    requestList,
                    maxSessionRotations: 1,
                    maxConcurrency: 1,
                    proxyConfiguration,
                    requestHandler: async () => {},
                });

                const spy = vitest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});

                await crawler.run([serverAddress]);

                expect(spy).toBeCalled();
                expect(spy.mock.calls[0][0]).toEqual(
                    'When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.',
                );
                expect(spy.mock.calls[1][0]).toEqual(expect.stringContaining(proxyError));
            } finally {
                await localStorageEmulator.destroy();
            }
        });
    });

    describe('Crawling context', () => {
        // This describe block manipulates log levels (global state), so tests must NOT be concurrent

        test('uses correct crawling context', async () => {
            const localStorageEmulator = new MemoryStorageEmulator();
            await localStorageEmulator.init();
            const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

            const actualLogLevel = log.getLevel();
            log.setLevel(log.LEVELS.OFF);

            try {
                const sources = ['http://example.com/'];
                const requestList = await RequestList.open(null, sources.slice());

                let prepareCrawlingContext: PuppeteerCrawlingContext;

                const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => {
                    prepareCrawlingContext = crawlingContext;
                    expect(crawlingContext.request).toBeInstanceOf(Request);
                    expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
                    expect(crawlingContext.session).toBeInstanceOf(Session);
                    expect(typeof crawlingContext.page).toBe('object');
                };

                const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {
                    expect(crawlingContext === prepareCrawlingContext).toEqual(true);
                    expect(crawlingContext.request).toBeInstanceOf(Request);
                    expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
                    expect(crawlingContext.session).toBeInstanceOf(Session);
                    expect(typeof crawlingContext.page).toBe('object');
                    expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
                    expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);

                    throw new Error('some error');
                };

                const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => {
                    expect(crawlingContext).toBe(prepareCrawlingContext);
                    expect(crawlingContext.request).toBeInstanceOf(Request);
                    expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
                    expect(crawlingContext.session).toBeInstanceOf(Session);
                    expect(typeof crawlingContext.page).toBe('object');
                    expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
                    expect(crawlingContext.crawler.browserPool).toBeInstanceOf(BrowserPool);
                    expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);

                    expect(crawlingContext.error).toBeInstanceOf(Error);
                    expect(error).toBeInstanceOf(Error);
                    expect(error.message).toEqual('some error');
                };

                const browserCrawler = new BrowserCrawlerTest({
                    browserPoolOptions: {
                        browserPlugins: [puppeteerPlugin],
                    },
                    requestList,
                    maxRequestRetries: 0,
                    maxConcurrency: 1,
                    useSessionPool: true,
                    requestHandler,
                    failedRequestHandler,
                });
                // @ts-expect-error Overriding protected method
                browserCrawler._navigationHandler = gotoFunction;

                await browserCrawler.run();
            } finally {
                log.setLevel(actualLogLevel);
                await localStorageEmulator.destroy();
            }
        });
    });

    // These tests cannot run concurrently because they use crawler.run([urls])
    // which creates internal request queues that can conflict
    test("enqueueLinks() should skip links that don't match the strategy post redirect", async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const succeeded: string[] = [];

            const crawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                maxConcurrency: 1,
                maxRequestRetries: 0,
                requestHandler: async ({ page, enqueueLinks }) => {
                    succeeded.push(await page.title());
                    await enqueueLinks({ strategy: EnqueueStrategy.SameOrigin });
                },
            });

            await crawler.run([`${serverAddress}/special/redirect`]);

            expect(succeeded).toHaveLength(1);
            expect(succeeded[0]).toEqual('Redirecting outside');
        } finally {
            await localStorageEmulator.destroy();
        }
    });

    test('enqueueLinks should respect maxCrawlDepth', async () => {
        const localStorageEmulator = new MemoryStorageEmulator();
        await localStorageEmulator.init();
        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);

        try {
            const succeeded: string[] = [];

            const crawler = new BrowserCrawlerTest({
                browserPoolOptions: {
                    browserPlugins: [puppeteerPlugin],
                },
                maxCrawlDepth: 1,
                maxRequestsPerCrawl: 10, // avoiding accidental runaway
                requestHandler: async ({ page, enqueueLinks }) => {
                    succeeded.push(await page.title());
                    await enqueueLinks({ strategy: EnqueueStrategy.All });
                },
            });

            await crawler.run([`${serverAddress}/special/html-type`]);

            expect(succeeded).toHaveLength(2);
            expect(succeeded).toEqual(['Example Domain', 'Example Domains']);
        } finally {
            await localStorageEmulator.destroy();
        }
    });
});
