import type { Server } from 'node:http';
import type { AddressInfo } from 'node:net';

import { Configuration, type Dictionary, EventType, KeyValueStore } from '@crawlee/core';
import type {
    AdaptivePlaywrightCrawlerContext,
    AdaptivePlaywrightCrawlerOptions,
    LoadedContext,
    Request,
} from '@crawlee/playwright';
import { AdaptivePlaywrightCrawler, RenderingTypePredictor, RequestList } from '@crawlee/playwright';
import { sleep } from 'crawlee';
import express from 'express';

import { startExpressAppPromise } from '../../shared/_helper';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';

describe('AdaptivePlaywrightCrawler', () => {
    // Set up an express server that will serve test pages
    const HOSTNAME = '127.0.0.1';
    let port: number;
    let server: Server;

    beforeAll(async () => {
        const app = express();
        server = await startExpressAppPromise(app, 0);
        port = (server.address() as AddressInfo).port;

        app.get('/static', (_req, res) => {
            res.status(200);
            res.send(`
                <html>
                    <head>
                        <title>Example Domain</title>
                    </head>
                    <body>
                        <h1>Heading</h1>
                        <a href="/static?q=1">Link 1</a>
                        <a href="/static?q=2">Link 2</a>
                        <a href="/static?q=3">Link 3</a>
                        <a href="/static?q=4">Link 4</a>
                        <a href="/static?q=5">Link 5</a>
                    </body>
                </html>
             `);
        });

        app.get('/dynamic', (_req, res) => {
            res.status(200);
            res.send(`
                <html>
                    <head>
                        <title>Example Domain</title>
                        <script type="text/javascript">
                            setTimeout(() => {
                                document.body.innerHTML = [
                                    '<h1>Heading</h1>',
                                    '<a href="/static?q=1">Link 1</a>',
                                    '<a href="/static?q=2">Link 2</a>',
                                    '<a href="/static?q=3">Link 3</a>',
                                    '<a href="/static?q=4">Link 4</a>',
                                    '<a href="/static?q=5">Link 5</a>',
                                ].join(" ")
                            }, 500)
                        </script>
                    </head>
                    <body>
                    </body>
                </html>
             `);
        });

        app.get('/external-links', (_req, res) => {
            res.status(200);
            res.send(`
                <html>
                    <head>
                        <title>Example Domain</title>
                    </head>
                    <body>
                        <h1>Heading</h1>
                        <a href="/external-redirect">External redirect</a>
                        <a href="https://google.com/">Outbound link</a>
                    </body>
                </html>
             `);
        });

        app.get('/external-redirect', (_req, res) => {
            res.status(302);
            res.setHeader('Location', 'https://google.com');
            res.send('Redirecting...');
        });
    });
    afterAll(async () => {
        server.close();
    });

    // Set up local storage emulator
    const localStorageEmulator = new MemoryStorageEmulator();

    beforeEach(async () => {
        await localStorageEmulator.init();
    });
    afterAll(async () => {
        await localStorageEmulator.destroy();
    });

    // Test setup helpers
    const makeOneshotCrawler = async (
        options: Required<Pick<AdaptivePlaywrightCrawlerOptions, 'requestHandler' | 'renderingTypePredictor'>> &
            Partial<AdaptivePlaywrightCrawlerOptions>,
        sources: string[],
    ) =>
        new AdaptivePlaywrightCrawler({
            renderingTypeDetectionRatio: 0.1,
            maxConcurrency: 1,
            maxRequestRetries: 0,
            maxRequestsPerCrawl: 1,
            requestList: await RequestList.open({ sources }),
            ...options,
        });

    const makeRiggedRenderingTypePredictor = (prediction: {
        detectionProbabilityRecommendation: number;
        renderingType: 'clientOnly' | 'static';
    }) => ({
        initialize: async () => {},
        predict: vi.fn((_request: Request) => prediction),
        storeResult: vi.fn((_request: Request, _renderingType: string) => {}),
    });

    describe('should detect page rendering type', () => {
        test.each([
            ['/static', 'static'],
            ['/dynamic', 'clientOnly'],
        ] as const)('for %s', async (path, expectedType) => {
            const renderingTypePredictor = makeRiggedRenderingTypePredictor({
                detectionProbabilityRecommendation: 1,
                renderingType: 'clientOnly',
            });
            const url = new URL(`http://${HOSTNAME}:${port}${path}`);

            const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
                async ({ pushData, parseWithCheerio }) => {
                    const $ = await parseWithCheerio('h1');
                    await pushData({
                        heading: $('h1').text(),
                    });
                },
            );

            const crawler = await makeOneshotCrawler(
                {
                    requestHandler,
                    renderingTypePredictor,
                },
                [url.toString()],
            );

            await crawler.run();

            // Check the detection result
            expect(renderingTypePredictor.predict).toHaveBeenCalledOnce();
            expect(renderingTypePredictor.predict.mock.lastCall?.[0]).toMatchObject({ url, label: undefined });

            expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
            expect(renderingTypePredictor.storeResult.mock.lastCall?.[0]).toMatchObject({ url, label: undefined });
            expect(renderingTypePredictor.storeResult.mock.lastCall?.[1]).toEqual(expectedType);

            // Check if the request handler was called twice
            expect(requestHandler).toHaveBeenCalledTimes(2);

            // Check if only one item was added to the dataset
            expect(await localStorageEmulator.getDatasetItems()).toEqual([{ heading: 'Heading' }]);
        });
    });

    test('should not store detection results on non-detection runs', async () => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType: 'static',
        });
        const url = new URL(`http://${HOSTNAME}:${port}/static`);

        const crawler = await makeOneshotCrawler(
            {
                requestHandler: async () => {},
                renderingTypePredictor,
            },
            [url.toString()],
        );

        await crawler.run();

        expect(renderingTypePredictor.predict).toHaveBeenCalledOnce();
        expect(renderingTypePredictor.predict.mock.lastCall?.[0]).toMatchObject({ url, label: undefined });

        expect(renderingTypePredictor.storeResult).not.toHaveBeenCalled();
    });

    test('should retry with browser if result checker returns false', async () => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType: 'static',
        });
        const url = new URL(`http://${HOSTNAME}:${port}/dynamic`);

        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
            async ({ pushData, querySelector }) => {
                await pushData({
                    heading: (await querySelector('h1')).text(),
                });
            },
        );

        const resultChecker: AdaptivePlaywrightCrawlerOptions['resultChecker'] = vi.fn(
            (result) =>
                result.datasetItems.length > 0 &&
                result.datasetItems.every(({ item }: Dictionary) => item.heading?.length > 0),
        );

        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                renderingTypePredictor,
                resultChecker,
            },
            [url.toString()],
        );

        await crawler.run();

        expect(requestHandler).toHaveBeenCalledTimes(2);
        expect(resultChecker).toHaveBeenCalledTimes(1);
    });

    describe('shouldPropagateError', () => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType: 'static',
        });
        const failedRequestHandler = vi.fn();
        const testError = new Error('HTTP handler failed');
        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async () => {
            throw testError;
        });

        beforeEach(() => {
            vi.clearAllMocks();
        });

        test('should fall back to browser when shouldPropagateError returns false', async () => {
            const shouldPropagateError = vi.fn(() => false);
            const url = new URL(`http://${HOSTNAME}:${port}/static`);

            const crawler = await makeOneshotCrawler(
                {
                    requestHandler,
                    renderingTypePredictor,
                    shouldPropagateError,
                    failedRequestHandler,
                },
                [url.toString()],
            );

            await crawler.run();

            expect(shouldPropagateError).toHaveBeenCalledOnce();
            expect(shouldPropagateError).toHaveBeenCalledWith(testError, expect.anything());
            expect(requestHandler).toHaveBeenCalledTimes(2);
        });

        test('should propagate error when shouldPropagateError returns true', async () => {
            const shouldPropagateError = vi.fn(() => true);
            const url = new URL(`http://${HOSTNAME}:${port}/static`);

            const crawler = await makeOneshotCrawler(
                {
                    requestHandler,
                    renderingTypePredictor,
                    shouldPropagateError,
                    failedRequestHandler,
                },
                [url.toString()],
            );

            await crawler.run();

            expect(shouldPropagateError).toHaveBeenCalledOnce();
            expect(shouldPropagateError).toHaveBeenCalledWith(testError, expect.anything());
            expect(requestHandler).toHaveBeenCalledTimes(1);
            expect(failedRequestHandler).toHaveBeenCalledOnce();
            expect(failedRequestHandler.mock.calls[0][1]).toBe(testError);
        });
    });

    test.each([
        ['static'],
        ['clientOnly'],
    ] as const)('crawlingContext.addRequests() should add requests correctly (%s)', async (renderingType) => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType,
        });
        const url = new URL(`http://${HOSTNAME}:${port}`).toString();

        let requestContext: LoadedContext<AdaptivePlaywrightCrawlerContext> | undefined;
        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async (context) => {
            const isStartUrl = context.request.url === url;

            if (isStartUrl) await context.addRequests([`${url}/1`]);
            else requestContext = context;
        };

        const crawler = await makeOneshotCrawler(
            { requestHandler, renderingTypePredictor, maxRequestsPerCrawl: 10 },
            [],
        );

        await crawler.run([{ url, crawlDepth: 2 }]);

        assert(requestContext);
        expect(requestContext.request).toMatchObject({ url: `${url}/1`, crawlDepth: 3 });
    });

    describe('should enqueue links correctly', () => {
        test.each([
            ['/static', 'static'],
            ['/dynamic', 'clientOnly'],
        ] as const)('for %s', async (path, renderingType) => {
            const renderingTypePredictor = makeRiggedRenderingTypePredictor({
                detectionProbabilityRecommendation: 0,
                renderingType,
            });
            const url = new URL(`http://${HOSTNAME}:${port}${path}`);
            const enqueuedRequests: Request[] = [];

            const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
                async ({ enqueueLinks, request }) => {
                    if (request.label === 'enqueued-url') {
                        enqueuedRequests.push(request);
                    } else {
                        await enqueueLinks({ label: 'enqueued-url' });
                    }
                },
            );

            const crawler = await makeOneshotCrawler(
                {
                    requestHandler,
                    renderingTypePredictor,
                    maxRequestsPerCrawl: 10,
                },
                [url.toString()],
            );

            await crawler.run();

            const enqueuedUrls = Array.from(enqueuedRequests).map((request) => request.url);
            expect(new Set(enqueuedUrls)).toEqual(
                new Set([
                    `http://${HOSTNAME}:${port}/static?q=1`,
                    `http://${HOSTNAME}:${port}/static?q=2`,
                    `http://${HOSTNAME}:${port}/static?q=3`,
                    `http://${HOSTNAME}:${port}/static?q=4`,
                    `http://${HOSTNAME}:${port}/static?q=5`,
                ]),
            );

            expect(enqueuedRequests[0]).toMatchObject({
                url: enqueuedUrls[0],
                label: 'enqueued-url',
                crawlDepth: 1,
            });
        });
    });

    test.each([
        ['static'],
        ['clientOnly'],
    ] as const)('should respect the strategy option for enqueueLinks (%s)', async (renderingType) => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType,
        });
        const url = new URL(`http://${HOSTNAME}:${port}/external-links`);
        const enqueuedUrls = new Set<string>();
        const visitedUrls = new Set<string>();

        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
            async ({ enqueueLinks, request }) => {
                visitedUrls.add(request.loadedUrl);

                if (!request.label) {
                    const result = await enqueueLinks({
                        label: 'enqueued-url',
                        strategy: 'same-hostname',
                    });

                    for (const processedRequest of result.processedRequests) {
                        enqueuedUrls.add(processedRequest.uniqueKey);
                    }
                }
            },
        );

        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                renderingTypePredictor,
                maxRequestsPerCrawl: 10,
            },
            [url.toString()],
        );

        await crawler.run();

        expect(new Set(visitedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-links`]));
        expect(new Set(enqueuedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-redirect`]));
    });

    test('should persist crawler state', async () => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType: 'static',
        });

        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ useState }) => {
            const state = await useState({ count: 0 });
            state.count += 1;
        });

        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                renderingTypePredictor,
                maxRequestsPerCrawl: 3,
            },
            [
                `http://${HOSTNAME}:${port}/static?q=1`,
                `http://${HOSTNAME}:${port}/static?q=2`,
                `http://${HOSTNAME}:${port}/static?q=3`,
            ],
        );

        await crawler.run();
        const state = await localStorageEmulator.getState();
        expect(state!.value).toEqual({ count: 3 });
    });

    test('should return deeply equal but not identical state objects across handler runs', async () => {
        // Force detection to happen
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 1,
            renderingType: 'clientOnly',
        });

        // We'll store state references to compare them later
        const stateReferences: any[] = [];

        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ useState }) => {
            const state = await useState({ data: { nested: { value: 42 } } });
            stateReferences.push(JSON.parse(JSON.stringify(state)));
            state.randomNumber = Math.random();
        });

        // Run the crawler
        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                renderingTypePredictor,
            },
            [`http://${HOSTNAME}:${port}/static`],
        );

        await crawler.run();

        // The request handler should have run twice (once in browser, once in HTTP-only mode for detection)
        expect(requestHandler).toHaveBeenCalledTimes(2);
        expect(stateReferences).toHaveLength(2);

        // The state objects should be deeply equal (same values)
        expect(stateReferences[0]).toEqual(stateReferences[1]);

        // But they should not be the same object instance (different references)
        // This is important to ensure that state objects are properly cloned between handler runs
        // and that modifications to one state object don't affect others
        expect(stateReferences[0]).not.toBe(stateReferences[1]);
    });

    test('should persist key-value store changes', async () => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType: 'static',
        });

        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
            async ({ request, getKeyValueStore }) => {
                const store = await getKeyValueStore();
                const search = new URLSearchParams(new URL(request.url).search);
                store.setValue(search.get('q'), { content: 42 });
            },
        );

        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                renderingTypePredictor,
                maxRequestsPerCrawl: 3,
            },
            [
                `http://${HOSTNAME}:${port}/static?q=1`,
                `http://${HOSTNAME}:${port}/static?q=2`,
                `http://${HOSTNAME}:${port}/static?q=3`,
            ],
        );

        await crawler.run();
        const store = localStorageEmulator.getKeyValueStore();

        expect((await store.getRecord('1'))!.value).toEqual({ content: 42 });
        expect((await store.getRecord('2'))!.value).toEqual({ content: 42 });
        expect((await store.getRecord('3'))!.value).toEqual({ content: 42 });
    });

    test('should not allow direct key-value store manipulation', async () => {
        const renderingTypePredictor = makeRiggedRenderingTypePredictor({
            detectionProbabilityRecommendation: 0,
            renderingType: 'static',
        });

        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async () => {
            const store = await KeyValueStore.open();
            await store.setValue('1', { content: 42 });
        });

        const failedRequestHandler = vi.fn();

        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                renderingTypePredictor,
                maxRequestsPerCrawl: 3,
                maxRequestRetries: 0,
                failedRequestHandler,
            },
            [`http://${HOSTNAME}:${port}/static`],
        );

        await crawler.run();
        expect(failedRequestHandler.mock.calls).toHaveLength(1);
        expect((failedRequestHandler.mock.calls[0][1] as Error).message).toEqual(
            'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',
        );

        const store = localStorageEmulator.getKeyValueStore();
        expect(await store.getRecord('1')).toBeUndefined();
    });

    test('should persist RenderingTypePredictor state on PERSIST_STATE events', async () => {
        const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ pushData }) => {
            await pushData({ content: 'test data' });
        });

        const crawler = await makeOneshotCrawler(
            {
                requestHandler,
                // Use a real RenderingTypePredictor instead of the mocked one
                renderingTypePredictor: new RenderingTypePredictor({ detectionRatio: 1 }),
            },
            [`http://${HOSTNAME}:${port}/static`],
        );

        // Run the crawler - this will initialize the RenderingTypePredictor and potentially store results
        await crawler.run();

        // Now emit a PERSIST_STATE event to trigger state persistence
        const events = Configuration.getEventManager();
        events.emit(EventType.PERSIST_STATE);

        // Wait a bit for the event to be processed
        await sleep(100);

        // Verify that the regression model was actually saved to the key-value store
        const store = await KeyValueStore.open();
        const storedState = await store.getValue<string>('rendering-type-predictor-state');
        expect(storedState).not.toBeNull();

        const parsedState = JSON.parse(storedState!);
        expect(parsedState).toHaveProperty('logreg');

        // Test that the persisted state can be successfully restored
        // by creating a new RenderingTypePredictor and seeing if it initializes without error
        const newPredictor = new RenderingTypePredictor({
            detectionRatio: 0.1,
            persistenceOptions: { persistStateKey: 'rendering-type-predictor-state' },
        });

        // This should not throw since we've persisted valid state
        await expect(newPredictor.initialize()).resolves.not.toThrow();
    });
});
