import { type AddRequestsBatchedOptions, cheerioCrawlerEnqueueLinks } from '@crawlee/cheerio';
import { enqueueLinks } from '@crawlee/core';
import { launchPlaywright } from '@crawlee/playwright';
import type { RequestQueueOperationOptions, Source } from '@crawlee/puppeteer';
import {
    browserCrawlerEnqueueLinks,
    Configuration,
    EnqueueStrategy,
    launchPuppeteer,
    RequestQueue,
} from '@crawlee/puppeteer';
import { type CheerioRoot, RobotsTxtFile } from '@crawlee/utils';
import { load } from 'cheerio';
import type { Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright';
import type { Browser as PuppeteerBrowser, Page as PuppeteerPage } from 'puppeteer';

import log from '@apify/log';

const apifyClient = Configuration.getStorageClient();

const HTML = `
<html>
    <head>
        <title>Example</title>
    </head>
    <body>
        <p>
            The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
        </p>
        <ul>
            <li>These aren't the Droids you're looking for</li>
            <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
            <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
        </ul>
        <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
        <p>
            Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
            just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
        </p>
        <a href="/x/absolutepath">This is a relative link.</a>
        <a href="y/relativepath">This is a relative link.</a>
        <a href="//example.absolute.com/hello">This is a link to a different subdomain</a>
        <a href="http://">Invalid URL link, this needs to be ignored</a>
    </body>
</html>
`;

function createRequestQueueMock() {
    const enqueued: Source[] = [];
    const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });

    // @ts-expect-error Override method for testing
    requestQueue.addRequests = async function (requests) {
        const processedRequests: Source[] = [];
        for await (const request of requests) {
            processedRequests.push(typeof request === 'string' ? { url: request } : request);
        }
        enqueued.push(...processedRequests);
        return { processedRequests, unprocessedRequests: [] as never[] };
    };

    return { enqueued, requestQueue };
}

describe('enqueueLinks()', () => {
    let ll: number;
    beforeAll(() => {
        ll = log.getLevel();
        log.setLevel(log.LEVELS.ERROR);
    });

    afterAll(() => {
        log.setLevel(ll);
    });

    describe.each([[launchPuppeteer], [launchPlaywright]] as const)('using %s', (method) => {
        let browser: PuppeteerBrowser | PlaywrightBrowser;
        let page: PuppeteerPage | PlaywrightPage;

        beforeEach(async () => {
            browser = (await method({ launchOptions: { headless: true } })) as PlaywrightBrowser | PuppeteerBrowser;
            page = await browser.newPage();
            await page.setContent(HTML);
        });

        afterEach(async () => {
            if (browser) await browser.close();
            page = null!;
            browser = null!;
        });

        test('works with item limit', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await browserCrawlerEnqueueLinks({
                options: { limit: 3, selector: '.click', strategy: EnqueueStrategy.All },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://another.com/a/fifth');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3]).toBe(undefined);
        });

        test('works with globs', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const globs = ['https://example.com/**/*', { glob: '?(http|https)://cool.com/', method: 'POST' as const }];

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    label: 'COOL',
                    globs,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({ label: 'COOL' });

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('OPTIONS');
            expect(enqueued[1].userData).toEqual({ label: 'COOL' });

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('POST');
            expect(enqueued[2].userData).toEqual({ label: 'COOL' });
        });

        test('does not throw with empty globs', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const globs = [
                'https://example.com/**/*',
                '',
                { glob: ' ' },
                // Empty string used to throw an error (https://console.apify.com/actors/aYG0l9s7dbB7j3gbS/issues/Wd0Ahfk9Vd2OPk4Uf)
                { glob: '' },
                { glob: '?(http|https)://cool.com/', method: 'POST' as const },
            ];

            await expect(
                browserCrawlerEnqueueLinks({
                    options: { selector: '.click', globs },
                    page,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).resolves.not.toThrow();

            expect(enqueued).toHaveLength(3);
        });

        test('works with regexps', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const regexps = [
                /^https:\/\/example\.com\/(\w|\/)+/,
                { regexp: /^(http|https):\/\/cool\.com\//, method: 'POST' as const, userData: { label: 'COOL' } },
            ];

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    regexps,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('OPTIONS');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('POST');
            expect(enqueued[2].userData).toEqual({ label: 'COOL' });
        });

        test('works with skipNavigation', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    skipNavigation: true,
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(2);

            for (const request of enqueued) {
                expect(request.skipNavigation).toBe(true);
            }
        });

        test('works with exclude glob', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const globs = ['https://example.com/**/*', { glob: '?(http|https)://cool.com/', method: 'POST' as const }];

            const exclude = ['**/first'];

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    label: 'COOL',
                    globs,
                    exclude,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(2);

            expect(enqueued[0].url).not.toBe('https://example.com/a/b/first');
            expect(enqueued[1].url).not.toBe('https://example.com/a/b/first');

            expect(enqueued[0].url).toBe('https://example.com/a/b/third');
            expect(enqueued[0].method).toBe('OPTIONS');
            expect(enqueued[0].userData).toEqual({ label: 'COOL' });

            expect(enqueued[1].url).toBe('http://cool.com/');
            expect(enqueued[1].method).toBe('POST');
            expect(enqueued[1].userData).toEqual({ label: 'COOL' });
        });

        test('works with exclude regexp', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const globs = ['https://example.com/**/*', { glob: '?(http|https)://cool.com/', method: 'POST' as const }];

            const exclude = [/first/];

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    label: 'COOL',
                    globs,
                    exclude,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(2);

            expect(enqueued[0].url).not.toBe('https://example.com/a/b/first');
            expect(enqueued[1].url).not.toBe('https://example.com/a/b/first');

            expect(enqueued[0].url).toBe('https://example.com/a/b/third');
            expect(enqueued[0].method).toBe('OPTIONS');
            expect(enqueued[0].userData).toEqual({ label: 'COOL' });

            expect(enqueued[1].url).toBe('http://cool.com/');
            expect(enqueued[1].method).toBe('POST');
            expect(enqueued[1].userData).toEqual({ label: 'COOL' });
        });

        test('works with pseudoUrls', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const pseudoUrls = [
                'https://example.com/[(\\w|-|/)*]',
                { purl: '[http|https]://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
            ];

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    pseudoUrls,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('OPTIONS');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('POST');
            expect(enqueued[2].userData).toEqual({ label: 'COOL' });
        });

        test('throws with RegExp pseudoUrls', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            const pseudoUrls = [/https:\/\/example\.com\/(\w|-|\/)*/, /(http|https):\/\/cool\.com\//];

            await expect(
                browserCrawlerEnqueueLinks({
                    // @ts-expect-error Type 'RegExp[]' is not assignable to type 'PseudoUrlInput[]'
                    options: { selector: '.click', pseudoUrls },
                    page,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).rejects.toThrow(/to be of type `string` but received type `RegExp`/);
        });

        test('works with undefined pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            await browserCrawlerEnqueueLinks({
                options: { selector: '.click', strategy: EnqueueStrategy.All },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(4);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://another.com/a/fifth');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3].url).toBe('http://cool.com/');
            expect(enqueued[3].method).toBe('GET');
            expect(enqueued[3].userData).toEqual({});
        });

        test('throws with null pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await expect(
                browserCrawlerEnqueueLinks({
                    // @ts-expect-error invalid input
                    options: { selector: '.click', pseudoUrls: null },
                    page,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).rejects.toThrow(/Expected property `pseudoUrls` to be of type `array` but received type `null`/);
        });

        test('works with empty pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await browserCrawlerEnqueueLinks({
                options: { selector: '.click', pseudoUrls: [], strategy: EnqueueStrategy.All },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(4);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://another.com/a/fifth');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3].url).toBe('http://cool.com/');
            expect(enqueued[3].method).toBe('GET');
            expect(enqueued[3].userData).toEqual({});
        });

        test('throws with sparse pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const pseudoUrls = ['https://example.com/[(\\w|-|/)*]', null, '[http|https]://cool.com/'];

            await expect(
                browserCrawlerEnqueueLinks({
                    // @ts-expect-error invalid input
                    options: { selector: '.click', pseudoUrls },
                    page,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).rejects.toThrow(/\(array `pseudoUrls`\) Any predicate failed with the following errors/);
            expect(enqueued).toHaveLength(0);
        });

        test('correctly resolves relative URLs with default strategy of same-hostname', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await browserCrawlerEnqueueLinks({
                options: { baseUrl: 'http://www.absolute.com/removethis/' },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(2);

            expect(enqueued[0].url).toBe('http://www.absolute.com/x/absolutepath');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('http://www.absolute.com/removethis/y/relativepath');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});
        });

        test('correctly resolves relative URLs with the strategy of same-domain', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await browserCrawlerEnqueueLinks({
                options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.SameDomain },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('http://www.absolute.com/x/absolutepath');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('http://www.absolute.com/removethis/y/relativepath');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://example.absolute.com/hello');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});
        });

        test('correctly resolves relative URLs with the strategy of all', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await browserCrawlerEnqueueLinks({
                options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.All },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(8);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/second');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://example.com/a/b/third');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3].url).toBe('https://another.com/a/fifth');
            expect(enqueued[3].method).toBe('GET');
            expect(enqueued[3].userData).toEqual({});

            expect(enqueued[4].url).toBe('http://cool.com/');
            expect(enqueued[4].method).toBe('GET');
            expect(enqueued[4].userData).toEqual({});

            expect(enqueued[5].url).toBe('http://www.absolute.com/x/absolutepath');
            expect(enqueued[5].method).toBe('GET');
            expect(enqueued[5].userData).toEqual({});

            expect(enqueued[6].url).toBe('http://www.absolute.com/removethis/y/relativepath');
            expect(enqueued[6].method).toBe('GET');
            expect(enqueued[6].userData).toEqual({});

            expect(enqueued[7].url).toBe('http://example.absolute.com/hello');
            expect(enqueued[7].method).toBe('GET');
            expect(enqueued[7].userData).toEqual({});
        });

        test('correctly works with transformRequestFunction', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            const pseudoUrls = ['https://example.com/[(\\w|-|/)*]', '[http|https]://cool.com/'];

            await browserCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    pseudoUrls,
                    transformRequestFunction: (request) => {
                        if (request.url.includes('example.com')) {
                            request.method = 'POST';
                        } else if (request.url.includes('cool.com')) {
                            request.userData!.foo = 'bar';
                        }
                        return request;
                    },
                },
                page,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('POST');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('POST');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData!.foo).toBe('bar');
        });
    });

    describe('using Cheerio', () => {
        let $: CheerioRoot;

        beforeEach(async () => {
            $ = load(HTML);
        });

        afterEach(async () => {
            $ = null!;
        });

        test('works with globs', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const globs = [
                'https://example.com/**/*',
                { glob: '?(http|https)://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
            ];

            await cheerioCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    globs,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('OPTIONS');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('POST');
            expect(enqueued[2].userData).toEqual({ label: 'COOL' });
        });

        test('does not throw with empty globs', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const globs = [
                'https://example.com/**/*',
                { glob: '?(http|https)://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
                '',
                { glob: ' ' },
            ];

            await expect(
                cheerioCrawlerEnqueueLinks({
                    options: { selector: '.click', globs },
                    $,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).resolves.not.toThrow();

            expect(enqueued).toHaveLength(3);
        });

        test('works with RegExps', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const regexps = [
                /^https:\/\/example\.com\/(\w|\/)+/,
                { regexp: /^(http|https):\/\/cool\.com\//, method: 'POST' as const, userData: { label: 'COOL' } },
            ];

            await cheerioCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    regexps,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('OPTIONS');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('POST');
            expect(enqueued[2].userData).toEqual({ label: 'COOL' });
        });

        test('works with string pseudoUrls', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const pseudoUrls = [
                'https://example.com/[(\\w|-|/)*]',
                { purl: '[http|https]://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
            ];

            await cheerioCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    userData: { label: 'DEFAULT' },
                    pseudoUrls,
                    transformRequestFunction: (request) => {
                        if (request.url.match(/example\.com\/a\/b\/third/)) {
                            request.method = 'OPTIONS';
                        }
                        return request;
                    },
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({ label: 'DEFAULT' });

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('OPTIONS');
            expect(enqueued[1].userData).toEqual({ label: 'DEFAULT' });

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('POST');
            expect(enqueued[2].userData).toEqual({ label: 'COOL' });
        });

        test('throws with RegExp pseudoUrls', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const pseudoUrls = [/https:\/\/example\.com\/(\w|-|\/)*/, /(http|https):\/\/cool\.com\//];

            await expect(
                cheerioCrawlerEnqueueLinks({
                    // @ts-expect-error Type 'RegExp[]' is not assignable to type 'PseudoUrlInput[]'
                    options: { selector: '.click', pseudoUrls },
                    $,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).rejects.toThrow(/to be of type `string` but received type `RegExp`/);
        });

        test('works with undefined pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await cheerioCrawlerEnqueueLinks({
                options: { selector: '.click', strategy: EnqueueStrategy.All },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(4);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://another.com/a/fifth');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3].url).toBe('http://cool.com/');
            expect(enqueued[3].method).toBe('GET');
            expect(enqueued[3].userData).toEqual({});
        });

        test('throws with null pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await expect(
                cheerioCrawlerEnqueueLinks({
                    // @ts-expect-error invalid input
                    options: { selector: '.click', pseudoUrls: null },
                    $,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).rejects.toThrow(/Expected property `pseudoUrls` to be of type `array` but received type `null`/);
        });

        test('works with empty pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await cheerioCrawlerEnqueueLinks({
                options: { selector: '.click', pseudoUrls: [], strategy: EnqueueStrategy.All },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(4);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://another.com/a/fifth');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3].url).toBe('http://cool.com/');
            expect(enqueued[3].method).toBe('GET');
            expect(enqueued[3].userData).toEqual({});
        });

        test('throws with sparse pseudoUrls[]', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const pseudoUrls = ['https://example.com/[(\\w|-|/)*]', null, '[http|https]://cool.com/'];

            await expect(
                cheerioCrawlerEnqueueLinks({
                    // @ts-expect-error invalid input
                    options: { selector: '.click', pseudoUrls },
                    $,
                    requestQueue,
                    originalRequestUrl: 'https://example.com',
                }),
            ).rejects.toThrow(/\(array `pseudoUrls`\) Any predicate failed with the following errors/);
            expect(enqueued).toHaveLength(0);
        });

        test('correctly resolves relative URLs with the strategy of all', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await cheerioCrawlerEnqueueLinks({
                options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.All },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(8);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/second');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('https://example.com/a/b/third');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});

            expect(enqueued[3].url).toBe('https://another.com/a/fifth');
            expect(enqueued[3].method).toBe('GET');
            expect(enqueued[3].userData).toEqual({});

            expect(enqueued[4].url).toBe('http://cool.com/');
            expect(enqueued[4].method).toBe('GET');
            expect(enqueued[4].userData).toEqual({});

            expect(enqueued[5].url).toBe('http://www.absolute.com/x/absolutepath');
            expect(enqueued[5].method).toBe('GET');
            expect(enqueued[5].userData).toEqual({});

            expect(enqueued[6].url).toBe('http://www.absolute.com/removethis/y/relativepath');
            expect(enqueued[6].method).toBe('GET');
            expect(enqueued[6].userData).toEqual({});

            expect(enqueued[7].url).toBe('http://example.absolute.com/hello');
            expect(enqueued[7].method).toBe('GET');
            expect(enqueued[7].userData).toEqual({});
        });

        test('correctly resolves relative URLs with the default strategy of same-hostname', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await cheerioCrawlerEnqueueLinks({
                options: { baseUrl: 'http://www.absolute.com/removethis/' },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(2);

            expect(enqueued[0].url).toBe('http://www.absolute.com/x/absolutepath');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('http://www.absolute.com/removethis/y/relativepath');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});
        });

        test('correctly resolves relative URLs with the strategy of same-domain', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await cheerioCrawlerEnqueueLinks({
                options: { baseUrl: 'http://www.absolute.com/removethis/', strategy: EnqueueStrategy.SameDomain },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('http://www.absolute.com/x/absolutepath');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('http://www.absolute.com/removethis/y/relativepath');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://example.absolute.com/hello');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData).toEqual({});
        });

        test('correctly resolves relative URLs with `urls` option', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            await cheerioCrawlerEnqueueLinks({
                options: {
                    baseUrl: 'http://www.absolute.com/removethis/',
                    urls: ['/relative/url1', '/relative/url2'],
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(2);

            expect(enqueued[0].url).toBe('http://www.absolute.com/relative/url1');
            expect(enqueued[0].method).toBe('GET');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('http://www.absolute.com/relative/url2');
            expect(enqueued[1].method).toBe('GET');
            expect(enqueued[1].userData).toEqual({});
        });

        test('correctly works with transformRequestFunction', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();
            const pseudoUrls = ['https://example.com/[(\\w|-|/)*]', '[http|https]://cool.com/'];

            await cheerioCrawlerEnqueueLinks({
                options: {
                    selector: '.click',
                    pseudoUrls,
                    transformRequestFunction: (request) => {
                        if (request.url.includes('example.com')) {
                            request.method = 'POST';
                        } else if (request.url.includes('cool.com')) {
                            request.userData!.foo = 'bar';
                        }
                        return request;
                    },
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(3);

            expect(enqueued[0].url).toBe('https://example.com/a/b/first');
            expect(enqueued[0].method).toBe('POST');
            expect(enqueued[0].userData).toEqual({});

            expect(enqueued[1].url).toBe('https://example.com/a/b/third');
            expect(enqueued[1].method).toBe('POST');
            expect(enqueued[1].userData).toEqual({});

            expect(enqueued[2].url).toBe('http://cool.com/');
            expect(enqueued[2].method).toBe('GET');
            expect(enqueued[2].userData!.foo).toBe('bar');
        });

        test('accepts forefront option', async () => {
            const enqueued: { request: Source; options?: RequestQueueOperationOptions }[] = [];
            const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });

            requestQueue.addRequests = async (requests, options) => {
                // copy the requests to the enqueued list, along with options that were passed to addRequests,
                // so that it doesn't matter how many calls were made
                for await (const request of requests) {
                    enqueued.push({ request: typeof request === 'string' ? { url: request } : request, options });
                }
                return { processedRequests: [], unprocessedRequests: [] };
            };

            await cheerioCrawlerEnqueueLinks({
                options: {
                    forefront: true,
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(5);

            for (let i = 0; i < 5; i++) {
                expect(enqueued[i].options!.forefront).toBe(true);
            }
        });

        test('accepts waitForAllRequestsToBeAdded option', async () => {
            const enqueued: { request: string | Source; options?: AddRequestsBatchedOptions }[] = [];
            const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });

            requestQueue.addRequestsBatched = async (requests, options) => {
                // copy the requests to the enqueued list, along with options that were passed to addRequests,
                // so that it doesn't matter how many calls were made
                for await (const request of requests) {
                    enqueued.push({ request: typeof request === 'string' ? { url: request } : request, options });
                }
                return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]), requestsOverLimit: [] };
            };

            await cheerioCrawlerEnqueueLinks({
                options: {
                    waitForAllRequestsToBeAdded: true,
                },
                $,
                requestQueue,
                originalRequestUrl: 'https://example.com',
            });

            expect(enqueued).toHaveLength(5);

            for (let i = 0; i < 5; i++) {
                expect(enqueued[i].options!.waitForAllRequestsToBeAdded).toBe(true);
            }
        });
    });

    describe('respectRobotsTxtFile option', () => {
        const robotsTxtFile = RobotsTxtFile.from(
            'http://example.com/robots.txt',
            `User-agent: *
             Disallow: /
             Allow: /yes

             User-agent: MyCrawler
             Disallow: /no
             Allow: /my-crawler
            `,
        );

        const urls = [
            'http://example.com/yes',
            'http://example.com/no',
            'http://example.com/no-globally',
            'http://example.com/my-crawler/anything',
        ];

        test('defaults to the catch-all user-agent when not provided', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            await enqueueLinks({ urls, requestQueue, robotsTxtFile });

            expect(enqueued.map((r) => r.url)).toEqual(['http://example.com/yes']);
        });

        test('applies rules for the configured user-agent', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            await enqueueLinks({
                urls,
                requestQueue,
                robotsTxtFile,
                respectRobotsTxtFile: { userAgent: 'MyCrawler' },
            });

            expect(enqueued.map((r) => r.url)).toEqual([
                'http://example.com/yes',
                'http://example.com/my-crawler/anything',
            ]);
        });

        test('skips filtering when set to false even if robotsTxtFile is provided', async () => {
            const { enqueued, requestQueue } = createRequestQueueMock();

            await enqueueLinks({
                urls,
                requestQueue,
                robotsTxtFile,
                respectRobotsTxtFile: false,
            });

            expect(enqueued.map((r) => r.url)).toEqual(urls);
        });
    });
});
