import { createAction, Property } from '@activepieces/pieces-framework';
import { dataFuelAuth } from '../common/auth';
import { AuthenticationType, httpClient, HttpMethod } from '@activepieces/pieces-common';
import { BASE_URL } from '../common/constants';
import { CrawlWebsiteResponse, ListScrapesResponse } from '../common/types';

export const crawlWebsiteAction = createAction({
	name: 'crawl-website',
	auth: dataFuelAuth,
	displayName: 'Crawl Website',
	description: 'Crawl a website into a markdown format.',
	audience: 'both',
	aiMetadata: {
		description: 'Crawls a website starting from a URL, following links up to a configurable depth and page limit, and returns each page as markdown; optionally extract structured data by supplying a JSON schema (OpenAI function-calling format) and an AI prompt. Use to gather content across many pages of a site, not just one. Submits a new crawl job and polls until it finishes, so each call starts fresh work and is not idempotent.',
		idempotent: false,
	},
	props: {
		url: Property.ShortText({
			displayName: 'URL',
			required: true,
		}),
		prompt: Property.LongText({
			displayName: 'AI Prompt',
			description: 'Prompt to crawl data',
			required: false,
		}),
		depth: Property.Number({
			displayName: 'Depth',
			description:
				'The depth of the crawl 1 depth mean only the first level of links will be scraped',
			required: true,
		}),
		limit: Property.Number({
			displayName: 'Limit',
			description: 'The maximum number of pages to scrape',
			required: true,
		}),
		jsonSchema: Property.Json({
			displayName: 'JSON Schema',
			required: false,
			description: `JSON schema definition for structured data extraction.Format should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs)`,
		}),
	},
	async run(context) {
		const { url, prompt, depth, limit, jsonSchema } = context.propsValue;

		const response = await httpClient.sendRequest<CrawlWebsiteResponse>({
			method: HttpMethod.POST,
			url: BASE_URL + '/crawl',
			authentication: {
				type: AuthenticationType.BEARER_TOKEN,
				token: context.auth.secret_text,
			},
			body: {
				url,
				ai_prompt: prompt,
				json_schema: jsonSchema,
				depth,
				limit,
			},
		});

		const jobId = response.body.job_id;
		let status = 'pending';
		const timeoutAt = Date.now() + 5 * 60 * 1000;

		while (status !== 'finished' && Date.now() < timeoutAt) {
			await new Promise((resolve) => setTimeout(resolve, 5000)); // wait 5 seconds

			const pollResponse = await httpClient.sendRequest<Array<ListScrapesResponse>>({
				method: HttpMethod.GET,
				url: BASE_URL + '/list_scrapes',
				authentication: {
					type: AuthenticationType.BEARER_TOKEN,
					token: context.auth.secret_text,
				},
				queryParams: {
					job_id: jobId,
					markdown: 'true',
				},
			});

			status = pollResponse.body[0].job_status;

			if (status === 'finished') return pollResponse.body;
		}
		throw new Error('Crawl Job timed out or failed.');
	},
});
