/**
 * Crawl Sitemap Tool - Sitemap-based web crawling via sitemap.xml discovery
 *
 * Discovers sitemaps from robots.txt or well-known locations, parses sitemap XML,
 * and crawls listed pages. Supports sitemap index files with recursive resolution.
 *
 * @see https://github.com/shaun0927/openchrome/issues/576
 */

import { MCPServer } from '../mcp-server';
import { MCPToolDefinition, MCPResult, ToolHandler, ToolContext, hasBudget } from '../types/mcp';
import { TOOL_ANNOTATIONS } from '../types/tool-annotations';
import { getSessionManager } from '../session-manager';
import { MAX_OUTPUT_CHARS } from '../config/defaults';
import { withTimeout } from '../utils/with-timeout';
import {
  normalizeUrl,
  parseSitemapXml,
  parseRobotsTxt,
  CrawlTracker,
  urlGlobToRegex,
} from '../utils/crawl-utils';
import {
  staticFetch,
  isStaticSufficient,
  extractBodyText,
  StaticFetchError,
  StaticReason,
} from '../utils/static-fetch';
import { buildTextMetrics } from '../core/metrics/token-estimate';
import { extractMainContent, toMarkdown } from '../core/extract/html-to-markdown';
import { applyContentFilter, ContentFilterMetrics, parseContentFilterType, ContentFilterType } from '../core/extract/content-filter';
import { sanitizeContent } from '../security/content-sanitizer';
import { getGlobalConfig } from '../config/global';
import {
  canReadCache,
  canWriteCache,
  CrawlCacheMetadata,
  CrawlCacheMode,
  CrawlCacheScope,
  CrawlContentCache,
  parseCrawlCacheMode,
  parseCrawlCacheScope,
} from '../core/crawl/content-cache';

const definition: MCPToolDefinition = {
  name: 'crawl_sitemap',
  description:
    'Crawl a website using its sitemap.xml. Auto-discovers sitemaps from robots.txt or /sitemap.xml. Supports sitemap index files and URL filtering.\n\nWhen to use: Extracting content from many pages of a site that publishes a sitemap.xml.\nWhen NOT to use: Use crawl for BFS discovery when no sitemap exists, or navigate for a single page.',
  inputSchema: {
    type: 'object',
    properties: {
      url: {
        type: 'string',
        description: 'Website URL (auto-discovers /sitemap.xml, /sitemap_index.xml)',
      },
      sitemap_url: {
        type: 'string',
        description: 'Explicit sitemap URL (skips auto-discovery)',
      },
      filter: {
        type: 'string',
        description: 'URL glob pattern to filter which sitemap URLs to visit',
      },
      max_pages: {
        type: 'number',
        description: 'Maximum number of pages to visit. Default: 50',
      },
      output_format: {
        type: 'string',
        enum: ['markdown', 'text', 'structured', 'markdown-clean'],
        description: 'Content format per page. "markdown-clean" uses cheerio+turndown to strip nav/footer/ads. Default: markdown',
      },
      onlyMainContent: {
        type: 'boolean',
        description: 'markdown-clean only: strip nav/header/footer/aside/ads. Default: true.',
      },
      includeLinks: {
        type: 'boolean',
        description: 'markdown-clean only: preserve <a> as markdown links. Default: true.',
      },
      query: {
        type: 'string',
        description: 'markdown-clean content_filter="bm25" query terms.',
      },
      content_filter: {
        type: 'string',
        enum: ['none', 'prune', 'bm25'],
        description: 'markdown-clean only: deterministic fit_markdown filter. Default: none.',
      },
      return_raw: {
        type: 'boolean',
        description: 'markdown-clean only: include raw_markdown in each page. Default: false.',
      },
      return_fit: {
        type: 'boolean',
        description: 'markdown-clean only: include fit_markdown and use it as content when filtering. Default: true when filtered.',
      },
      concurrency: {
        type: 'number',
        description: 'Max concurrent page fetches. Default: 3',
      },
      engine: {
        type: 'string',
        enum: ['auto', 'static', 'cdp'],
        description:
          'Fetch engine: "cdp" (default, opens a Chrome tab per page), "static" (Node fetch only, fails closed on insufficient pages), or "auto" (static first, fall back to CDP when static is insufficient).',
      },
      cache_mode: {
        type: 'string',
        enum: ['disabled', 'enabled', 'read_only', 'write_only', 'bypass'],
        description: 'Opt-in crawl content cache mode. Default: disabled.',
      },
      cache_ttl_ms: {
        type: 'number',
        description: 'Maximum age for enabled/read_only cache hits. Omit for no TTL expiry.',
      },
      cache_scope: {
        type: 'string',
        enum: ['public', 'session'],
        description: 'Cache namespace/safety scope. Default: public.',
      },
      include_metrics: {
        type: 'boolean',
        description: 'When true, include approximate output size/token metrics in the JSON result. Default: false.',
      },
    },
    required: ['url'],
  },
  annotations: TOOL_ANNOTATIONS.crawl_sitemap,
};

// ---------------------------------------------------------------------------
// Concurrency limiter (same pattern as crawl.ts / batch-paginate.ts)
// ---------------------------------------------------------------------------

function createLimiter(concurrency: number) {
  let active = 0;
  const queue: Array<() => void> = [];

  return async function <T>(fn: () => Promise<T>): Promise<T> {
    if (active >= concurrency) {
      await new Promise<void>((resolve) => queue.push(resolve));
    }
    active++;
    try {
      return await fn();
    } finally {
      active--;
      if (queue.length > 0) {
        const next = queue.shift()!;
        next();
      }
    }
  };
}

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

interface CrawledPage {
  url: string;
  title: string;
  content: string;
  raw_markdown?: string;
  fit_markdown?: string;
  filter?: ContentFilterMetrics;
  links_found: number;
  error?: string;
  engine_used?: 'static' | 'cdp';
  static_reason?: StaticReason;
  cache?: CrawlCacheMetadata;
}

type EngineMode = 'auto' | 'static' | 'cdp';

interface CrawlSitemapSummary {
  total_pages: number;
  succeeded: number;
  failed: number;
  duration_ms: number;
  sitemap_source: string;
}

// ---------------------------------------------------------------------------
// Fetch raw text from a URL via static fetch (robots.txt / sitemap XML are
// plain text — no Chrome tab needed).
// ---------------------------------------------------------------------------

async function fetchRawText(
  _sessionId: string,
  url: string,
  context?: ToolContext,
): Promise<string | null> {
  try {
    const { html, status } = await staticFetch(url, { signal: context?.signal });
    if (status < 200 || status >= 300) return null;
    return html || null;
  } catch (err) {
    console.error(
      `[crawl_sitemap] Failed to fetch ${url}: ${err instanceof Error ? err.message : String(err)}`,
    );
    return null;
  }
}

// ---------------------------------------------------------------------------
// Sitemap discovery: robots.txt -> /sitemap.xml -> /sitemap_index.xml
// ---------------------------------------------------------------------------

async function discoverSitemapUrls(
  sessionId: string,
  origin: string,
  context?: ToolContext,
): Promise<{ urls: string[]; source: string }> {
  // Step 1: Try robots.txt
  const robotsText = await fetchRawText(sessionId, `${origin}/robots.txt`, context);
  if (robotsText && (robotsText.toLowerCase().includes('user-agent') || robotsText.toLowerCase().includes('sitemap'))) {
    const rules = parseRobotsTxt(robotsText);
    if (rules.sitemaps.length > 0) {
      console.error(`[crawl_sitemap] Found ${rules.sitemaps.length} sitemap(s) in robots.txt`);
      return { urls: rules.sitemaps, source: `${origin}/robots.txt` };
    }
  }

  // Step 2: Try /sitemap.xml
  const sitemapXml = await fetchRawText(sessionId, `${origin}/sitemap.xml`, context);
  if (sitemapXml && (sitemapXml.includes('<urlset') || sitemapXml.includes('<sitemapindex'))) {
    console.error(`[crawl_sitemap] Found sitemap at ${origin}/sitemap.xml`);
    return { urls: [`${origin}/sitemap.xml`], source: `${origin}/sitemap.xml` };
  }

  // Step 3: Try /sitemap_index.xml
  const indexXml = await fetchRawText(sessionId, `${origin}/sitemap_index.xml`, context);
  if (indexXml && (indexXml.includes('<urlset') || indexXml.includes('<sitemapindex'))) {
    console.error(`[crawl_sitemap] Found sitemap at ${origin}/sitemap_index.xml`);
    return { urls: [`${origin}/sitemap_index.xml`], source: `${origin}/sitemap_index.xml` };
  }

  return { urls: [], source: 'none' };
}

// ---------------------------------------------------------------------------
// Resolve sitemap URLs (handles sitemap index recursion up to 2 levels)
// ---------------------------------------------------------------------------

async function resolveSitemapPageUrls(
  sessionId: string,
  sitemapUrls: string[],
  filterPattern: string | undefined,
  maxPages: number,
  context?: ToolContext,
  depth = 0,
): Promise<string[]> {
  const pageUrls: string[] = [];
  const filterRegex = filterPattern ? urlGlobToRegex(filterPattern) : null;

  for (const sitemapUrl of sitemapUrls) {
    if (pageUrls.length >= maxPages) break;

    // Check budget before fetching each sitemap
    if (context && !hasBudget(context, 10_000)) {
      console.error('[crawl_sitemap] Budget limit approaching, stopping sitemap resolution');
      break;
    }

    const xml = await fetchRawText(sessionId, sitemapUrl, context);
    if (!xml) {
      console.error(`[crawl_sitemap] Could not fetch sitemap: ${sitemapUrl}`);
      continue;
    }

    const parsed = parseSitemapXml(xml);

    if (parsed.isSitemapIndex) {
      // Recurse into child sitemaps (up to 2 levels deep)
      if (depth < 2) {
        console.error(
          `[crawl_sitemap] Sitemap index with ${parsed.sitemapIndexUrls.length} child sitemaps (depth=${depth})`,
        );
        const childUrls = await resolveSitemapPageUrls(
          sessionId,
          parsed.sitemapIndexUrls,
          filterPattern,
          maxPages - pageUrls.length,
          context,
          depth + 1,
        );
        pageUrls.push(...childUrls);
      } else {
        console.error(`[crawl_sitemap] Max sitemap index depth reached, skipping: ${sitemapUrl}`);
      }
    } else {
      // Regular sitemap — collect page URLs
      for (const entry of parsed.urls) {
        if (pageUrls.length >= maxPages) break;

        const normalized = normalizeUrl(entry.loc);

        // Apply filter if provided
        if (filterRegex && !filterRegex.test(normalized)) continue;

        pageUrls.push(normalized);
      }
    }
  }

  return pageUrls;
}

// ---------------------------------------------------------------------------
// Static engine — Node fetch path. Returns null + reason on insufficiency so
// the caller (auto mode) can fall back to CDP.
// ---------------------------------------------------------------------------

function cleanMarkdownFromHtml(
  html: string,
  cleanOpts: { onlyMainContent: boolean; includeLinks: boolean; contentFilter?: ContentFilterType; query?: string; returnRaw?: boolean; returnFit?: boolean },
): { content: string; raw_markdown?: string; fit_markdown?: string; filter?: ContentFilterMetrics } {
  const { html: cleaned } = extractMainContent(html, { onlyMainContent: cleanOpts.onlyMainContent });
  let cleanMd = toMarkdown(cleaned, { includeLinks: cleanOpts.includeLinks });
  const cfg = getGlobalConfig();
  if (cfg.security?.sanitize_content !== false) {
    const sanitized = sanitizeContent(cleanMd);
    cleanMd = sanitized.text + sanitized.sanitizationNote;
  }
  const filterType = cleanOpts.contentFilter ?? 'none';
  if (filterType !== 'none' || cleanOpts.returnRaw || cleanOpts.returnFit === true) {
    return applyContentFilter(cleanMd, {
      type: filterType,
      query: cleanOpts.query,
      returnRaw: cleanOpts.returnRaw,
      returnFit: cleanOpts.returnFit !== false,
    });
  }
  return { content: cleanMd };
}

function staticBuildContent(html: string): { title: string; content: string } {
  const titleMatch = html.match(/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i);
  const title = titleMatch ? titleMatch[1].trim() : '';
  const { text } = extractBodyText(html);
  return { title, content: text };
}

function staticCountLinks(html: string, baseUrl: string): number {
  const re = /<a\s[^>]*href\s*=\s*["']([^"']+)["'][^>]*>/gi;
  let count = 0;
  let m: RegExpExecArray | null;
  while ((m = re.exec(html)) !== null) {
    const href = m[1];
    if (!href) continue;
    if (
      href.startsWith('javascript:') ||
      href.startsWith('mailto:') ||
      href.startsWith('tel:')
    ) {
      continue;
    }
    try {
      const resolved = new URL(href, baseUrl).toString();
      if (resolved.startsWith('http://') || resolved.startsWith('https://')) {
        count++;
      }
    } catch {
      // skip malformed
    }
  }
  return count;
}

async function fetchPageStatic(
  url: string,
  outputFormat: string,
  cleanOpts: { onlyMainContent: boolean; includeLinks: boolean; contentFilter?: ContentFilterType; query?: string; returnRaw?: boolean; returnFit?: boolean },
  context?: ToolContext,
): Promise<
  | { ok: true; page: CrawledPage }
  | { ok: false; reason: StaticReason; error?: string }
> {
  try {
    const { html, status, contentType, finalUrl } = await staticFetch(url, {
      signal: context?.signal,
    });
    const sufficiency = isStaticSufficient(html, status, contentType);
    if (!sufficiency.ok) {
      return { ok: false, reason: sufficiency.reason };
    }

    let title = '';
    let content = '';
    let cleanExtra: { raw_markdown?: string; fit_markdown?: string; filter?: ContentFilterMetrics } = {};
    if (outputFormat === 'structured') {
      const titleMatch = html.match(/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i);
      title = titleMatch ? titleMatch[1].trim() : '';
      const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body\s*>/i);
      content = bodyMatch ? bodyMatch[1] : html;
    } else if (outputFormat === 'markdown-clean') {
      const titleMatch = html.match(/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i);
      title = titleMatch ? titleMatch[1].trim() : '';
      {
        const cleanResult = cleanMarkdownFromHtml(html, cleanOpts);
        content = cleanResult.content;
        cleanExtra = {
          ...(cleanResult.raw_markdown ? { raw_markdown: cleanResult.raw_markdown } : {}),
          ...(cleanResult.fit_markdown ? { fit_markdown: cleanResult.fit_markdown } : {}),
          ...(cleanResult.filter ? { filter: cleanResult.filter } : {}),
        };
      }
    } else {
      const built = staticBuildContent(html);
      title = built.title;
      content = built.content;
    }

    if (content.length > MAX_OUTPUT_CHARS) {
      content = content.slice(0, MAX_OUTPUT_CHARS) + '...[truncated]';
    }

    return {
      ok: true,
      page: {
        url,
        title,
        content,
        ...cleanExtra,
        links_found: staticCountLinks(html, finalUrl),
      },
    };
  } catch (err) {
    const reason: StaticReason =
      err instanceof StaticFetchError ? err.reason : 'fetch-error';
    return {
      ok: false,
      reason,
      error: err instanceof Error ? err.message : String(err),
    };
  }
}

// ---------------------------------------------------------------------------
// Single page fetch (same pattern as crawl.ts)
// ---------------------------------------------------------------------------

async function fetchPage(
  sessionId: string,
  url: string,
  outputFormat: string,
  cleanOpts: { onlyMainContent: boolean; includeLinks: boolean; contentFilter?: ContentFilterType; query?: string; returnRaw?: boolean; returnFit?: boolean },
  context?: ToolContext,
): Promise<CrawledPage> {
  const sessionManager = getSessionManager();
  let targetId: string | null = null;

  try {
    const { targetId: tid, page } = await sessionManager.createTarget(sessionId, url);
    targetId = tid;

    // Small settle delay for dynamic content
    await new Promise((r) => setTimeout(r, 500));

    if (outputFormat === 'markdown-clean') {
      const fullHtml = await withTimeout(
        page.content(),
        15000,
        'crawl_sitemap.page.content',
        context,
      );
      const linkResult = await withTimeout(
        page.evaluate(() => {
          const title = document.title || '';
          let linksCount = 0;
          document.querySelectorAll('a[href]').forEach((a) => {
            const href = (a as HTMLAnchorElement).href;
            if (href && (href.startsWith('http://') || href.startsWith('https://'))) {
              linksCount += 1;
            }
          });
          return { title, linksCount };
        }),
        15000,
        'crawl_sitemap.page.linkScan',
        context,
      );
      await sessionManager.closeTarget(sessionId, tid);
      targetId = null;

      const cleanResult = cleanMarkdownFromHtml(fullHtml, cleanOpts);
      let cleanMd = cleanResult.content;
      if (cleanMd.length > MAX_OUTPUT_CHARS) {
        cleanMd = cleanMd.slice(0, MAX_OUTPUT_CHARS) + '...[truncated]';
      }
      return {
        url,
        title: linkResult.title,
        content: cleanMd,
        ...(cleanResult.raw_markdown ? { raw_markdown: cleanResult.raw_markdown } : {}),
        ...(cleanResult.fit_markdown ? { fit_markdown: cleanResult.fit_markdown } : {}),
        ...(cleanResult.filter ? { filter: cleanResult.filter } : {}),
        links_found: linkResult.linksCount,
      };
    }

    const result = await withTimeout(
      page.evaluate((format: string) => {
        const title = document.title || '';

        // Collect links
        const links: string[] = [];
        const anchors = document.querySelectorAll('a[href]');
        anchors.forEach((a) => {
          const href = (a as HTMLAnchorElement).href;
          if (href && !href.startsWith('javascript:') && !href.startsWith('mailto:') && !href.startsWith('tel:')) {
            links.push(href);
          }
        });

        // Extract content based on format
        let content = '';
        if (format === 'markdown') {
          const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, {
            acceptNode(n: Node) {
              const t = (n as HTMLElement).tagName?.toLowerCase();
              if (t === 'script' || t === 'style' || t === 'noscript') return NodeFilter.FILTER_REJECT;
              return NodeFilter.FILTER_ACCEPT;
            },
          });
          const parts: string[] = [];
          let node: Node | null = walker.currentNode;

          while (node) {
            const el = node as HTMLElement;
            const tag = el.tagName?.toLowerCase();

            if (tag === 'h1') parts.push(`\n# ${el.textContent?.trim()}\n`);
            else if (tag === 'h2') parts.push(`\n## ${el.textContent?.trim()}\n`);
            else if (tag === 'h3') parts.push(`\n### ${el.textContent?.trim()}\n`);
            else if (tag === 'h4') parts.push(`\n#### ${el.textContent?.trim()}\n`);
            else if (tag === 'h5') parts.push(`\n##### ${el.textContent?.trim()}\n`);
            else if (tag === 'h6') parts.push(`\n###### ${el.textContent?.trim()}\n`);
            else if (tag === 'p') {
              const text = el.textContent?.trim();
              if (text) parts.push(`\n${text}\n`);
            }
            else if (tag === 'li') {
              const text = el.textContent?.trim();
              if (text) parts.push(`- ${text}`);
            }
            else if (tag === 'pre' || tag === 'code') {
              const text = el.textContent?.trim();
              if (text && tag === 'pre') parts.push(`\n\`\`\`\n${text}\n\`\`\`\n`);
            }
            else if (tag === 'blockquote') {
              const text = el.textContent?.trim();
              if (text) parts.push(`\n> ${text}\n`);
            }

            node = walker.nextNode();
          }

          content = parts.join('\n').replace(/\n{3,}/g, '\n\n').trim();

          if (!content) {
            content = document.body.innerText || '';
          }
        } else if (format === 'text') {
          content = document.body.innerText || '';
        } else {
          content = document.body.innerHTML || '';
        }

        return { title, content, linksCount: links.length };
      }, outputFormat),
      15000,
      'crawl_sitemap.page.evaluate',
      context,
    );

    await sessionManager.closeTarget(sessionId, tid);
    targetId = null;

    // Truncate content if too large
    let content = result.content;
    if (content.length > MAX_OUTPUT_CHARS) {
      content = content.slice(0, MAX_OUTPUT_CHARS) + '...[truncated]';
    }

    return {
      url,
      title: result.title,
      content,
      links_found: result.linksCount,
    };
  } catch (err) {
    if (targetId) {
      try {
        await sessionManager.closeTarget(sessionId, targetId);
      } catch {
        // ignore cleanup errors
      }
    }
    return {
      url,
      title: '',
      content: '',
      links_found: 0,
      error: err instanceof Error ? err.message : String(err),
    };
  }
}

// ---------------------------------------------------------------------------
// Handler
// ---------------------------------------------------------------------------

const handler: ToolHandler = async (
  sessionId: string,
  args: Record<string, unknown>,
  context?: ToolContext,
): Promise<MCPResult> => {
  const url = args.url as string;
  if (!url) {
    return {
      content: [{ type: 'text', text: 'Error: url is required' }],
      isError: true,
    };
  }

  // Validate URL
  let parsedUrl: URL;
  try {
    parsedUrl = new URL(url);
  } catch {
    return {
      content: [{ type: 'text', text: `Error: Invalid URL "${url}"` }],
      isError: true,
    };
  }
  if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
    return {
      content: [{ type: 'text', text: 'Error: url must use http or https scheme' }],
      isError: true,
    };
  }

  const sitemapUrlArg = args.sitemap_url as string | undefined;
  const filterPattern = args.filter as string | undefined;
  const maxPages = args.max_pages != null ? Number(args.max_pages) : 50;
  const outputFormat = (args.output_format as string) || 'markdown';
  let cacheMode: CrawlCacheMode;
  let cacheScope: CrawlCacheScope;
  try {
    cacheMode = parseCrawlCacheMode(args.cache_mode);
    cacheScope = parseCrawlCacheScope(args.cache_scope);
  } catch (err) {
    return {
      content: [{ type: 'text', text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
      isError: true,
    };
  }
  const cacheTtlMs = typeof args.cache_ttl_ms === 'number' ? args.cache_ttl_ms : undefined;
  const cleanOpts = {
    onlyMainContent: args.onlyMainContent !== false,
    includeLinks: args.includeLinks !== false,
    contentFilter: parseContentFilterType(args.content_filter),
    query: args.query as string | undefined,
    returnRaw: args.return_raw === true,
    returnFit: args.return_fit !== false,
  };
  const concurrency = args.concurrency != null ? Math.max(1, Math.min(10, Number(args.concurrency))) : 3;

  const includeMetrics = args.include_metrics === true;
  const engineArg = args.engine as string | undefined;
  let engine: EngineMode = 'cdp';
  if (engineArg === 'static' || engineArg === 'auto' || engineArg === 'cdp') {
    engine = engineArg;
  } else if (engineArg !== undefined) {
    return {
      content: [{ type: 'text', text: `Error: engine must be one of "auto", "static", "cdp"` }],
      isError: true,
    };
  }
  const engineExplicit = engineArg !== undefined;

  const startTime = Date.now();
  const tracker = new CrawlTracker();
  const crawlCache = new CrawlContentCache<CrawledPage>();
  const pages: CrawledPage[] = [];
  let sitemapSource = '';

  try {
    // -----------------------------------------------------------------------
    // Step 1: Discover or use provided sitemap URL(s)
    // -----------------------------------------------------------------------
    let sitemapUrls: string[];

    if (sitemapUrlArg) {
      // Validate sitemap URL scheme
      try {
        const sp = new URL(sitemapUrlArg);
        if (sp.protocol !== 'http:' && sp.protocol !== 'https:') {
          return { content: [{ type: 'text', text: 'Error: sitemap_url must use http or https scheme' }], isError: true };
        }
      } catch {
        return { content: [{ type: 'text', text: `Error: Invalid sitemap_url "${sitemapUrlArg}"` }], isError: true };
      }
      // Explicit sitemap URL provided
      sitemapUrls = [sitemapUrlArg];
      sitemapSource = sitemapUrlArg;
      console.error(`[crawl_sitemap] Using explicit sitemap: ${sitemapUrlArg}`);
    } else {
      // Auto-discover from robots.txt -> /sitemap.xml -> /sitemap_index.xml
      const origin = parsedUrl.origin;
      const discovery = await discoverSitemapUrls(sessionId, origin, context);
      sitemapUrls = discovery.urls;
      sitemapSource = discovery.source;

      if (sitemapUrls.length === 0) {
        return {
          content: [
            {
              type: 'text',
              text: JSON.stringify(
                {
                  error: 'No sitemap found',
                  tried: [
                    `${origin}/robots.txt`,
                    `${origin}/sitemap.xml`,
                    `${origin}/sitemap_index.xml`,
                  ],
                  suggestion:
                    'Provide an explicit sitemap_url parameter or use the crawl tool for BFS-based crawling.',
                },
                null,
                2,
              ),
            },
          ],
          isError: true,
        };
      }
    }

    // -----------------------------------------------------------------------
    // Step 2: Resolve all page URLs from sitemap(s)
    // -----------------------------------------------------------------------
    const pageUrls = await resolveSitemapPageUrls(
      sessionId,
      sitemapUrls,
      filterPattern,
      maxPages,
      context,
    );

    if (pageUrls.length === 0) {
      return {
        content: [
          {
            type: 'text',
            text: JSON.stringify(
              {
                error: 'Sitemap found but no matching URLs',
                sitemap_source: sitemapSource,
                filter: filterPattern || 'none',
                suggestion: 'Check the filter pattern or try without a filter.',
              },
              null,
              2,
            ),
          },
        ],
        isError: true,
      };
    }

    console.error(`[crawl_sitemap] Resolved ${pageUrls.length} page URLs from sitemap`);

    // -----------------------------------------------------------------------
    // Step 3: Crawl pages with concurrency limiter
    // -----------------------------------------------------------------------
    const limiter = createLimiter(concurrency);
    const urlsToVisit = pageUrls.slice(0, maxPages);

    // Process in batches to allow budget checking between batches
    const batchSize = concurrency;
    for (let i = 0; i < urlsToVisit.length; i += batchSize) {
      // Check budget before each batch
      if (context && !hasBudget(context, 15_000)) {
        console.error('[crawl_sitemap] Deadline approaching, stopping page crawl');
        break;
      }

      const batch = urlsToVisit.slice(i, i + batchSize);
      const batchResults = await Promise.all(
        batch.map((pageUrl) =>
          limiter(async () => {
            // Skip already visited URLs (deduplication)
            if (tracker.hasVisited(pageUrl)) {
              return null;
            }
            const cacheKey = crawlCache.key({
              url: pageUrl,
              outputFormat,
              engine,
              cacheScope,
              sessionFingerprint: sessionId,
              dimensions: {
                filter: filterPattern,
                onlyMainContent: cleanOpts.onlyMainContent,
                includeLinks: cleanOpts.includeLinks,
              },
            });
            if (canReadCache(cacheMode)) {
              const cached = crawlCache.read(cacheKey, cacheTtlMs);
              if (cached && !cached.stale) {
                tracker.visit(pageUrl);
                return {
                  ...cached.entry.page,
                  cache: crawlCache.metadata('hit', {
                    key: cacheKey,
                    scope: cacheScope,
                    hit: true,
                    createdAt: cached.entry.createdAt,
                    content_length: cached.entry.contentLength,
                  }),
                };
              }
            }
            tracker.visit(pageUrl);

            let page: CrawledPage;
            let staticReason: StaticReason | undefined;
            let engineUsed: 'static' | 'cdp' | undefined;

            if (engine === 'static' || engine === 'auto') {
              const staticResult = await fetchPageStatic(pageUrl, outputFormat, cleanOpts, context);
              if (staticResult.ok) {
                page = staticResult.page;
                engineUsed = 'static';
              } else if (engine === 'static') {
                page = {
                  url: pageUrl,
                  title: '',
                  content: '',
                  links_found: 0,
                  error: `static-insufficient: ${staticResult.reason}`,
                };
                engineUsed = 'static';
                staticReason = staticResult.reason;
              } else {
                page = await fetchPage(sessionId, pageUrl, outputFormat, cleanOpts, context);
                engineUsed = 'cdp';
                staticReason = staticResult.reason;
              }
            } else {
              page = await fetchPage(sessionId, pageUrl, outputFormat, cleanOpts, context);
              if (engineExplicit) engineUsed = 'cdp';
            }

            if (engineExplicit && engineUsed) {
              page.engine_used = engineUsed;
            }
            if (staticReason) {
              page.static_reason = staticReason;
            }
            if (cacheMode !== 'disabled') {
              const cacheStatus = cacheMode === 'write_only'
                ? 'write_only'
                : cacheMode === 'bypass'
                  ? 'bypass'
                  : 'miss';
              let cacheMeta = crawlCache.metadata(cacheStatus, {
                key: cacheKey,
                scope: cacheScope,
                hit: false,
                write: 'disabled',
              });
              if (canWriteCache(cacheMode) && !page.error) {
                const written = crawlCache.write({
                  key: cacheKey,
                  sourceUrl: pageUrl,
                  finalUrl: page.url,
                  page,
                  links: [],
                  cacheScope,
                });
                cacheMeta = crawlCache.metadata(cacheStatus, {
                  key: cacheKey,
                  scope: cacheScope,
                  hit: false,
                  write: written.stored ? 'stored' : 'skipped',
                  write_skipped_reason: written.reason,
                  createdAt: written.entry?.createdAt,
                  content_length: page.content.length,
                });
              }
              page.cache = cacheMeta;
            }

            return page;
          }),
        ),
      );

      for (const result of batchResults) {
        if (result) {
          pages.push(result);
        }
      }

      // #869 — emit progress per batch (after pages are pushed so `progress`
      // is monotonic). No-op when the client did not supply a progressToken.
      context?.reportProgress?.({
        progress: pages.length,
        total: urlsToVisit.length,
        message: batch[batch.length - 1] ?? undefined,
      });
    }

    // -----------------------------------------------------------------------
    // Step 4: Build output
    // -----------------------------------------------------------------------
    const durationMs = Date.now() - startTime;
    const succeeded = pages.filter((p) => !p.error).length;
    const failed = pages.filter((p) => p.error).length;

    const summary: CrawlSitemapSummary = {
      total_pages: pages.length,
      succeeded,
      failed,
      duration_ms: durationMs,
      sitemap_source: sitemapSource,
    };

    const buildOutput = (outputPages: CrawledPage[]) => includeMetrics
      ? {
          summary: {
            ...summary,
            metrics: {
              returned_chars: outputPages.reduce((sum, p) => sum + p.content.length, 0),
              estimated_tokens: outputPages.reduce((sum, p) => sum + buildTextMetrics(p.content).estimated_tokens, 0),
              truncated_pages: outputPages.filter((p) => p.content.includes('...[truncated]')).length,
              mode: `crawl_sitemap:${outputFormat}`,
            },
          },
          pages: outputPages.map((p) => ({
            ...p,
            metrics: buildTextMetrics(p.content, { mode: outputFormat }),
          })),
        }
      : { summary, pages: outputPages };

    // Ensure output fits within limits
    let outputJson = JSON.stringify(buildOutput(pages), null, 2);
    if (outputJson.length > MAX_OUTPUT_CHARS) {
      // Truncate page contents progressively to fit
      const truncatedPages = pages.map((p) => ({
        ...p,
        content:
          p.content.length > 2000
            ? p.content.slice(0, 2000) + '...[truncated]'
            : p.content,
      }));
      outputJson = JSON.stringify(buildOutput(truncatedPages), null, 2);

      // If still too large, remove content entirely
      if (outputJson.length > MAX_OUTPUT_CHARS) {
        const minimalPages = pages.map((p) => ({
          url: p.url,
          title: p.title,
          links_found: p.links_found,
          content_length: p.content.length,
          error: p.error,
          ...(includeMetrics && { metrics: buildTextMetrics('', { mode: outputFormat, truncated: true }) }),
        }));
        // Per-page metrics are computed from empty strings (content omitted),
        // so the summary metrics must align with what is actually emitted —
        // not the original full-content pages.
        const emptyPageMetrics = buildTextMetrics('', { mode: outputFormat, truncated: true });
        const minimalSummary = includeMetrics
          ? {
              ...summary,
              metrics: {
                returned_chars: minimalPages.reduce(
                  (sum, p) => sum + (p.metrics?.returned_chars ?? 0),
                  0,
                ),
                estimated_tokens: minimalPages.reduce(
                  (sum, p) => sum + (p.metrics?.estimated_tokens ?? emptyPageMetrics.estimated_tokens),
                  0,
                ),
                truncated_pages: pages.length,
                mode: `crawl_sitemap:${outputFormat}`,
              },
            }
          : summary;
        outputJson = JSON.stringify(
          { summary: minimalSummary, pages: minimalPages, note: 'Content omitted due to size constraints' },
          null,
          2,
        );
        if (outputJson.length > MAX_OUTPUT_CHARS) {
          outputJson = JSON.stringify({
            summary: minimalSummary,
            pages: minimalPages.map(({ url, title, links_found, content_length, error }) => ({ url, title, links_found, content_length, error })),
            note: 'Content omitted due to size constraints',
          }, null, 2);
        }
      }
    }

    return {
      content: [{ type: 'text', text: outputJson }],
    };
  } catch (error) {
    return {
      content: [
        {
          type: 'text',
          text: `crawl_sitemap error: ${error instanceof Error ? error.message : String(error)}`,
        },
      ],
      isError: true,
    };
  }
};

export function registerCrawlSitemapTool(server: MCPServer): void {
  server.registerTool('crawl_sitemap', handler, definition);
}
