import Foundation
#if canImport(WebKit)
import CoreProtocols
import SharedConstants
import WebKit
#endif

// MARK: - HTML to Markdown Converter
// File length: 580+ lines | Type body length: 400+ lines
// Disabling: file_length (400 line limit), type_body_length (250 line limit)

extension Core.Parser {
    /// Converts HTML documentation to clean Markdown
    public struct HTML: Core.Protocols.ContentTransformer, @unchecked Sendable {
        public typealias RawContent = String

        public init() {}

        // MARK: - ContentTransformer Protocol

        /// Transform HTML content to Markdown (protocol conformance)
        public func transform(_ content: String, url: URL) -> String? {
            Self.convert(content, url: url)
        }

        /// Extract links from HTML content (protocol conformance)
        public func extractLinks(from content: String) -> [URL] {
            Self.extractLinks(from: content)
        }

        // MARK: - Static API (backwards compatible)

        /// Extract links from HTML content
        public static func extractLinks(from html: String) -> [URL] {
            var links: [URL] = []
            let pattern = #"<a[^>]+href=["']([^"']+)["'][^>]*>"#
            if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) {
                let nsString = html as NSString
                let matches = regex.matches(in: html, range: NSRange(location: 0, length: nsString.length))
                for match in matches where match.numberOfRanges >= 2 {
                    let hrefRange = match.range(at: 1)
                    let href = nsString.substring(with: hrefRange)
                    if let url = URL(string: href) {
                        links.append(url)
                    }
                }
            }
            return links
        }

        /// Convert HTML string to Markdown
        public static func convert(_ html: String, url: URL) -> String {
            var markdown = ""

            // Add front matter with metadata
            markdown += "---\n"
            markdown += "source: \(url.absoluteString)\n"
            markdown += "crawled: \(ISO8601DateFormatter().string(from: Date()))\n"
            markdown += "---\n\n"

            // Extract title
            if let title = extractTitle(from: html) {
                markdown += "# \(title)\n\n"
            }

            // Extract main content
            let content = extractMainContent(from: html)
            markdown += convertHTMLToMarkdown(content)

            return markdown
        }

        // MARK: - Extraction

        private static func extractTitle(from html: String) -> String? {
            // Try to extract title from <h1> or <title> tags
            if let range = html.range(of: #"<h1[^>]*>(.*?)</h1>"#, options: .regularExpression) {
                let titleHTML = String(html[range])
                let title = stripHTML(titleHTML)
                // Skip if it's a JavaScript warning
                if !isJavaScriptWarning(title) {
                    return title
                }
            }

            if let range = html.range(of: #"<title>(.*?)</title>"#, options: .regularExpression) {
                let titleHTML = String(html[range])
                let title = stripHTML(titleHTML)
                // Skip if it's a JavaScript warning
                if !isJavaScriptWarning(title) {
                    return title
                }
            }

            return nil
        }

        private static func isJavaScriptWarning(_ text: String) -> Bool {
            let lowercased = text.lowercased().trimmingCharacters(in: .whitespacesAndNewlines)
            return lowercased.contains("requires javascript") ||
                lowercased.contains("enable javascript") ||
                lowercased.contains("javascript is required")
        }

        // MARK: - Error-page detection (#284)

        /// Returns true if `html` looks like Apple's React SPA rendering a
        /// no-content sub-view (HTTP 200 OK with the SPA shell but the inner
        /// doc-loader returned 404). These slip past `looksLikeHTTPErrorPage`
        /// because the HTTP status IS 200; we catch them with their
        /// distinctive React-app strings (#284 follow-up).
        public static func looksLikeJavaScriptFallback(html: String) -> Bool {
            // Apple's specific React sub-view phrases. Pinned literals because
            // they are unique to the React app's no-content states; real Apple
            // documentation pages do not quote either sentence.
            if html.contains("The page you're looking for can't be found") {
                return true
            }
            if html.contains("An unknown error occurred") {
                return true
            }
            return false
        }

        /// Returns true if `html` looks like an HTTP error response template
        /// (Apple's CDN sometimes serves a styled 403/404/502 page with HTTP
        /// 200 status, which then gets indexed as if it were documentation).
        /// Used by the crawler's WebView fallback as a defense-in-depth check
        /// before saving the page. The JSON API path already filters by
        /// HTTP status code, so this gate fires only on rendered-HTML inputs.
        public static func looksLikeHTTPErrorPage(html: String) -> Bool {
            guard let title = extractTitle(from: html) else {
                // No title means we'll skip via the existing nil-title gate; let
                // that path handle it rather than double-counting.
                return false
            }
            return looksLikeHTTPErrorPage(title: title, html: html)
        }

        /// Pure decision over a pre-extracted `title` + the surrounding `html`.
        /// Split out so unit tests can exercise the rule against synthetic
        /// inputs without round-tripping through the full HTML title-extractor.
        static func looksLikeHTTPErrorPage(title: String, html: String) -> Bool {
            let trimmed = title.trimmingCharacters(in: .whitespacesAndNewlines)

            // Direct prefix: "403 Forbidden", "502 Bad Gateway", etc.
            // The space/end-anchor matches the issue spec so a real Apple page
            // titled "404 Not Found · Apple Developer Documentation" still trips
            // it, while "Routing404" or "Error404Type" do not.
            if trimmed.range(of: #"^(403|404|429|500|502|503|504)(\s|$)"#, options: .regularExpression) != nil {
                return true
            }

            // Defense-in-depth: a very short rendered body whose title contains
            // a known error phrase. The issue spec calls for a content-body
            // word-count (not the converted-markdown word-count, which would
            // include the front-matter overhead `convert()` always emits).
            let errorPhrases = ["Forbidden", "Bad Gateway", "Not Found", "Service Unavailable", "Gateway Timeout"]
            let containsErrorPhrase = errorPhrases.contains { trimmed.contains($0) }
            if containsErrorPhrase, countBodyWords(in: html) < 10 {
                return true
            }

            return false
        }

        /// Approximate body word-count for the error-page heuristic. Strips
        /// HTML tags + entities from the `<body>` region (or the whole input
        /// if no body tag), then counts whitespace-separated tokens. Cheap
        /// enough to run on every WebView fallback page; the only consumer is
        /// `looksLikeHTTPErrorPage`.
        private static func countBodyWords(in html: String) -> Int {
            let body: String
            let bodyOptions: NSString.CompareOptions = [.regularExpression, .caseInsensitive]
            if let range = html.range(of: #"<body[^>]*>(.*?)</body>"#, options: bodyOptions) {
                body = String(html[range])
            } else {
                body = html
            }
            let stripped = body
                .replacingOccurrences(of: "<[^>]+>", with: " ", options: .regularExpression)
                .replacingOccurrences(of: "&[a-z0-9#]+;", with: " ", options: bodyOptions)
            return stripped.split(whereSeparator: { $0.isWhitespace }).count
        }

        private static func extractMainContent(from html: String) -> String {
            // Try to extract main content area - need dotMatchesLineSeparators for multiline content
            let options: NSRegularExpression.Options = [.caseInsensitive, .dotMatchesLineSeparators]

            if let regex = try? NSRegularExpression(pattern: #"<main[^>]*>(.*?)</main>"#, options: options),
               let match = regex.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)),
               let range = Range(match.range, in: html) {
                return String(html[range])
            }

            if let regex = try? NSRegularExpression(pattern: #"<article[^>]*>(.*?)</article>"#, options: options),
               let match = regex.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)),
               let range = Range(match.range, in: html) {
                return String(html[range])
            }

            // Fallback to body content
            if let regex = try? NSRegularExpression(pattern: #"<body[^>]*>(.*?)</body>"#, options: options),
               let match = regex.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)),
               let range = Range(match.range, in: html) {
                return String(html[range])
            }

            return html
        }

        // MARK: - Conversion

        private static func convertHTMLToMarkdown(_ html: String) -> String {
            var markdown = html

            // Step 1: Remove unwanted sections BEFORE extracting content
            markdown = removeUnwantedSections(markdown)

            // Step 2: Extract and protect code blocks to prevent them from being mangled
            var codeBlocks: [String: String] = [:]
            markdown = extractAndProtectCodeBlocks(markdown, into: &codeBlocks)

            // Step 3: Remove UI clutter
            markdown = removeJavaScriptWarnings(markdown)
            markdown = removeAccessibilityInstructions(markdown)

            // Step 4: Convert HTML elements to markdown
            markdown = convertHeaders(markdown)
            markdown = convertInlineFormatting(markdown)
            markdown = convertLinks(markdown)
            markdown = convertLists(markdown)
            markdown = convertParagraphs(markdown)

            // Step 5: Final cleanup
            markdown = stripHTML(markdown)
            markdown = cleanupWhitespace(markdown)
            markdown = decodeHTMLEntities(markdown)

            // Step 6: Restore protected code blocks
            markdown = restoreCodeBlocks(markdown, from: codeBlocks)

            return markdown
        }

        private static func extractAndProtectCodeBlocks(
            _ html: String,
            into storage: inout [String: String]
        ) -> String {
            var result = html
            var blockIndex = 0
            let regexOptions: NSRegularExpression.Options = [.caseInsensitive, .dotMatchesLineSeparators]

            // Extract <pre><code> blocks with language
            let pattern = Shared.Constants.Pattern.htmlCodeBlockWithLanguage
            if let regex = try? NSRegularExpression(pattern: pattern, options: regexOptions) {
                let nsString = result as NSString
                let matches = regex.matches(in: result, range: NSRange(location: 0, length: nsString.length))

                for match in matches.reversed() where match.numberOfRanges >= 3 {
                    let languageRange = match.range(at: 1)
                    let codeRange = match.range(at: 2)

                    if languageRange.location != NSNotFound, codeRange.location != NSNotFound {
                        let language = nsString.substring(with: languageRange).lowercased()
                        var code = nsString.substring(with: codeRange)

                        // Strip HTML tags and decode entities from code
                        code = stripHTML(code)
                        code = decodeHTMLEntities(code)

                        let placeholder = "___CODEBLOCK_\(blockIndex)___"
                        storage[placeholder] = "```\(language)\n\(code)\n```"
                        result = (result as NSString).replacingCharacters(in: match.range, with: placeholder)
                        blockIndex += 1
                    }
                }
            }

            // Fallback: Extract <pre><code> blocks without language
            let fallbackPattern = #"<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>"#
            if let regex = try? NSRegularExpression(pattern: fallbackPattern, options: regexOptions) {
                let nsString = result as NSString
                let matches = regex.matches(in: result, range: NSRange(location: 0, length: nsString.length))

                for match in matches.reversed() where match.numberOfRanges >= 2 {
                    let codeRange = match.range(at: 1)
                    if codeRange.location != NSNotFound {
                        var code = nsString.substring(with: codeRange)

                        // Strip HTML tags and decode entities from code
                        code = stripHTML(code)
                        code = decodeHTMLEntities(code)

                        let placeholder = "___CODEBLOCK_\(blockIndex)___"
                        storage[placeholder] = "```\n\(code)\n```"
                        result = (result as NSString).replacingCharacters(in: match.range, with: placeholder)
                        blockIndex += 1
                    }
                }
            }

            return result
        }

        private static func restoreCodeBlocks(_ markdown: String, from storage: [String: String]) -> String {
            var result = markdown
            for (placeholder, code) in storage {
                result = result.replacingOccurrences(of: placeholder, with: "\n\n\(code)\n\n")
            }
            return result
        }

        private static func convertHeaders(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(
                of: #"<h1[^>]*>(.*?)</h1>"#, with: "# $1\n\n", options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"<h2[^>]*>(.*?)</h2>"#, with: "## $1\n\n", options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"<h3[^>]*>(.*?)</h3>"#, with: "### $1\n\n", options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"<h4[^>]*>(.*?)</h4>"#, with: "#### $1\n\n", options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"<h5[^>]*>(.*?)</h5>"#, with: "##### $1\n\n", options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"<h6[^>]*>(.*?)</h6>"#, with: "###### $1\n\n", options: .regularExpression
            )
            return result
        }

        private static func convertInlineFormatting(_ markdown: String) -> String {
            var result = markdown
            let regexOptions: NSRegularExpression.Options = [.caseInsensitive, .dotMatchesLineSeparators]

            // Inline code (use dotMatchesLineSeparators to handle multiline code)
            if let regex = try? NSRegularExpression(pattern: #"<code[^>]*>(.*?)</code>"#, options: regexOptions) {
                let nsString = result as NSString
                let range = NSRange(location: 0, length: nsString.length)
                result = regex.stringByReplacingMatches(in: result, options: [], range: range, withTemplate: "`$1`")
            }

            // Bold
            if let regex = try? NSRegularExpression(pattern: #"<(strong|b)[^>]*>(.*?)</\1>"#, options: regexOptions) {
                let nsString = result as NSString
                let range = NSRange(location: 0, length: nsString.length)
                result = regex.stringByReplacingMatches(in: result, options: [], range: range, withTemplate: "**$2**")
            }

            // Italic
            if let regex = try? NSRegularExpression(pattern: #"<(em|i)[^>]*>(.*?)</\1>"#, options: regexOptions) {
                let nsString = result as NSString
                let range = NSRange(location: 0, length: nsString.length)
                result = regex.stringByReplacingMatches(in: result, options: [], range: range, withTemplate: "*$2*")
            }

            return result
        }

        private static func convertLinks(_ markdown: String) -> String {
            markdown.replacingOccurrences(
                of: #"<a[^>]*href=[\"']([^\"']*)[\"'][^>]*>(.*?)</a>"#,
                with: "[$2]($1)",
                options: .regularExpression
            )
        }

        private static func convertLists(_ markdown: String) -> String {
            var result = markdown

            // Convert list items first (need dotMatchesLineSeparators for multiline items)
            if let regex = try? NSRegularExpression(
                pattern: #"<li[^>]*>(.*?)</li>"#,
                options: [.caseInsensitive, .dotMatchesLineSeparators]
            ) {
                let nsString = result as NSString
                result = regex.stringByReplacingMatches(
                    in: result,
                    options: [],
                    range: NSRange(location: 0, length: nsString.length),
                    withTemplate: "\n- $1\n"
                )
            }

            // Then remove list containers
            if let regex = try? NSRegularExpression(
                pattern: #"<ul[^>]*>(.*?)</ul>"#,
                options: [.caseInsensitive, .dotMatchesLineSeparators]
            ) {
                let nsString = result as NSString
                result = regex.stringByReplacingMatches(
                    in: result,
                    options: [],
                    range: NSRange(location: 0, length: nsString.length),
                    withTemplate: "$1\n\n"
                )
            }

            if let regex = try? NSRegularExpression(
                pattern: #"<ol[^>]*>(.*?)</ol>"#,
                options: [.caseInsensitive, .dotMatchesLineSeparators]
            ) {
                let nsString = result as NSString
                result = regex.stringByReplacingMatches(
                    in: result,
                    options: [],
                    range: NSRange(location: 0, length: nsString.length),
                    withTemplate: "$1\n\n"
                )
            }

            return result
        }

        private static func convertParagraphs(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(of: #"<p[^>]*>(.*?)</p>"#, with: "$1\n\n", options: .regularExpression)
            result = result.replacingOccurrences(of: "<br[^>]*>", with: "\n", options: .regularExpression)
            return result
        }

        private static func cleanupWhitespace(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(of: #"\n{3,}"#, with: "\n\n", options: .regularExpression)
            result = result.trimmingCharacters(in: .whitespacesAndNewlines)
            return result
        }

        // MARK: - Utilities

        private static func removeUnwantedSections(_ html: String) -> String {
            var result = html

            // Remove noscript tags and their content
            result = result.replacingOccurrences(
                of: #"<noscript[^>]*>.*?</noscript>"#,
                with: "",
                options: [.regularExpression, .caseInsensitive]
            )

            // Remove script tags
            result = result.replacingOccurrences(
                of: #"<script[^>]*>.*?</script>"#,
                with: "",
                options: [.regularExpression, .caseInsensitive]
            )

            // Remove style tags
            result = result.replacingOccurrences(
                of: #"<style[^>]*>.*?</style>"#,
                with: "",
                options: [.regularExpression, .caseInsensitive]
            )

            // Remove navigation elements
            result = result.replacingOccurrences(
                of: #"<nav[^>]*>.*?</nav>"#,
                with: "",
                options: [.regularExpression, .caseInsensitive]
            )

            // Remove header/footer elements (often contain navigation)
            result = result.replacingOccurrences(
                of: #"<header[^>]*>.*?</header>"#,
                with: "",
                options: [.regularExpression, .caseInsensitive]
            )
            result = result.replacingOccurrences(
                of: #"<footer[^>]*>.*?</footer>"#,
                with: "",
                options: [.regularExpression, .caseInsensitive]
            )

            // Remove SVG elements (icons, graphics) - use dotMatchesLineSeparators for multiline SVG
            if let regex = try? NSRegularExpression(
                pattern: #"<svg[^>]*>.*?</svg>"#,
                options: [.caseInsensitive, .dotMatchesLineSeparators]
            ) {
                let nsString = result as NSString
                result = regex.stringByReplacingMatches(
                    in: result,
                    options: [],
                    range: NSRange(location: 0, length: nsString.length),
                    withTemplate: ""
                )
            }

            return result
        }

        private static func removeJavaScriptWarnings(_ html: String) -> String {
            var result = html

            // Common JavaScript warning patterns - remove from anywhere in text
            let warnings = [
                "This page requires JavaScript.",
                "This page requires JavaScript",
                "Please turn on JavaScript",
                "Please enable JavaScript",
                "JavaScript is required",
                "Enable JavaScript to view",
            ]

            for warning in warnings {
                result = result.replacingOccurrences(of: warning, with: "", options: .caseInsensitive)
            }

            // Remove common heading patterns for JavaScript warnings (with any number of #)
            result = result.replacingOccurrences(
                of: #"#{1,6}\s*This page requires JavaScript\.?\s*\n+"#,
                with: "",
                options: .regularExpression
            )

            // Also remove if it appears as plain text at start of line
            if let regex = try? NSRegularExpression(
                pattern: #"^\s*This page requires JavaScript\.?\s*\n+"#,
                options: [.anchorsMatchLines]
            ) {
                let nsString = result as NSString
                result = regex.stringByReplacingMatches(
                    in: result,
                    options: [],
                    range: NSRange(location: 0, length: nsString.length),
                    withTemplate: ""
                )
            }

            return result
        }

        private static func removeAccessibilityInstructions(_ markdown: String) -> String {
            var result = markdown
            result = removeNavigationInstructions(result)
            result = removeSymbolIndicators(result)
            result = removeSkipNavigation(result)
            result = removeArtifacts(result)
            result = cleanupStrayCharacters(result)
            return result
        }

        private static func removeNavigationInstructions(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(
                of: #"To navigate the symbols, press Up Arrow, Down Arrow, Left Arrow or Right Arrow\s*"#,
                with: "",
                options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"\d+ items were found\. Tab back to navigate through them\.\s*"#,
                with: "",
                options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"/\s*Navigator is ready -\s*"#,
                with: "",
                options: .regularExpression
            )
            return result
        }

        private static func removeSymbolIndicators(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(
                of: #"\s*[A-Za-z0-9#]*\d+ of \d+ symbols inside [^\n\[\]]+\s*"#,
                with: " ",
                options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"containing \d+ symbols"#,
                with: "",
                options: .regularExpression
            )
            return result
        }

        private static func removeSkipNavigation(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(
                of: #"\[\s*Skip Navigation\s*\]\s*\(#[^)]*\)"#,
                with: "",
                options: .regularExpression
            )
            result = result.replacingOccurrences(
                of: #"\[\s*Skip Navigation\s*\]"#,
                with: "",
                options: .regularExpression
            )
            return result
        }

        private static func removeArtifacts(_ markdown: String) -> String {
            var result = markdown
            result = result.replacingOccurrences(of: #"\[object Object\]"#, with: "", options: .regularExpression)
            result = result.replacingOccurrences(of: #"\s*data-v-[a-z0-9]+="[^"]*""#, with: "", options: .regularExpression)
            result = result.replacingOccurrences(of: #"  +"#, with: " ", options: .regularExpression)
            result = result.replacingOccurrences(of: #"\n\n\n+"#, with: "\n\n", options: .regularExpression)
            return result
        }

        private static func cleanupStrayCharacters(_ markdown: String) -> String {
            var result = markdown
            if let regex = try? NSRegularExpression(
                pattern: #"^\s*">+\s*"#,
                options: [.anchorsMatchLines]
            ) {
                let nsString = result as NSString
                result = regex.stringByReplacingMatches(
                    in: result,
                    options: [],
                    range: NSRange(location: 0, length: nsString.length),
                    withTemplate: ""
                )
            }
            result = result.replacingOccurrences(
                of: #"\n\s*">+\s*\n"#,
                with: "\n",
                options: .regularExpression
            )
            return result
        }

        private static func stripHTML(_ html: String) -> String {
            html.replacingOccurrences(of: "<[^>]+>", with: "", options: .regularExpression)
        }

        private static func decodeHTMLEntities(_ text: String) -> String {
            var result = text

            // Common HTML entities
            let entities: [String: String] = [
                "&amp;": "&",
                "&lt;": "<",
                "&gt;": ">",
                "&quot;": "\"",
                "&apos;": "'",
                "&nbsp;": " ",
                "&#39;": "'",
                "&#x27;": "'",
            ]

            for (entity, replacement) in entities {
                result = result.replacingOccurrences(of: entity, with: replacement)
            }

            // Numeric entities (&#123;) - simple regex replacement
            if let regex = try? NSRegularExpression(pattern: #"&#(\d+);"#, options: []) {
                let nsString = result as NSString
                let matches = regex.matches(in: result, range: NSRange(location: 0, length: nsString.length))

                for match in matches.reversed() where match.numberOfRanges >= 2 {
                    let numberRange = match.range(at: 1)
                    let numberStr = nsString.substring(with: numberRange)

                    if let number = Int(numberStr),
                       let scalar = Unicode.Scalar(number) {
                        let replacement = String(Character(scalar))
                        result = (result as NSString).replacingCharacters(in: match.range, with: replacement)
                    }
                }
            }

            return result
        }
    }
}

// closes extension Core.Parser

// MARK: - String Extension for Regex Replacements

extension String {
    func replacingOccurrences(
        of pattern: String,
        with template: String,
        options: NSRegularExpression.Options = [],
        using closure: ((Substring) -> String)? = nil
    ) -> String {
        guard let regex = try? NSRegularExpression(pattern: pattern, options: options) else {
            return self
        }

        let range = NSRange(startIndex..., in: self)
        var result = self

        let matches = regex.matches(in: self, range: range).reversed()

        for match in matches {
            if let closure {
                if let matchRange = Range(match.range, in: self) {
                    let replacement = closure(self[matchRange])
                    result.replaceSubrange(matchRange, with: replacement)
                }
            } else {
                result = regex.stringByReplacingMatches(
                    in: result,
                    range: match.range,
                    withTemplate: template
                )
            }
        }

        return result
    }
}

// MARK: - HTML to StructuredDocumentationPage Converter

extension Core.Parser.HTML {
    /// Convert HTML to a StructuredDocumentationPage (best-effort extraction)
    /// Note: This is less structured than JSON API output since HTML doesn't
    /// have explicit semantic structure
    public static func toStructuredPage(
        _ html: String,
        url: URL,
        source: Shared.Models.StructuredDocumentationPage.Source = .appleWebKit,
        depth: Int? = nil
    ) -> Shared.Models.StructuredDocumentationPage? {
        // Extract title
        guard let title = extractTitle(from: html) else {
            return nil
        }

        // Detect source from URL
        let detectedSource = detectSource(from: url, provided: source)

        // Extract kind (best-effort from HTML structure)
        let kind = detectKind(from: html, url: url)

        // Extract abstract (first paragraph after title)
        let abstract = extractAbstract(from: html)

        // Extract declaration (first code block that looks like a declaration)
        let declaration = extractDeclarationFromHTML(from: html)

        // Extract code examples
        let codeExamples = extractCodeExamplesFromHTML(from: html)

        // Extract overview (paragraphs before sections)
        let overview = extractOverviewFromHTML(from: html)

        // Extract sections (H2 headers and their content)
        let sections = extractSectionsFromHTML(from: html)

        // Generate markdown representation
        let markdown = convert(html, url: url)

        // Hash canonical structured fields, not raw `html` — page bytes carry
        // volatile cache/build metadata that doesn't reach our parsed output,
        // so `sha256(of: html)` was non-deterministic across runs (#199).
        let page = Shared.Models.StructuredDocumentationPage(
            id: Shared.Models.StructuredDocumentationPage.deterministicID(for: url),
            url: url,
            title: title,
            kind: kind,
            source: detectedSource,
            abstract: abstract,
            declaration: declaration,
            overview: overview,
            sections: sections,
            codeExamples: codeExamples,
            rawMarkdown: markdown,
            crawledAt: Date(),
            contentHash: "",
            crawlDepth: depth
        )
        return page.with(contentHash: page.canonicalContentHash)
    }

    // MARK: - Private Helpers for Structured Page

    private static func detectSource(
        from url: URL,
        provided: Shared.Models.StructuredDocumentationPage.Source
    ) -> Shared.Models.StructuredDocumentationPage.Source {
        guard let host = url.host?.lowercased() else {
            return provided
        }

        if host.contains("developer.apple.com") {
            return .appleWebKit
        } else if host.contains("swift.org") || host.contains("docs.swift.org") {
            return .swiftOrg
        } else if host.contains("github.com") || host.contains("github.io") {
            return .github
        }

        return provided
    }

    private static func detectKind(
        from html: String,
        url: URL
    ) -> Shared.Models.StructuredDocumentationPage.Kind {
        let lowercased = html.lowercased()

        // Check URL path for hints
        let path = url.path.lowercased()
        if path.contains("/tutorials/") {
            return .tutorial
        }

        // Check HTML content for role indicators
        if let roleMatch = lowercased.range(
            of: #"<span[^>]*class="[^"]*role[^"]*"[^>]*>([^<]+)</span>"#,
            options: .regularExpression
        ) {
            let role = String(lowercased[roleMatch]).lowercased()
            if role.contains("protocol") { return .protocol }
            if role.contains("class") { return .class }
            if role.contains("struct") { return .struct }
            if role.contains("enum") { return .enum }
            if role.contains("function") { return .function }
            if role.contains("property") { return .property }
            if role.contains("method") { return .method }
        }

        // Check for declaration patterns in code blocks
        if lowercased.contains("protocol "), lowercased.contains("<code") {
            return .protocol
        }
        if lowercased.contains("class "), lowercased.contains("<code") {
            return .class
        }
        if lowercased.contains("struct "), lowercased.contains("<code") {
            return .struct
        }
        if lowercased.contains("enum "), lowercased.contains("<code") {
            return .enum
        }

        // Check meta description for hints
        if let metaMatch = lowercased.range(
            of: #"<meta[^>]*name="description"[^>]*content="([^"]+)""#,
            options: .regularExpression
        ) {
            let desc = String(lowercased[metaMatch])
            if desc.contains("protocol") { return .protocol }
            if desc.contains("class") { return .class }
            if desc.contains("struct") { return .struct }
        }

        return .unknown
    }

    private static func extractAbstract(from html: String) -> String? {
        // Look for og:description meta tag
        if let regex = try? NSRegularExpression(
            pattern: #"<meta[^>]*property="og:description"[^>]*content="([^"]+)""#,
            options: .caseInsensitive
        ) {
            let nsString = html as NSString
            if let match = regex.firstMatch(
                in: html,
                range: NSRange(location: 0, length: nsString.length)
            ),
                match.numberOfRanges >= 2 {
                let contentRange = match.range(at: 1)
                let content = nsString.substring(with: contentRange)
                let decoded = decodeHTMLEntities(content)
                if !decoded.isEmpty {
                    return decoded
                }
            }
        }

        // Fallback to first <p> in main content
        let mainContent = extractMainContent(from: html)
        if let regex = try? NSRegularExpression(
            pattern: #"<p[^>]*>(.*?)</p>"#,
            options: [.caseInsensitive, .dotMatchesLineSeparators]
        ) {
            let nsString = mainContent as NSString
            if let match = regex.firstMatch(
                in: mainContent,
                range: NSRange(location: 0, length: nsString.length)
            ),
                match.numberOfRanges >= 2 {
                let contentRange = match.range(at: 1)
                var content = nsString.substring(with: contentRange)
                content = stripHTML(content)
                content = decodeHTMLEntities(content)
                let trimmed = content.trimmingCharacters(in: .whitespacesAndNewlines)
                if !trimmed.isEmpty, trimmed.count > 10 {
                    return trimmed
                }
            }
        }

        return nil
    }

    private static func extractDeclarationFromHTML(
        from html: String
    ) -> Shared.Models.StructuredDocumentationPage.Declaration? {
        let mainContent = extractMainContent(from: html)
        let regexOptions: NSRegularExpression.Options = [.caseInsensitive, .dotMatchesLineSeparators]

        // Look for code block with language indicator
        let pattern = Shared.Constants.Pattern.htmlCodeBlockWithLanguage
        if let regex = try? NSRegularExpression(pattern: pattern, options: regexOptions) {
            let nsString = mainContent as NSString
            if let match = regex.firstMatch(
                in: mainContent,
                range: NSRange(location: 0, length: nsString.length)
            ),
                match.numberOfRanges >= 3 {
                let languageRange = match.range(at: 1)
                let codeRange = match.range(at: 2)

                if languageRange.location != NSNotFound, codeRange.location != NSNotFound {
                    let language = nsString.substring(with: languageRange).lowercased()
                    var code = nsString.substring(with: codeRange)
                    code = stripHTML(code)
                    code = decodeHTMLEntities(code)
                    let trimmed = code.trimmingCharacters(in: .whitespacesAndNewlines)

                    // Check if this looks like a declaration
                    if looksLikeDeclaration(trimmed) {
                        return Shared.Models.StructuredDocumentationPage.Declaration(
                            code: trimmed,
                            language: language.isEmpty ? "swift" : language
                        )
                    }
                }
            }
        }

        // Fallback: look for any code block that looks like a declaration
        let fallbackPattern = #"<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>"#
        if let regex = try? NSRegularExpression(pattern: fallbackPattern, options: regexOptions) {
            let nsString = mainContent as NSString
            let matches = regex.matches(
                in: mainContent,
                range: NSRange(location: 0, length: nsString.length)
            )

            for match in matches where match.numberOfRanges >= 2 {
                let codeRange = match.range(at: 1)
                if codeRange.location != NSNotFound {
                    var code = nsString.substring(with: codeRange)
                    code = stripHTML(code)
                    code = decodeHTMLEntities(code)
                    let trimmed = code.trimmingCharacters(in: .whitespacesAndNewlines)

                    if looksLikeDeclaration(trimmed) {
                        return Shared.Models.StructuredDocumentationPage.Declaration(
                            code: trimmed,
                            language: "swift"
                        )
                    }
                }
            }
        }

        return nil
    }

    private static func looksLikeDeclaration(_ code: String) -> Bool {
        let keywords = ["protocol ", "class ", "struct ", "enum ", "func ", "var ", "let ", "@"]
        return keywords.contains { code.hasPrefix($0) }
    }

    private static func extractCodeExamplesFromHTML(
        from html: String
    ) -> [Shared.Models.StructuredDocumentationPage.CodeExample] {
        var examples: [Shared.Models.StructuredDocumentationPage.CodeExample] = []
        let mainContent = extractMainContent(from: html)
        let regexOptions: NSRegularExpression.Options = [.caseInsensitive, .dotMatchesLineSeparators]

        // Extract code blocks with language
        let pattern = Shared.Constants.Pattern.htmlCodeBlockWithLanguage
        if let regex = try? NSRegularExpression(pattern: pattern, options: regexOptions) {
            let nsString = mainContent as NSString
            let matches = regex.matches(
                in: mainContent,
                range: NSRange(location: 0, length: nsString.length)
            )

            for match in matches where match.numberOfRanges >= 3 {
                let languageRange = match.range(at: 1)
                let codeRange = match.range(at: 2)

                if codeRange.location != NSNotFound {
                    let language = languageRange.location != NSNotFound
                        ? nsString.substring(with: languageRange).lowercased()
                        : nil
                    var code = nsString.substring(with: codeRange)
                    code = stripHTML(code)
                    code = decodeHTMLEntities(code)
                    let trimmed = code.trimmingCharacters(in: .whitespacesAndNewlines)

                    // Skip if this looks like a declaration (already captured)
                    if !looksLikeDeclaration(trimmed), !trimmed.isEmpty {
                        examples.append(Shared.Models.StructuredDocumentationPage.CodeExample(
                            code: trimmed,
                            language: language
                        ))
                    }
                }
            }
        }

        return examples
    }

    private static func extractOverviewFromHTML(from html: String) -> String? {
        let mainContent = extractMainContent(from: html)
        var paragraphs: [String] = []
        let regexOptions: NSRegularExpression.Options = [.caseInsensitive, .dotMatchesLineSeparators]

        if let regex = try? NSRegularExpression(
            pattern: #"<p[^>]*>(.*?)</p>"#,
            options: regexOptions
        ) {
            let nsString = mainContent as NSString
            let matches = regex.matches(
                in: mainContent,
                range: NSRange(location: 0, length: nsString.length)
            )

            // Take up to first 3 paragraphs as overview
            for match in matches.prefix(3) where match.numberOfRanges >= 2 {
                let contentRange = match.range(at: 1)
                if contentRange.location != NSNotFound {
                    var content = nsString.substring(with: contentRange)
                    content = stripHTML(content)
                    content = decodeHTMLEntities(content)
                    let trimmed = content.trimmingCharacters(in: .whitespacesAndNewlines)
                    if !trimmed.isEmpty, trimmed.count > 20 {
                        paragraphs.append(trimmed)
                    }
                }
            }
        }

        return paragraphs.isEmpty ? nil : paragraphs.joined(separator: "\n\n")
    }

    private static func extractSectionsFromHTML(
        from html: String
    ) -> [Shared.Models.StructuredDocumentationPage.Section] {
        var sections: [Shared.Models.StructuredDocumentationPage.Section] = []
        let mainContent = extractMainContent(from: html)

        // Find all H2 headers
        if let regex = try? NSRegularExpression(
            pattern: #"<h2[^>]*>(.*?)</h2>"#,
            options: [.caseInsensitive, .dotMatchesLineSeparators]
        ) {
            let nsString = mainContent as NSString
            let matches = regex.matches(
                in: mainContent,
                range: NSRange(location: 0, length: nsString.length)
            )

            for match in matches where match.numberOfRanges >= 2 {
                let titleRange = match.range(at: 1)
                if titleRange.location != NSNotFound {
                    var title = nsString.substring(with: titleRange)
                    title = stripHTML(title)
                    title = decodeHTMLEntities(title)
                    let trimmedTitle = title.trimmingCharacters(in: .whitespacesAndNewlines)

                    if !trimmedTitle.isEmpty {
                        sections.append(Shared.Models.StructuredDocumentationPage.Section(
                            title: trimmedTitle
                        ))
                    }
                }
            }
        }

        return sections
    }
}
