package sqlite

import (
	"bytes"
	"context"
	"crypto/sha256"
	"database/sql"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/yazanabuashour/openclerk/internal/domain"
	"html"
	"io"
	"io/fs"
	"mime"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"rsc.io/pdf"
	"sort"
	"strings"
	"time"
	"unicode/utf8"
)

type pdfExtraction struct {
	Metadata domain.SourcePDFMetadata
	Text     string
	Pages    int
}

type webExtraction struct {
	Title string
	Text  string
}

type sourceDownload struct {
	Body       []byte
	MIMEType   string
	SourceType string
	FinalURL   string
}

type normalizedSourceURLRequest struct {
	URL           string
	Mode          string
	RequestedType string
	LookupURLs    []string
}

type noteMutationLabels struct {
	WriteAsset     string
	WriteNote      string
	RestoreAsset   string
	RestoreNote    string
	RestoreIndexed string
}

var sourceHTTPClient = defaultSourceHTTPClient
var allowPrivateSourceHostsForTests = false

const (
	maxSourceURLUpdateImpactRefs = 200
	maxSourceInspectionLinks     = 40
	maxSourceInspectionLinkScans = 1000
	maxSourceTextPreviewRunes    = 1200
	maxSourcePDFPages            = 200
	maxSourcePDFTextObjects      = 200000
	maxSourcePDFExtractedRunes   = 1000000
	maxSourcePDFExtractionTime   = 10 * time.Second
)

var blockedPublicSourceIPPrefixes = []netip.Prefix{
	netip.MustParsePrefix("0.0.0.0/8"),
	netip.MustParsePrefix("100.64.0.0/10"),
	netip.MustParsePrefix("192.0.0.0/24"),
	netip.MustParsePrefix("192.0.2.0/24"),
	netip.MustParsePrefix("198.18.0.0/15"),
	netip.MustParsePrefix("198.51.100.0/24"),
	netip.MustParsePrefix("203.0.113.0/24"),
	netip.MustParsePrefix("240.0.0.0/4"),
	netip.MustParsePrefix("255.255.255.255/32"),
	netip.MustParsePrefix("100::/64"),
	netip.MustParsePrefix("2001::/23"),
	netip.MustParsePrefix("2001:db8::/32"),
	netip.MustParsePrefix("2002::/16"),
	netip.MustParsePrefix("64:ff9b:1::/48"),
}

func defaultSourceHTTPClient(checkRedirect func(*http.Request, []*http.Request) error) *http.Client {
	return &http.Client{
		Timeout:       30 * time.Second,
		CheckRedirect: checkRedirect,
		Transport: &http.Transport{
			DialContext: dialPublicSourceContext,
		},
	}
}

func (s *Store) IngestSourceURL(ctx context.Context, input domain.SourceURLInput) (domain.SourceIngestionResult, error) {
	request, err := normalizeSourceURLRequest(input)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	switch request.Mode {
	case sourceURLModeCreate:
		return s.createSourceURL(ctx, input, request.URL, request.RequestedType, request.LookupURLs)
	case sourceURLModeUpdate:
		return s.updateSourceURL(ctx, input, request.URL, request.RequestedType, request.LookupURLs)
	default:
		return domain.SourceIngestionResult{}, domain.ValidationError("source mode must be create or update", map[string]any{"mode": input.Mode})
	}
}

func normalizeSourceURLRequest(input domain.SourceURLInput) (normalizedSourceURLRequest, error) {
	sourceURL, lookupURLs, err := normalizeDocumentSourceURLWithAliases(input.URL)
	if err != nil {
		return normalizedSourceURLRequest{}, err
	}
	mode, err := normalizeSourceURLMode(input.Mode)
	if err != nil {
		return normalizedSourceURLRequest{}, err
	}
	requestedType, err := normalizeSourceType(input.SourceType)
	if err != nil {
		return normalizedSourceURLRequest{}, err
	}
	return normalizedSourceURLRequest{
		URL:           sourceURL,
		Mode:          mode,
		RequestedType: requestedType,
		LookupURLs:    lookupURLs,
	}, nil
}

func (s *Store) InspectSourceURL(ctx context.Context, input domain.SourceURLInput) (domain.SourceURLInspection, error) {
	sourceURL, _, err := normalizeDocumentSourceURLWithAliases(input.URL)
	if err != nil {
		return domain.SourceURLInspection{}, err
	}
	requestedType, err := normalizeSourceType(input.SourceType)
	if err != nil {
		return domain.SourceURLInspection{}, err
	}
	download, err := downloadSource(ctx, sourceURL, requestedType)
	if err != nil {
		return domain.SourceURLInspection{}, err
	}
	sha := sha256.Sum256(download.Body)
	shaHex := hex.EncodeToString(sha[:])
	inspection := domain.SourceURLInspection{
		SourceURL:  sourceURL,
		SourceType: download.SourceType,
		SHA256:     shaHex,
		SizeBytes:  int64(len(download.Body)),
		MIMEType:   download.MIMEType,
	}
	switch download.SourceType {
	case sourceTypePDF:
		extracted, err := extractPDFBytes(download.Body)
		if err != nil {
			return domain.SourceURLInspection{}, err
		}
		inspection.Title = resolvedInspectionTitle(input.Title, extracted.Metadata.Title, sourceURL)
		inspection.TextPreview = sourceTextPreview(extracted.Text)
		inspection.PageCount = extracted.Pages
		inspection.PDFMetadata = extracted.Metadata
	case sourceTypeWeb:
		linkBaseURL := firstNonEmpty(download.FinalURL, sourceURL)
		extracted, err := extractWebSource(linkBaseURL, download.MIMEType, download.Body)
		if err != nil {
			return domain.SourceURLInspection{}, err
		}
		inspection.Title = resolvedInspectionTitle(input.Title, extracted.Title, sourceURL)
		inspection.TextPreview = sourceTextPreview(extracted.Text)
		inspection.Links = extractSourceLinks(linkBaseURL, download.MIMEType, download.Body, maxSourceInspectionLinks)
	default:
		return domain.SourceURLInspection{}, domain.ValidationError("source URL returned an unsupported content type", map[string]any{"mime_type": download.MIMEType})
	}
	return inspection, nil
}

func (s *Store) createSourceURL(ctx context.Context, input domain.SourceURLInput, sourceURL string, requestedType string, lookupURLs []string) (domain.SourceIngestionResult, error) {
	sourcePath, err := normalizeSourceDocumentPath(input.PathHint)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if exists, err := s.sourceURLExists(ctx, lookupURLs); err != nil {
		return domain.SourceIngestionResult{}, err
	} else if exists {
		return domain.SourceIngestionResult{}, domain.AlreadyExistsError("source URL", sourceURL)
	}
	if _, err := s.lstatVaultPath(sourcePath, "stat source document path"); err == nil {
		return domain.SourceIngestionResult{}, domain.AlreadyExistsError("document path", sourcePath)
	} else if !errors.Is(err, fs.ErrNotExist) {
		return domain.SourceIngestionResult{}, domain.InternalError("stat source document path", err)
	}

	download, err := downloadSource(ctx, sourceURL, requestedType)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	sourceURL = firstNonEmpty(download.FinalURL, sourceURL)
	if exists, err := s.sourceURLExists(ctx, []string{sourceURL}); err != nil {
		return domain.SourceIngestionResult{}, err
	} else if exists {
		return domain.SourceIngestionResult{}, domain.AlreadyExistsError("source URL", sourceURL)
	}
	switch download.SourceType {
	case sourceTypePDF:
		return s.createPDFSourceURL(ctx, input, sourceURL, sourcePath, download)
	case sourceTypeWeb:
		return s.createWebSourceURL(ctx, input, sourceURL, sourcePath, download)
	default:
		return domain.SourceIngestionResult{}, domain.ValidationError("source URL returned an unsupported content type", map[string]any{"mime_type": download.MIMEType})
	}
}

func (s *Store) createPDFSourceURL(ctx context.Context, input domain.SourceURLInput, sourceURL string, sourcePath string, download sourceDownload) (domain.SourceIngestionResult, error) {
	assetPath, err := normalizeSourceAssetPath(input.AssetPathHint)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	assetAbsPath := filepath.Join(s.vaultRoot, filepath.FromSlash(assetPath))
	if _, err := s.lstatVaultPath(assetPath, "stat source asset path"); err == nil {
		return domain.SourceIngestionResult{}, domain.AlreadyExistsError("asset path", assetPath)
	} else if !errors.Is(err, fs.ErrNotExist) {
		return domain.SourceIngestionResult{}, domain.InternalError("stat source asset path", err)
	}

	if err := s.writeNewVaultFile(assetPath, download.Body, "write source asset"); err != nil {
		if errors.Is(err, fs.ErrExist) {
			return domain.SourceIngestionResult{}, domain.AlreadyExistsError("asset path", assetPath)
		}
		return domain.SourceIngestionResult{}, err
	}
	assetWritten := true
	defer func() {
		if assetWritten {
			if _, err := s.lstatVaultPath(sourcePath, "stat source document path"); errors.Is(err, fs.ErrNotExist) {
				_ = s.removeVaultPath(assetPath, "remove source asset after failed source note create")
			}
		}
	}()

	extracted, err := extractPDF(assetAbsPath)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	capturedAt := s.now().UTC()
	sha := sha256.Sum256(download.Body)
	shaHex := hex.EncodeToString(sha[:])
	title := resolvedSourceTitle(input.Title, extracted.Metadata.Title, sourcePath)
	body := buildPDFSourceNoteBody(sourceURL, sourcePath, assetPath, title, shaHex, int64(len(download.Body)), download.MIMEType, capturedAt, extracted)
	document, err := s.CreateDocument(ctx, domain.CreateDocumentInput{
		Path:  sourcePath,
		Title: title,
		Body:  body,
	})
	if err != nil {
		if _, statErr := s.lstatVaultPath(sourcePath, "stat source document path"); statErr == nil {
			assetWritten = false
		}
		return domain.SourceIngestionResult{}, err
	}
	assetWritten = false
	citations, err := s.sourceDocumentCitations(ctx, document)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if len(citations) == 0 {
		return domain.SourceIngestionResult{}, domain.InternalError("validate source ingestion citations", errors.New("created source has no indexed citations"))
	}
	if err := validateIngestedSource(document, sourceURL, sourcePath, assetPath, assetAbsPath); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	return domain.SourceIngestionResult{
		DocID:       document.DocID,
		SourcePath:  sourcePath,
		SourceURL:   sourceURL,
		SourceType:  sourceTypePDF,
		AssetPath:   assetPath,
		DerivedPath: sourcePath,
		Citations:   citations,
		SHA256:      shaHex,
		SizeBytes:   int64(len(download.Body)),
		MIMEType:    download.MIMEType,
		PageCount:   extracted.Pages,
		CapturedAt:  capturedAt,
		PDFMetadata: extracted.Metadata,
	}, nil
}

func (s *Store) createWebSourceURL(ctx context.Context, input domain.SourceURLInput, sourceURL string, sourcePath string, download sourceDownload) (domain.SourceIngestionResult, error) {
	if strings.TrimSpace(input.AssetPathHint) != "" {
		return domain.SourceIngestionResult{}, domain.ValidationError("source.asset_path_hint is not supported for web source ingestion", nil)
	}
	extracted, err := extractWebSource(sourceURL, download.MIMEType, download.Body)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	capturedAt := s.now().UTC()
	sha := sha256.Sum256(download.Body)
	shaHex := hex.EncodeToString(sha[:])
	title := resolvedSourceTitle(input.Title, extracted.Title, sourcePath)
	body := buildWebSourceNoteBody(sourceURL, sourcePath, title, shaHex, int64(len(download.Body)), download.MIMEType, capturedAt, extracted)
	document, err := s.CreateDocument(ctx, domain.CreateDocumentInput{
		Path:  sourcePath,
		Title: title,
		Body:  body,
	})
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	citations, err := s.sourceDocumentCitations(ctx, document)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if len(citations) == 0 {
		return domain.SourceIngestionResult{}, domain.InternalError("validate web source ingestion citations", errors.New("created source has no indexed citations"))
	}
	if err := validateIngestedSource(document, sourceURL, sourcePath, "", ""); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	return domain.SourceIngestionResult{
		DocID:       document.DocID,
		SourcePath:  sourcePath,
		SourceURL:   sourceURL,
		SourceType:  sourceTypeWeb,
		DerivedPath: sourcePath,
		Citations:   citations,
		SHA256:      shaHex,
		SizeBytes:   int64(len(download.Body)),
		MIMEType:    download.MIMEType,
		CapturedAt:  capturedAt,
	}, nil
}

func (s *Store) updateSourceURL(ctx context.Context, input domain.SourceURLInput, sourceURL string, requestedType string, lookupURLs []string) (domain.SourceIngestionResult, error) {
	document, storedSourceURL, err := s.sourceDocumentByURL(ctx, lookupURLs)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if storedSourceURL == "" {
		storedSourceURL = sourceURL
	}
	storedType := strings.TrimSpace(document.Metadata["source_type"])
	if storedType == "" {
		storedType = sourceTypePDF
	}
	if requestedType != "" && requestedType != storedType {
		return domain.SourceIngestionResult{}, domain.ConflictError("source_type does not match existing source", map[string]any{"source_url": sourceURL, "source_type": requestedType, "existing_source_type": storedType})
	}
	switch storedType {
	case sourceTypePDF:
		return s.updatePDFSourceURL(ctx, input, sourceURL, storedSourceURL, document)
	case sourceTypeWeb:
		return s.updateWebSourceURL(ctx, input, sourceURL, storedSourceURL, document)
	default:
		return domain.SourceIngestionResult{}, domain.ConflictError("source URL belongs to an unsupported source type", map[string]any{"source_url": sourceURL, "source_type": storedType, "path": document.Path})
	}
}

func (s *Store) ensureSourceURLUpdateTarget(ctx context.Context, docID string, storedSourceURL string, finalSourceURL string) error {
	if strings.TrimSpace(finalSourceURL) == "" || finalSourceURL == storedSourceURL {
		return nil
	}
	existing, _, err := s.sourceDocumentByURL(ctx, []string{finalSourceURL})
	if err != nil {
		var appErr *domain.Error
		if errors.As(err, &appErr) && appErr.Status == 404 {
			return nil
		}
		return err
	}
	if existing.DocID == docID {
		return nil
	}
	return domain.AlreadyExistsError("source URL", finalSourceURL)
}

func (s *Store) updatePDFSourceURL(ctx context.Context, input domain.SourceURLInput, sourceURL string, storedSourceURL string, document domain.Document) (domain.SourceIngestionResult, error) {
	sourcePath, assetPath, err := validateStoredPDFSourceDocument(s.vaultRoot, document, storedSourceURL)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if strings.TrimSpace(input.PathHint) != "" {
		requestPath, err := normalizeSourceDocumentPath(input.PathHint)
		if err != nil {
			return domain.SourceIngestionResult{}, err
		}
		if requestPath != sourcePath {
			return domain.SourceIngestionResult{}, domain.ConflictError("source path hint does not match existing source", map[string]any{"source_url": sourceURL, "path_hint": requestPath, "existing_path": sourcePath})
		}
	}
	if strings.TrimSpace(input.AssetPathHint) != "" {
		requestAssetPath, err := normalizeSourceAssetPath(input.AssetPathHint)
		if err != nil {
			return domain.SourceIngestionResult{}, err
		}
		if requestAssetPath != assetPath {
			return domain.SourceIngestionResult{}, domain.ConflictError("source asset path hint does not match existing source", map[string]any{"source_url": sourceURL, "asset_path_hint": requestAssetPath, "existing_asset_path": assetPath})
		}
	}
	assetAbsPath := filepath.Join(s.vaultRoot, filepath.FromSlash(assetPath))
	if err := validateExistingVaultFile(s.vaultRoot, assetAbsPath, "validate existing source asset"); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if _, err := s.statVaultPath(assetPath, "validate existing source asset"); err != nil {
		return domain.SourceIngestionResult{}, err
	}

	download, err := downloadSource(ctx, sourceURL, sourceTypePDF)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	sourceURL = firstNonEmpty(download.FinalURL, sourceURL)
	if err := s.ensureSourceURLUpdateTarget(ctx, document.DocID, storedSourceURL, sourceURL); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	sha := sha256.Sum256(download.Body)
	shaHex := hex.EncodeToString(sha[:])
	if strings.TrimSpace(document.Metadata["sha256"]) == shaHex {
		result, err := s.sourceIngestionResultFromDocument(ctx, document)
		if err != nil {
			return domain.SourceIngestionResult{}, err
		}
		return s.withSourceURLUpdateImpact(ctx, result, sourceURL, shaHex, shaHex, false)
	}

	tempAssetPath := assetPath + ".openclerk-update-" + hashID("asset", sourceURL, shaHex)
	tempAssetAbsPath := filepath.Join(s.vaultRoot, filepath.FromSlash(tempAssetPath))
	if err := s.writeNewVaultFile(tempAssetPath, download.Body, "write source asset staging file"); err != nil {
		if errors.Is(err, fs.ErrExist) {
			return domain.SourceIngestionResult{}, domain.AlreadyExistsError("source asset staging path", tempAssetPath)
		}
		return domain.SourceIngestionResult{}, err
	}
	defer func() {
		_ = s.removeVaultPath(tempAssetPath, "remove source asset staging file")
	}()
	extracted, err := extractPDF(tempAssetAbsPath)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}

	sourceAbsPath := filepath.Join(s.vaultRoot, filepath.FromSlash(sourcePath))
	if err := validateExistingVaultFile(s.vaultRoot, sourceAbsPath, "validate existing source document"); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	oldBody := document.Body
	oldAssetBytes, err := s.readVaultFileLimited(assetPath, maxSourceDownloadBytes, "read existing source asset")
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	capturedAt := s.now().UTC()
	title := resolvedSourceTitle(input.Title, extracted.Metadata.Title, sourcePath)
	body := buildPDFSourceNoteBody(sourceURL, sourcePath, assetPath, title, shaHex, int64(len(download.Body)), download.MIMEType, capturedAt, extracted)
	if err := s.replaceSourceAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, download.Body, body, title); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	updated, err := s.GetDocument(ctx, document.DocID)
	if err != nil {
		return domain.SourceIngestionResult{}, s.restoreSourceAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, err)
	}
	citations, err := s.sourceDocumentCitations(ctx, updated)
	if err != nil {
		return domain.SourceIngestionResult{}, s.restoreSourceAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, err)
	}
	if len(citations) == 0 {
		return domain.SourceIngestionResult{}, s.restoreSourceAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, domain.InternalError("validate source update citations", errors.New("updated source has no indexed citations")))
	}
	if err := validateIngestedSource(updated, sourceURL, sourcePath, assetPath, assetAbsPath); err != nil {
		return domain.SourceIngestionResult{}, s.restoreSourceAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, err)
	}
	result := domain.SourceIngestionResult{
		DocID:       updated.DocID,
		SourcePath:  sourcePath,
		SourceURL:   sourceURL,
		SourceType:  sourceTypePDF,
		AssetPath:   assetPath,
		DerivedPath: sourcePath,
		Citations:   citations,
		SHA256:      shaHex,
		SizeBytes:   int64(len(download.Body)),
		MIMEType:    download.MIMEType,
		PageCount:   extracted.Pages,
		CapturedAt:  capturedAt,
		PDFMetadata: extracted.Metadata,
	}
	return s.withSourceURLUpdateImpact(ctx, result, sourceURL, strings.TrimSpace(document.Metadata["sha256"]), shaHex, true)
}

func (s *Store) updateWebSourceURL(ctx context.Context, input domain.SourceURLInput, sourceURL string, storedSourceURL string, document domain.Document) (domain.SourceIngestionResult, error) {
	sourcePath, err := validateStoredWebSourceDocument(s.vaultRoot, document, storedSourceURL)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	if strings.TrimSpace(input.PathHint) != "" {
		requestPath, err := normalizeSourceDocumentPath(input.PathHint)
		if err != nil {
			return domain.SourceIngestionResult{}, err
		}
		if requestPath != sourcePath {
			return domain.SourceIngestionResult{}, domain.ConflictError("source path hint does not match existing source", map[string]any{"source_url": sourceURL, "path_hint": requestPath, "existing_path": sourcePath})
		}
	}
	if strings.TrimSpace(input.AssetPathHint) != "" {
		return domain.SourceIngestionResult{}, domain.ConflictError("web source does not have an asset path", map[string]any{"source_url": sourceURL, "asset_path_hint": input.AssetPathHint, "existing_path": sourcePath})
	}
	download, err := downloadSource(ctx, sourceURL, sourceTypeWeb)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	sourceURL = firstNonEmpty(download.FinalURL, sourceURL)
	if err := s.ensureSourceURLUpdateTarget(ctx, document.DocID, storedSourceURL, sourceURL); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	sha := sha256.Sum256(download.Body)
	shaHex := hex.EncodeToString(sha[:])
	if strings.TrimSpace(document.Metadata["sha256"]) == shaHex {
		result, err := s.sourceIngestionResultFromDocument(ctx, document)
		if err != nil {
			return domain.SourceIngestionResult{}, err
		}
		return s.withSourceURLUpdateImpact(ctx, result, sourceURL, shaHex, shaHex, false)
	}
	extracted, err := extractWebSource(sourceURL, download.MIMEType, download.Body)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	sourceAbsPath := filepath.Join(s.vaultRoot, filepath.FromSlash(sourcePath))
	if err := validateExistingVaultFile(s.vaultRoot, sourceAbsPath, "validate existing web source document"); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	oldBody := document.Body
	capturedAt := s.now().UTC()
	title := resolvedSourceTitle(input.Title, extracted.Title, sourcePath)
	body := buildWebSourceNoteBody(sourceURL, sourcePath, title, shaHex, int64(len(download.Body)), download.MIMEType, capturedAt, extracted)
	if err := s.replaceSourceNote(ctx, sourcePath, oldBody, body, title); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	updated, err := s.GetDocument(ctx, document.DocID)
	if err != nil {
		return domain.SourceIngestionResult{}, s.restoreSourceNote(ctx, sourcePath, oldBody, err)
	}
	citations, err := s.sourceDocumentCitations(ctx, updated)
	if err != nil {
		return domain.SourceIngestionResult{}, s.restoreSourceNote(ctx, sourcePath, oldBody, err)
	}
	if len(citations) == 0 {
		return domain.SourceIngestionResult{}, s.restoreSourceNote(ctx, sourcePath, oldBody, domain.InternalError("validate web source update citations", errors.New("updated source has no indexed citations")))
	}
	if err := validateIngestedSource(updated, sourceURL, sourcePath, "", ""); err != nil {
		return domain.SourceIngestionResult{}, s.restoreSourceNote(ctx, sourcePath, oldBody, err)
	}
	result := domain.SourceIngestionResult{
		DocID:       updated.DocID,
		SourcePath:  sourcePath,
		SourceURL:   sourceURL,
		SourceType:  sourceTypeWeb,
		DerivedPath: sourcePath,
		Citations:   citations,
		SHA256:      shaHex,
		SizeBytes:   int64(len(download.Body)),
		MIMEType:    download.MIMEType,
		CapturedAt:  capturedAt,
	}
	return s.withSourceURLUpdateImpact(ctx, result, sourceURL, strings.TrimSpace(document.Metadata["sha256"]), shaHex, true)
}

func (s *Store) withSourceURLUpdateImpact(ctx context.Context, result domain.SourceIngestionResult, sourceURL string, previousSHA string, newSHA string, changed bool) (domain.SourceIngestionResult, error) {
	result.UpdateStatus = "no_op"
	if changed {
		result.UpdateStatus = "changed"
	}
	result.NormalizedSourceURL = sourceURL
	result.SourceDocID = result.DocID
	result.PreviousSHA256 = previousSHA
	result.NewSHA256 = newSHA
	result.Changed = changed
	result.DuplicateStatus = "existing_source_matched_no_duplicate_created"
	result.SynthesisRepaired = false

	if !changed {
		result.StaleDependents = []domain.SourceStaleDependent{}
		result.ProjectionRefs = []domain.SourceProjectionRef{}
		result.ProvenanceRefs = []domain.SourceProvenanceRef{}
		result.NoRepairWarning = "Source refresh detected no content change; no synthesis repair was performed."
		return result, nil
	}

	cursor := ""
	for {
		projections, err := s.ListProjectionStates(ctx, domain.ProjectionStateQuery{Projection: "synthesis", Limit: 100, Cursor: cursor})
		if err != nil {
			return domain.SourceIngestionResult{}, err
		}
		for _, projection := range projections.Projections {
			staleRefs := splitPathList(projection.Details["stale_source_refs"])
			if !strings.EqualFold(projection.Freshness, "stale") || !sourcePathListContains(staleRefs, result.SourcePath) {
				continue
			}
			if len(result.StaleDependents) >= maxSourceURLUpdateImpactRefs {
				continue
			}
			dependentPath := strings.TrimSpace(projection.Details["synthesis_path"])
			result.StaleDependents = append(result.StaleDependents, domain.SourceStaleDependent{
				Path:            dependentPath,
				DocID:           projection.RefID,
				Projection:      projection.Projection,
				Freshness:       projection.Freshness,
				StaleSourceRefs: staleRefs,
			})
			result.ProjectionRefs = append(result.ProjectionRefs, domain.SourceProjectionRef{
				Projection: projection.Projection,
				RefKind:    projection.RefKind,
				RefID:      projection.RefID,
				Freshness:  projection.Freshness,
				SourceRef:  projection.SourceRef,
			})
		}
		if !projections.PageInfo.HasMore {
			break
		}
		cursor = projections.PageInfo.NextCursor
	}
	sort.Slice(result.StaleDependents, func(i, j int) bool {
		return result.StaleDependents[i].Path < result.StaleDependents[j].Path
	})
	sort.Slice(result.ProjectionRefs, func(i, j int) bool {
		if result.ProjectionRefs[i].Projection != result.ProjectionRefs[j].Projection {
			return result.ProjectionRefs[i].Projection < result.ProjectionRefs[j].Projection
		}
		return result.ProjectionRefs[i].RefID < result.ProjectionRefs[j].RefID
	})

	provenanceRefs, err := s.sourceURLUpdateProvenanceRefs(ctx, result.DocID, previousSHA, newSHA, result.ProjectionRefs)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	result.ProvenanceRefs = provenanceRefs
	if len(result.StaleDependents) == 0 {
		result.NoRepairWarning = "Source refresh did not repair dependent synthesis automatically."
		return result, nil
	}
	paths := make([]string, 0, len(result.StaleDependents))
	for _, dependent := range result.StaleDependents {
		if dependent.Path != "" {
			paths = append(paths, dependent.Path)
		}
	}
	result.NoRepairWarning = "Source refresh did not repair dependent synthesis: " + strings.Join(paths, ", ")
	if len(result.StaleDependents) >= maxSourceURLUpdateImpactRefs {
		result.NoRepairWarning += fmt.Sprintf("; response capped at %d stale dependents", maxSourceURLUpdateImpactRefs)
	}
	return result, nil
}

func (s *Store) sourceURLUpdateProvenanceRefs(ctx context.Context, sourceDocID string, previousSHA string, newSHA string, projections []domain.SourceProjectionRef) ([]domain.SourceProvenanceRef, error) {
	refs := []domain.SourceProvenanceRef{}
	sourceEvents, err := s.ListProvenanceEvents(ctx, domain.ProvenanceEventQuery{RefKind: "source", RefID: sourceDocID, Limit: 20})
	if err != nil {
		return nil, err
	}
	for _, event := range sourceEvents.Events {
		if event.EventType != "source_updated" ||
			event.Details["previous_sha256"] != previousSHA ||
			event.Details["new_sha256"] != newSHA {
			continue
		}
		refs = append(refs, sourceProvenanceRef(event))
		break
	}

	if len(projections) == 0 {
		return refs, nil
	}
	for _, projection := range projections {
		projectionEvents, err := s.ListProvenanceEvents(ctx, domain.ProvenanceEventQuery{RefKind: "projection", RefID: projection.Projection + ":" + projection.RefID, Limit: 20})
		if err != nil {
			return nil, err
		}
		for _, event := range projectionEvents.Events {
			if event.EventType != "projection_invalidated" {
				continue
			}
			refs = append(refs, sourceProvenanceRef(event))
			break
		}
	}
	sort.Slice(refs, func(i, j int) bool {
		if refs[i].EventType != refs[j].EventType {
			return refs[i].EventType < refs[j].EventType
		}
		return refs[i].RefID < refs[j].RefID
	})
	return refs, nil
}

func sourcePathListContains(values []string, target string) bool {
	for _, value := range values {
		if value == target {
			return true
		}
	}
	return false
}

func sourceProvenanceRef(event domain.ProvenanceEvent) domain.SourceProvenanceRef {
	return domain.SourceProvenanceRef{
		EventID:   event.EventID,
		EventType: event.EventType,
		RefKind:   event.RefKind,
		RefID:     event.RefID,
		SourceRef: event.SourceRef,
	}
}

func (s *Store) sourceDocumentByURL(ctx context.Context, sourceURLs []string) (domain.Document, string, error) {
	for _, sourceURL := range uniqueNonEmptyStrings(sourceURLs) {
		var docID string
		err := s.db.QueryRowContext(ctx, `
SELECT doc_id
FROM document_metadata
WHERE key_name = 'source_url' AND value_text = ?
ORDER BY doc_id
LIMIT 1`, sourceURL).Scan(&docID)
		if errors.Is(err, sql.ErrNoRows) {
			continue
		}
		if err != nil {
			return domain.Document{}, "", domain.InternalError("query source URL document", err)
		}
		document, err := s.GetDocument(ctx, docID)
		return document, sourceURL, err
	}
	return domain.Document{}, "", domain.NotFoundError("source URL", firstNonEmpty(sourceURLs...))
}

func (s *Store) sourceURLExists(ctx context.Context, sourceURLs []string) (bool, error) {
	for _, sourceURL := range uniqueNonEmptyStrings(sourceURLs) {
		var found string
		err := s.db.QueryRowContext(ctx, `
SELECT doc_id
FROM document_metadata
WHERE key_name = 'source_url' AND value_text = ?
LIMIT 1`, sourceURL).Scan(&found)
		if errors.Is(err, sql.ErrNoRows) {
			continue
		}
		if err != nil {
			return false, domain.InternalError("query duplicate source URL", err)
		}
		return true, nil
	}
	return false, nil
}

func (s *Store) sourceIngestionResultFromDocument(ctx context.Context, document domain.Document) (domain.SourceIngestionResult, error) {
	sourceURL := strings.TrimSpace(document.Metadata["source_url"])
	sourceType := strings.TrimSpace(document.Metadata["source_type"])
	if sourceType == "" {
		sourceType = sourceTypePDF
	}
	assetPath := strings.TrimSpace(document.Metadata["asset_path"])
	var err error
	switch sourceType {
	case sourceTypePDF:
		_, assetPath, err = validateStoredPDFSourceDocument(s.vaultRoot, document, sourceURL)
	case sourceTypeWeb:
		_, err = validateStoredWebSourceDocument(s.vaultRoot, document, sourceURL)
	default:
		err = domain.ConflictError("source URL belongs to an unsupported source type", map[string]any{"source_url": sourceURL, "source_type": sourceType, "path": document.Path})
	}
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	derivedPath := strings.TrimSpace(document.Metadata["derived_path"])
	if derivedPath == "" {
		derivedPath = document.Path
	}
	if sourceURL == "" || (sourceType == sourceTypePDF && assetPath == "") {
		return domain.SourceIngestionResult{}, domain.InternalError("source document is missing ingestion metadata", fmt.Errorf("path %q", document.Path))
	}
	assetAbsPath := ""
	if assetPath != "" {
		assetAbsPath = filepath.Join(s.vaultRoot, filepath.FromSlash(assetPath))
	}
	if err := validateIngestedSource(document, sourceURL, document.Path, assetPath, assetAbsPath); err != nil {
		return domain.SourceIngestionResult{}, err
	}
	citations, err := s.sourceDocumentCitations(ctx, document)
	if err != nil {
		return domain.SourceIngestionResult{}, err
	}
	return domain.SourceIngestionResult{
		DocID:       document.DocID,
		SourcePath:  document.Path,
		SourceURL:   sourceURL,
		SourceType:  sourceType,
		AssetPath:   assetPath,
		DerivedPath: derivedPath,
		Citations:   citations,
		SHA256:      strings.TrimSpace(document.Metadata["sha256"]),
		SizeBytes:   parseInt64Metadata(document.Metadata["size_bytes"]),
		MIMEType:    strings.TrimSpace(document.Metadata["mime_type"]),
		PageCount:   parseIntMetadata(document.Metadata["page_count"]),
		CapturedAt:  parseTimeMetadata(document.Metadata["captured_at"]),
		PDFMetadata: domain.SourcePDFMetadata{
			Title:         strings.TrimSpace(document.Metadata["pdf_title"]),
			Author:        strings.TrimSpace(document.Metadata["pdf_author"]),
			PublishedDate: strings.TrimSpace(document.Metadata["pdf_published_date"]),
		},
	}, nil
}

func (s *Store) replaceSourceAssetAndNote(ctx context.Context, sourcePath string, assetPath string, oldBody string, oldAssetBytes []byte, newAssetBytes []byte, newBody string, title string) error {
	return s.replaceIngestedAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, newAssetBytes, newBody, title, noteMutationLabels{
		WriteAsset:     "write source asset",
		WriteNote:      "write source note",
		RestoreAsset:   "restore source asset after failed update",
		RestoreNote:    "restore source note after failed update",
		RestoreIndexed: "restore indexed source after failed update",
	})
}

func (s *Store) restoreSourceAssetAndNote(ctx context.Context, sourcePath string, assetPath string, oldBody string, oldAssetBytes []byte, cause error) error {
	return s.restoreIngestedAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, cause, noteMutationLabels{
		RestoreAsset:   "restore source asset after failed update",
		RestoreNote:    "restore source note after failed update",
		RestoreIndexed: "restore indexed source after failed update",
	})
}

func (s *Store) replaceSourceNote(ctx context.Context, sourcePath string, oldBody string, newBody string, title string) error {
	return s.replaceIngestedNote(ctx, sourcePath, oldBody, newBody, title, noteMutationLabels{
		WriteNote:      "write source note",
		RestoreNote:    "restore source note after failed update",
		RestoreIndexed: "restore indexed source after failed update",
	})
}

func (s *Store) restoreSourceNote(ctx context.Context, sourcePath string, oldBody string, cause error) error {
	return s.restoreIngestedNote(ctx, sourcePath, oldBody, cause, noteMutationLabels{
		RestoreNote:    "restore source note after failed update",
		RestoreIndexed: "restore indexed source after failed update",
	})
}

func (s *Store) replaceIngestedAssetAndNote(ctx context.Context, sourcePath string, assetPath string, oldBody string, oldAssetBytes []byte, newAssetBytes []byte, newBody string, title string, labels noteMutationLabels) error {
	if err := s.writeExistingVaultFile(assetPath, newAssetBytes, labels.WriteAsset); err != nil {
		return err
	}
	if err := s.writeExistingVaultFile(sourcePath, []byte(newBody), labels.WriteNote); err != nil {
		return s.restoreIngestedAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, err, labels)
	}
	if err := s.syncDocumentFromDisk(ctx, sourcePath, title); err != nil {
		return s.restoreIngestedAssetAndNote(ctx, sourcePath, assetPath, oldBody, oldAssetBytes, err, labels)
	}
	return nil
}

func (s *Store) restoreIngestedAssetAndNote(ctx context.Context, sourcePath string, assetPath string, oldBody string, oldAssetBytes []byte, cause error, labels noteMutationLabels) error {
	if restoreErr := s.writeExistingVaultFile(assetPath, oldAssetBytes, labels.RestoreAsset); restoreErr != nil {
		return domain.InternalError(labels.RestoreAsset, errors.Join(cause, restoreErr))
	}
	if restoreErr := s.writeExistingVaultFile(sourcePath, []byte(oldBody), labels.RestoreNote); restoreErr != nil {
		return domain.InternalError(labels.RestoreNote, errors.Join(cause, restoreErr))
	}
	if restoreErr := s.syncDocumentFromDisk(ctx, sourcePath, ""); restoreErr != nil {
		return domain.InternalError(labels.RestoreIndexed, errors.Join(cause, restoreErr))
	}
	return cause
}

func (s *Store) replaceIngestedNote(ctx context.Context, sourcePath string, oldBody string, newBody string, title string, labels noteMutationLabels) error {
	if err := s.writeExistingVaultFile(sourcePath, []byte(newBody), labels.WriteNote); err != nil {
		return err
	}
	if err := s.syncDocumentFromDisk(ctx, sourcePath, title); err != nil {
		return s.restoreIngestedNote(ctx, sourcePath, oldBody, err, labels)
	}
	return nil
}

func (s *Store) restoreIngestedNote(ctx context.Context, sourcePath string, oldBody string, cause error, labels noteMutationLabels) error {
	if restoreErr := s.writeExistingVaultFile(sourcePath, []byte(oldBody), labels.RestoreNote); restoreErr != nil {
		return domain.InternalError(labels.RestoreNote, errors.Join(cause, restoreErr))
	}
	if restoreErr := s.syncDocumentFromDisk(ctx, sourcePath, ""); restoreErr != nil {
		return domain.InternalError(labels.RestoreIndexed, errors.Join(cause, restoreErr))
	}
	return cause
}

func downloadSource(ctx context.Context, sourceURL string, requestedType string) (sourceDownload, error) {
	if fixturePath, ok, err := resolveEvalSourceFixturePath(sourceURL); err != nil {
		return sourceDownload{}, err
	} else if ok {
		body, err := osReadFile(fixturePath)
		if err != nil {
			return sourceDownload{}, domain.InternalError("read eval source fixture", err)
		}
		if len(body) > maxSourceDownloadBytes {
			return sourceDownload{}, domain.ValidationError("source URL exceeds maximum supported size", map[string]any{"max_bytes": maxSourceDownloadBytes})
		}
		mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(fixturePath)))
		if mimeType == "" {
			mimeType = "application/octet-stream"
		}
		kind, mimeType, err := classifySourceBody(sourceURL, body, mimeType, requestedType)
		if err != nil {
			return sourceDownload{}, err
		}
		return sourceDownload{Body: body, MIMEType: mimeType, SourceType: kind, FinalURL: sourceURL}, nil
	}
	parsedURL, err := validateSourceFetchURL(sourceURL)
	if err != nil {
		return sourceDownload{}, err
	}
	if err := validatePublicSourceHost(ctx, parsedURL); err != nil {
		return sourceDownload{}, err
	}
	request, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
	if err != nil {
		return sourceDownload{}, domain.ValidationError("source url must be fetchable", map[string]any{"url": sourceURL})
	}
	var redirectValidationErr error
	client := sourceHTTPClient(func(request *http.Request, _ []*http.Request) error {
		if request == nil || request.URL == nil {
			redirectValidationErr = domain.ValidationError("source url must be fetchable", map[string]any{"url": sourceURL})
			return redirectValidationErr
		}
		redirectValidationErr = validateSourceFetchURLTarget(ctx, request.URL)
		return redirectValidationErr
	})
	if client == nil {
		client = defaultSourceHTTPClient(nil)
	}
	response, err := client.Do(request)
	if err != nil {
		if redirectValidationErr != nil {
			return sourceDownload{}, redirectValidationErr
		}
		return sourceDownload{}, domain.InternalError("download source URL", err)
	}
	defer func() {
		_ = response.Body.Close()
	}()
	if response.StatusCode < 200 || response.StatusCode >= 300 {
		if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusProxyAuthRequired {
			return sourceDownload{}, domain.ValidationError("source URL requires unsupported authenticated or restricted access", map[string]any{"status": response.StatusCode})
		}
		return sourceDownload{}, domain.ValidationError("source URL returned a non-success status", map[string]any{"status": response.StatusCode})
	}
	body, err := io.ReadAll(io.LimitReader(response.Body, maxSourceDownloadBytes+1))
	if err != nil {
		return sourceDownload{}, domain.InternalError("read source URL response", err)
	}
	if len(body) > maxSourceDownloadBytes {
		return sourceDownload{}, domain.ValidationError("source URL exceeds maximum supported size", map[string]any{"max_bytes": maxSourceDownloadBytes})
	}
	mimeType := ""
	if parsed, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")); err == nil {
		mimeType = parsed
	} else {
		mimeType = http.DetectContentType(body)
	}
	finalURL := response.Request.URL
	if finalURL == nil {
		finalURL = parsedURL
	}
	if err := validateSourceFetchURLTarget(ctx, finalURL); err != nil {
		return sourceDownload{}, err
	}
	finalSourceURL := finalURL.String()
	kind, mimeType, err := classifySourceBody(finalSourceURL, body, mimeType, requestedType)
	if err != nil {
		return sourceDownload{}, err
	}
	return sourceDownload{Body: body, MIMEType: mimeType, SourceType: kind, FinalURL: finalSourceURL}, nil
}

func validateSourceFetchURL(sourceURL string) (*url.URL, error) {
	parsed, err := url.Parse(sourceURL)
	if err != nil || parsed.Hostname() == "" {
		return nil, domain.ValidationError("source url must be fetchable", map[string]any{"url": sourceURL})
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return nil, domain.ValidationError("source URL must use http or https", map[string]any{"scheme": parsed.Scheme})
	}
	if parsed.User != nil {
		return nil, domain.ValidationError("source URL must not include userinfo", nil)
	}
	return parsed, nil
}

func validatePublicSourceHost(ctx context.Context, parsed *url.URL) error {
	if allowPrivateSourceHostsForTests {
		return nil
	}
	host := strings.TrimSpace(parsed.Hostname())
	if host == "" {
		return domain.ValidationError("source url must be fetchable", map[string]any{"url": parsed.String()})
	}
	lowerHost := strings.ToLower(host)
	if lowerHost == "localhost" || strings.HasSuffix(lowerHost, ".localhost") || strings.HasSuffix(lowerHost, ".local") {
		return domain.ValidationError("source URL must be publicly fetchable", map[string]any{"host": host})
	}
	if ip := net.ParseIP(host); ip != nil {
		if !isPublicSourceIP(ip) {
			return domain.ValidationError("source URL must be publicly fetchable", map[string]any{"host": host})
		}
		return nil
	}
	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
	if err != nil {
		return domain.ValidationError("source URL host must resolve publicly", map[string]any{"host": host})
	}
	for _, addr := range addrs {
		if !isPublicSourceIP(addr.IP) {
			return domain.ValidationError("source URL must be publicly fetchable", map[string]any{"host": host})
		}
	}
	return nil
}

func validateSourceFetchURLTarget(ctx context.Context, parsed *url.URL) error {
	if parsed == nil || parsed.Hostname() == "" {
		return domain.ValidationError("source url must be fetchable", map[string]any{"url": ""})
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return domain.ValidationError("source URL must use http or https", map[string]any{"scheme": parsed.Scheme})
	}
	if parsed.User != nil {
		return domain.ValidationError("source URL must not include userinfo", nil)
	}
	return validatePublicSourceHost(ctx, parsed)
}

func dialPublicSourceContext(ctx context.Context, network string, address string) (net.Conn, error) {
	if allowPrivateSourceHostsForTests {
		dialer := &net.Dialer{Timeout: 30 * time.Second}
		return dialer.DialContext(ctx, network, address)
	}
	host, port, err := net.SplitHostPort(address)
	if err != nil {
		return nil, err
	}
	parsed := &url.URL{Scheme: "http", Host: net.JoinHostPort(host, port)}
	if err := validatePublicSourceHost(ctx, parsed); err != nil {
		return nil, err
	}
	ips := []net.IP{}
	if ip := net.ParseIP(host); ip != nil {
		ips = append(ips, ip)
	} else {
		addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
		if err != nil {
			return nil, domain.ValidationError("source URL host must resolve publicly", map[string]any{"host": host})
		}
		for _, addr := range addrs {
			if !isPublicSourceIP(addr.IP) {
				return nil, domain.ValidationError("source URL must be publicly fetchable", map[string]any{"host": host})
			}
			ips = append(ips, addr.IP)
		}
	}
	if len(ips) == 0 {
		return nil, domain.ValidationError("source URL host must resolve publicly", map[string]any{"host": host})
	}
	dialer := &net.Dialer{Timeout: 30 * time.Second}
	var lastErr error
	for _, ip := range ips {
		if !isPublicSourceIP(ip) {
			return nil, domain.ValidationError("source URL must be publicly fetchable", map[string]any{"host": host})
		}
		conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
		if err == nil {
			return conn, nil
		}
		lastErr = err
	}
	return nil, lastErr
}

func isPublicSourceIP(ip net.IP) bool {
	addr, ok := netip.AddrFromSlice(ip)
	if !ok {
		return false
	}
	addr = addr.Unmap()
	if !addr.IsGlobalUnicast() ||
		addr.IsUnspecified() ||
		addr.IsLoopback() ||
		addr.IsPrivate() ||
		addr.IsLinkLocalUnicast() ||
		addr.IsMulticast() {
		return false
	}
	for _, prefix := range blockedPublicSourceIPPrefixes {
		if prefix.Contains(addr) {
			return false
		}
	}
	return true
}

func validateStoredPDFSourceDocument(vaultRoot string, document domain.Document, sourceURL string) (string, string, error) {
	sourcePath, err := validateStoredSourceDocument(vaultRoot, document, sourceURL, sourceTypePDF)
	if err != nil {
		return "", "", err
	}
	assetPath, err := normalizeSourceAssetPath(document.Metadata["asset_path"])
	if err != nil {
		return "", "", domain.ConflictError("source document has invalid asset_path metadata", map[string]any{"source_url": sourceURL, "path": document.Path})
	}
	assetAbsPath := filepath.Join(vaultRoot, filepath.FromSlash(assetPath))
	if err := validateExistingVaultFile(vaultRoot, assetAbsPath, "validate existing source asset"); err != nil {
		return "", "", err
	}
	return sourcePath, assetPath, nil
}

func validateStoredWebSourceDocument(vaultRoot string, document domain.Document, sourceURL string) (string, error) {
	sourcePath, err := validateStoredSourceDocument(vaultRoot, document, sourceURL, sourceTypeWeb)
	if err != nil {
		return "", err
	}
	if strings.TrimSpace(document.Metadata["asset_path"]) != "" {
		return "", domain.ConflictError("web source document has unexpected asset_path metadata", map[string]any{"source_url": sourceURL, "path": document.Path})
	}
	return sourcePath, nil
}

func validateStoredSourceDocument(vaultRoot string, document domain.Document, sourceURL string, sourceType string) (string, error) {
	sourcePath, err := normalizeSourceDocumentPath(document.Path)
	if err != nil {
		return "", domain.ConflictError("source document has invalid path", map[string]any{"source_url": sourceURL, "path": document.Path})
	}
	required := map[string]string{
		"type":        "source",
		"source_type": sourceType,
		"modality":    "markdown",
		"source_url":  sourceURL,
	}
	for key, want := range required {
		if got := strings.TrimSpace(document.Metadata[key]); got != want {
			return "", domain.ConflictError("source document metadata does not match ingestion-owned source", map[string]any{"source_url": sourceURL, "path": document.Path, "key": key})
		}
	}
	if derived := strings.TrimSpace(document.Metadata["derived_path"]); derived != "" && derived != sourcePath {
		return "", domain.ConflictError("source document has invalid derived_path metadata", map[string]any{"source_url": sourceURL, "path": document.Path})
	}
	sourceAbsPath := filepath.Join(vaultRoot, filepath.FromSlash(sourcePath))
	if err := validateExistingVaultFile(vaultRoot, sourceAbsPath, "validate existing source document"); err != nil {
		return "", err
	}
	return sourcePath, nil
}

func validateExistingVaultFile(vaultRoot string, absPath string, label string) error {
	rootClean, err := filepath.Abs(vaultRoot)
	if err != nil {
		return domain.InternalError("resolve vault root", err)
	}
	candidateClean, err := filepath.Abs(absPath)
	if err != nil {
		return domain.InternalError(label, err)
	}
	rel, err := filepath.Rel(rootClean, candidateClean)
	if err != nil {
		return domain.InternalError(label, err)
	}
	if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
		return domain.ValidationError("source path must stay inside the vault root", nil)
	}
	root, err := os.OpenRoot(rootClean)
	if err != nil {
		return domain.InternalError(label, err)
	}
	defer func() {
		_ = root.Close()
	}()
	if _, err := root.Stat(vaultRootName(filepath.ToSlash(rel))); err != nil {
		if strings.Contains(strings.ToLower(err.Error()), "escapes") {
			return domain.ValidationError("source path must stay inside the vault root", nil)
		}
		return domain.InternalError(label, err)
	}
	return nil
}

func resolveEvalSourceFixturePath(sourceURL string) (string, bool, error) {
	if strings.TrimSpace(os.Getenv(evalSourceFixtureEnableEnv)) != "1" {
		return "", false, nil
	}
	root := strings.TrimSpace(os.Getenv(evalSourceFixtureRootEnv))
	if root == "" {
		return "", false, nil
	}
	parsed, err := url.Parse(sourceURL)
	if err != nil || parsed.Hostname() != evalSourceFixtureHost {
		return "", false, nil
	}
	clean := path.Clean("/" + strings.TrimPrefix(parsed.EscapedPath(), "/"))
	if clean == "/" || strings.Contains(clean, "%2f") || strings.Contains(clean, "%2F") {
		return "", false, domain.ValidationError("eval source fixture path is invalid", map[string]any{"path": parsed.Path})
	}
	rel := strings.TrimPrefix(clean, "/")
	target := filepath.Join(root, filepath.FromSlash(rel))
	rootClean, err := filepath.Abs(root)
	if err != nil {
		return "", false, domain.InternalError("resolve eval source fixture root", err)
	}
	rootReal, err := filepath.EvalSymlinks(rootClean)
	if err != nil {
		return "", false, domain.InternalError("resolve eval source fixture root", err)
	}
	targetClean, err := filepath.Abs(target)
	if err != nil {
		return "", false, domain.InternalError("resolve eval source fixture path", err)
	}
	targetReal, err := filepath.EvalSymlinks(targetClean)
	if err != nil {
		return "", false, domain.InternalError("resolve eval source fixture path", err)
	}
	if targetReal != rootReal && !strings.HasPrefix(targetReal, rootReal+string(os.PathSeparator)) {
		return "", false, domain.ValidationError("eval source fixture path escapes root", map[string]any{"path": parsed.Path})
	}
	return targetReal, true, nil
}

func looksLikePDF(body []byte) bool {
	prefixLen := min(len(body), 1024)
	return bytes.Contains(body[:prefixLen], []byte("%PDF-"))
}

func looksLikeHTML(body []byte) bool {
	prefixLen := min(len(body), 4096)
	prefix := strings.ToLower(string(body[:prefixLen]))
	return strings.Contains(prefix, "<!doctype html") || strings.Contains(prefix, "<html") || strings.Contains(prefix, "<head") || strings.Contains(prefix, "<body")
}

func classifySourceBody(sourceURL string, body []byte, mimeType string, requestedType string) (string, string, error) {
	baseMIME := sourceBaseMIMEType(mimeType)
	if looksLikePDF(body) {
		if requestedType == sourceTypeWeb {
			return "", "", domain.ValidationError("source URL returned a PDF, not a web page", nil)
		}
		if baseMIME == "" || baseMIME == "application/octet-stream" || baseMIME == "text/plain" {
			mimeType = "application/pdf"
		}
		return sourceTypePDF, mimeType, nil
	}
	if baseMIME == "text/html" || looksLikeHTML(body) {
		if requestedType == sourceTypePDF {
			return "", "", domain.ValidationError("source URL did not return a PDF", nil)
		}
		return sourceTypeWeb, "text/html", nil
	}
	if isMarkdownSource(sourceURL, mimeType) {
		if requestedType == sourceTypePDF {
			return "", "", domain.ValidationError("source URL did not return a PDF", nil)
		}
		return sourceTypeWeb, "text/markdown", nil
	}
	if requestedType == sourceTypePDF {
		return "", "", domain.ValidationError("source URL did not return a PDF", nil)
	}
	if requestedType == sourceTypeWeb {
		return "", "", domain.ValidationError("source URL did not return an HTML or Markdown web source", map[string]any{"mime_type": mimeType})
	}
	return "", "", domain.ValidationError("source URL returned an unsupported content type", map[string]any{"mime_type": mimeType})
}

func isMarkdownSource(sourceURL string, mimeType string) bool {
	normalizedMIME := sourceBaseMIMEType(mimeType)
	if normalizedMIME == "text/markdown" || normalizedMIME == "text/x-markdown" {
		return true
	}
	if normalizedMIME != "" && normalizedMIME != "text/plain" && normalizedMIME != "application/octet-stream" {
		return false
	}
	parsed, err := url.Parse(sourceURL)
	if err != nil {
		return false
	}
	return isMarkdownPathExtension(strings.ToLower(path.Ext(parsed.Path)))
}

func sourceBaseMIMEType(mimeType string) string {
	normalized := strings.ToLower(strings.TrimSpace(mimeType))
	if parsed, _, err := mime.ParseMediaType(normalized); err == nil {
		return strings.ToLower(strings.TrimSpace(parsed))
	}
	return normalized
}

func isMarkdownPathExtension(ext string) bool {
	switch ext {
	case ".md", ".markdown", ".mdown", ".mkd":
		return true
	default:
		return false
	}
}

func extractPDF(assetPath string) (pdfExtraction, error) {
	started := time.Now()
	reader, err := pdf.Open(assetPath)
	if err != nil {
		return pdfExtraction{}, domain.ValidationError("source asset is not a readable PDF", nil)
	}
	pages := reader.NumPage()
	if pages < 1 {
		return pdfExtraction{}, domain.ValidationError("source PDF must contain at least one page", nil)
	}
	if pages > maxSourcePDFPages {
		return pdfExtraction{}, domain.ValidationError("source PDF exceeds maximum supported page count", map[string]any{"max_pages": maxSourcePDFPages})
	}
	info := reader.Trailer().Key("Info")
	metadata := domain.SourcePDFMetadata{
		Title:         pdfText(info.Key("Title")),
		Author:        pdfText(info.Key("Author")),
		PublishedDate: firstNonEmpty(pdfText(info.Key("CreationDate")), pdfText(info.Key("ModDate"))),
	}
	pageTexts := make([]string, 0, pages)
	totalTextObjects := 0
	totalRunes := 0
	for pageNum := 1; pageNum <= pages; pageNum++ {
		if time.Since(started) > maxSourcePDFExtractionTime {
			return pdfExtraction{}, domain.ValidationError("source PDF extraction exceeded time limit", map[string]any{"max_seconds": int(maxSourcePDFExtractionTime / time.Second)})
		}
		pageContent := reader.Page(pageNum).Content()
		totalTextObjects += len(pageContent.Text)
		if totalTextObjects > maxSourcePDFTextObjects {
			return pdfExtraction{}, domain.ValidationError("source PDF exceeds maximum supported text object count", map[string]any{"max_text_objects": maxSourcePDFTextObjects})
		}
		texts := append([]pdf.Text(nil), pageContent.Text...)
		sort.SliceStable(texts, func(i, j int) bool {
			if texts[i].Y == texts[j].Y {
				return texts[i].X < texts[j].X
			}
			return texts[i].Y > texts[j].Y
		})
		var pageText strings.Builder
		var previous *pdf.Text
		for idx := range texts {
			text := texts[idx]
			if text.S != "" {
				totalRunes += utf8.RuneCountInString(text.S)
				if totalRunes > maxSourcePDFExtractedRunes {
					return pdfExtraction{}, domain.ValidationError("source PDF exceeds maximum supported extracted text size", map[string]any{"max_runes": maxSourcePDFExtractedRunes})
				}
				if previous != nil {
					if text.Y < previous.Y-previous.FontSize*0.5 {
						pageText.WriteString("\n")
					} else if gap := text.X - (previous.X + previous.W); gap > max(previous.FontSize*0.2, 1) {
						pageText.WriteString(" ")
					}
				}
				pageText.WriteString(text.S)
				previous = &texts[idx]
			}
		}
		if value := strings.TrimSpace(pageText.String()); value != "" {
			pageTexts = append(pageTexts, fmt.Sprintf("Page %d\n%s", pageNum, value))
		}
	}
	return pdfExtraction{
		Metadata: metadata,
		Text:     strings.TrimSpace(strings.Join(pageTexts, "\n\n")),
		Pages:    pages,
	}, nil
}

func extractPDFBytes(body []byte) (pdfExtraction, error) {
	file, err := os.CreateTemp("", "openclerk-source-inspect-*.pdf")
	if err != nil {
		return pdfExtraction{}, domain.InternalError("create source inspection PDF staging file", err)
	}
	tempPath := file.Name()
	defer func() {
		_ = osRemove(tempPath)
	}()
	if _, err := file.Write(body); err != nil {
		_ = file.Close()
		return pdfExtraction{}, domain.InternalError("write source inspection PDF staging file", err)
	}
	if err := file.Close(); err != nil {
		return pdfExtraction{}, domain.InternalError("close source inspection PDF staging file", err)
	}
	return extractPDF(tempPath)
}

func pdfText(value pdf.Value) string {
	text := strings.TrimSpace(value.Text())
	if text != "" {
		return text
	}
	text = strings.TrimSpace(value.TextFromUTF16())
	if text != "" {
		return text
	}
	return strings.TrimSpace(value.RawString())
}

func resolvedSourceTitle(requestTitle string, pdfTitle string, sourcePath string) string {
	if title := strings.TrimSpace(requestTitle); title != "" {
		return title
	}
	if title := strings.TrimSpace(pdfTitle); title != "" {
		return title
	}
	return strings.TrimSuffix(path.Base(sourcePath), path.Ext(sourcePath))
}

func resolvedInspectionTitle(requestTitle string, extractedTitle string, sourceURL string) string {
	if title := strings.TrimSpace(requestTitle); title != "" {
		return title
	}
	if title := strings.TrimSpace(extractedTitle); title != "" {
		return title
	}
	parsed, err := url.Parse(sourceURL)
	if err == nil {
		base := strings.TrimSuffix(path.Base(parsed.Path), path.Ext(parsed.Path))
		if base != "" && base != "." && base != "/" {
			return base
		}
		if parsed.Hostname() != "" {
			return parsed.Hostname()
		}
	}
	return "source"
}

func buildPDFSourceNoteBody(sourceURL string, sourcePath string, assetPath string, title string, sha string, sizeBytes int64, mimeType string, capturedAt time.Time, extracted pdfExtraction) string {
	var body strings.Builder
	body.WriteString("---\n")
	body.WriteString("type: source\n")
	body.WriteString("source_type: pdf\n")
	body.WriteString("modality: markdown\n")
	body.WriteString("source_url: " + frontmatterScalar(sourceURL) + "\n")
	body.WriteString("asset_path: " + frontmatterScalar(assetPath) + "\n")
	body.WriteString("derived_path: " + frontmatterScalar(sourcePath) + "\n")
	body.WriteString("sha256: " + frontmatterScalar(sha) + "\n")
	_, _ = fmt.Fprintf(&body, "size_bytes: %d\n", sizeBytes)
	body.WriteString("mime_type: " + frontmatterScalar(mimeType) + "\n")
	_, _ = fmt.Fprintf(&body, "page_count: %d\n", extracted.Pages)
	body.WriteString("captured_at: " + frontmatterScalar(capturedAt.Format(time.RFC3339Nano)) + "\n")
	if extracted.Metadata.Title != "" {
		body.WriteString("pdf_title: " + frontmatterScalar(extracted.Metadata.Title) + "\n")
	}
	if extracted.Metadata.Author != "" {
		body.WriteString("pdf_author: " + frontmatterScalar(extracted.Metadata.Author) + "\n")
	}
	if extracted.Metadata.PublishedDate != "" {
		body.WriteString("pdf_published_date: " + frontmatterScalar(extracted.Metadata.PublishedDate) + "\n")
	}
	body.WriteString("---\n")
	body.WriteString("# " + sourceNoteInline(title) + "\n\n")
	body.WriteString("## Summary\n")
	body.WriteString("PDF source ingested from " + sourceURL + ".\n\n")
	body.WriteString("## Source Asset\n")
	body.WriteString("- Source URL: " + sourceURL + "\n")
	body.WriteString("- Asset path: " + assetPath + "\n")
	body.WriteString("- SHA256: " + sha + "\n")
	_, _ = fmt.Fprintf(&body, "- Size bytes: %d\n", sizeBytes)
	_, _ = fmt.Fprintf(&body, "- Page count: %d\n\n", extracted.Pages)
	if extracted.Metadata.Title != "" || extracted.Metadata.Author != "" || extracted.Metadata.PublishedDate != "" {
		body.WriteString("## PDF Metadata\n")
		if extracted.Metadata.Title != "" {
			body.WriteString("- Title: " + sourceNoteInline(extracted.Metadata.Title) + "\n")
		}
		if extracted.Metadata.Author != "" {
			body.WriteString("- Author: " + sourceNoteInline(extracted.Metadata.Author) + "\n")
		}
		if extracted.Metadata.PublishedDate != "" {
			body.WriteString("- Published date: " + sourceNoteInline(extracted.Metadata.PublishedDate) + "\n")
		}
		body.WriteString("\n")
	}
	body.WriteString("## Extracted Text\n")
	if extracted.Text != "" {
		body.WriteString(sourceNoteTextBlock(extracted.Text))
	} else {
		body.WriteString(sourceNoteTextBlock(sourceMetadataFallbackText(extracted.Metadata, title)))
	}
	return body.String()
}

func buildWebSourceNoteBody(sourceURL string, sourcePath string, title string, sha string, sizeBytes int64, mimeType string, capturedAt time.Time, extracted webExtraction) string {
	var body strings.Builder
	body.WriteString("---\n")
	body.WriteString("type: source\n")
	body.WriteString("source_type: web\n")
	body.WriteString("modality: markdown\n")
	body.WriteString("source_url: " + frontmatterScalar(sourceURL) + "\n")
	body.WriteString("derived_path: " + frontmatterScalar(sourcePath) + "\n")
	body.WriteString("sha256: " + frontmatterScalar(sha) + "\n")
	_, _ = fmt.Fprintf(&body, "size_bytes: %d\n", sizeBytes)
	body.WriteString("mime_type: " + frontmatterScalar(mimeType) + "\n")
	body.WriteString("captured_at: " + frontmatterScalar(capturedAt.Format(time.RFC3339Nano)) + "\n")
	if extracted.Title != "" {
		body.WriteString("source_title: " + frontmatterScalar(extracted.Title) + "\n")
	}
	body.WriteString("---\n")
	body.WriteString("# " + sourceNoteInline(title) + "\n\n")
	body.WriteString("## Summary\n")
	body.WriteString("Web source ingested from " + sourceURL + ".\n\n")
	body.WriteString("## Source Page\n")
	body.WriteString("- Source URL: " + sourceURL + "\n")
	body.WriteString("- SHA256: " + sha + "\n")
	_, _ = fmt.Fprintf(&body, "- Size bytes: %d\n", sizeBytes)
	if extracted.Title != "" {
		body.WriteString("- Page title: " + sourceNoteInline(extracted.Title) + "\n")
	}
	body.WriteString("\n## Extracted Text\n")
	if extracted.Text != "" {
		body.WriteString(sourceNoteTextBlock(extracted.Text))
	} else {
		body.WriteString(sourceNoteTextBlock("No visible page text was found."))
	}
	return body.String()
}

func sourceNoteInline(value string) string {
	return html.EscapeString(markdownLine(value))
}

func sourceNoteTextBlock(value string) string {
	text := strings.TrimSpace(strings.ReplaceAll(value, "\r\n", "\n"))
	fence := "```"
	for strings.Contains(text, fence) {
		fence += "`"
	}
	return fence + "text\n" + text + "\n" + fence + "\n"
}

var (
	htmlScriptStylePattern  = regexp.MustCompile(`(?is)<(script|style|noscript|svg|template)\b[^>]*>.*?</(script|style|noscript|svg|template)>`)
	htmlCommentPattern      = regexp.MustCompile(`(?is)<!--.*?-->`)
	htmlTitlePattern        = regexp.MustCompile(`(?is)<title\b[^>]*>(.*?)</title>`)
	htmlTagPattern          = regexp.MustCompile(`(?is)<[^>]+>`)
	htmlWhitespacePattern   = regexp.MustCompile(`[ \t\r\n]+`)
	htmlAnchorPattern       = regexp.MustCompile(`(?is)<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^'"\s>]+))[^>]*>(.*?)</a>`)
	markdownLinkPattern     = regexp.MustCompile(`(?m)\[([^\]\n]{0,180})\]\(([^)\s]+)(?:\s+["'][^)]*["'])?\)`)
	markdownAutolinkPattern = regexp.MustCompile(`(?i)<(https?://[^>\s]+)>`)
)

func extractWebPage(body []byte) (webExtraction, error) {
	raw := string(body)
	lower := strings.ToLower(raw)
	for _, marker := range []string{"captcha", "sign in to continue", "login required", "access denied", "enable cookies"} {
		if strings.Contains(lower, marker) {
			return webExtraction{}, domain.ValidationError("source URL appears to require unsupported interactive, authenticated, or restricted access", nil)
		}
	}
	withoutHidden := htmlScriptStylePattern.ReplaceAllString(raw, " ")
	withoutHidden = htmlCommentPattern.ReplaceAllString(withoutHidden, " ")
	title := ""
	if match := htmlTitlePattern.FindStringSubmatch(withoutHidden); len(match) > 1 {
		title = cleanExtractedHTMLText(match[1])
	}
	text := htmlTagPattern.ReplaceAllString(withoutHidden, " ")
	text = cleanExtractedHTMLText(text)
	if text == "" && title == "" {
		return webExtraction{}, domain.ValidationError("source URL did not expose visible HTML text", nil)
	}
	return webExtraction{Title: title, Text: text}, nil
}

func extractWebSource(sourceURL string, mimeType string, body []byte) (webExtraction, error) {
	if isMarkdownSource(sourceURL, mimeType) {
		return extractMarkdownSource(body)
	}
	return extractWebPage(body)
}

func extractMarkdownSource(body []byte) (webExtraction, error) {
	text := strings.TrimSpace(strings.ReplaceAll(string(body), "\r\n", "\n"))
	if text == "" {
		return webExtraction{}, domain.ValidationError("source URL did not expose Markdown text", nil)
	}
	return webExtraction{Title: markdownSourceTitle(text), Text: text}, nil
}

func markdownSourceTitle(text string) string {
	lines := strings.Split(text, "\n")
	bodyStart := 0
	if len(lines) > 0 && strings.TrimSpace(lines[0]) == "---" {
		for idx := 1; idx < len(lines) && idx < 80; idx++ {
			line := strings.TrimSpace(lines[idx])
			if line == "---" {
				bodyStart = idx + 1
				break
			}
			if key, value, ok := strings.Cut(line, ":"); ok && strings.EqualFold(strings.TrimSpace(key), "title") {
				if title := cleanMarkdownTitle(value); title != "" {
					return title
				}
			}
		}
	}
	for _, line := range lines[bodyStart:] {
		trimmed := strings.TrimSpace(line)
		hashes := 0
		for hashes < len(trimmed) && trimmed[hashes] == '#' {
			hashes++
		}
		if hashes > 0 && hashes <= 6 && hashes < len(trimmed) && trimmed[hashes] == ' ' {
			if title := cleanMarkdownTitle(trimmed[hashes:]); title != "" {
				return title
			}
		}
	}
	return ""
}

func cleanMarkdownTitle(value string) string {
	value = strings.TrimSpace(value)
	value = strings.Trim(value, `"'`)
	return strings.TrimSpace(value)
}

func cleanExtractedHTMLText(value string) string {
	value = html.UnescapeString(value)
	value = strings.ReplaceAll(value, "\u00a0", " ")
	value = htmlWhitespacePattern.ReplaceAllString(value, " ")
	return strings.TrimSpace(value)
}

func sourceTextPreview(value string) string {
	preview := strings.Join(strings.Fields(value), " ")
	runes := []rune(preview)
	if len(runes) <= maxSourceTextPreviewRunes {
		return preview
	}
	return string(runes[:maxSourceTextPreviewRunes])
}

func extractSourceLinks(sourceURL string, mimeType string, body []byte, limit int) []domain.SourceURLInspectionLink {
	if limit <= 0 {
		return nil
	}
	if isMarkdownSource(sourceURL, mimeType) {
		return extractMarkdownSourceLinks(sourceURL, string(body), limit)
	}
	return extractHTMLSourceLinks(sourceURL, string(body), limit)
}

func extractMarkdownSourceLinks(sourceURL string, body string, limit int) []domain.SourceURLInspectionLink {
	links := []domain.SourceURLInspectionLink{}
	for _, match := range boundedRegexpSubmatches(markdownLinkPattern, body, maxSourceInspectionLinkScans) {
		if len(match) < 3 {
			continue
		}
		links = appendInspectionLink(links, sourceURL, match[2], match[1])
		if len(links) >= limit {
			return links
		}
	}
	for _, match := range boundedRegexpSubmatches(markdownAutolinkPattern, body, maxSourceInspectionLinkScans) {
		if len(match) < 2 {
			continue
		}
		links = appendInspectionLink(links, sourceURL, match[1], "")
		if len(links) >= limit {
			return links
		}
	}
	return links
}

func extractHTMLSourceLinks(sourceURL string, body string, limit int) []domain.SourceURLInspectionLink {
	links := []domain.SourceURLInspectionLink{}
	for _, match := range boundedRegexpSubmatches(htmlAnchorPattern, body, maxSourceInspectionLinkScans) {
		if len(match) < 5 {
			continue
		}
		href := firstNonEmpty(match[1], match[2], match[3])
		text := cleanExtractedHTMLText(htmlTagPattern.ReplaceAllString(match[4], " "))
		links = appendInspectionLink(links, sourceURL, href, text)
		if len(links) >= limit {
			return links
		}
	}
	return links
}

func boundedRegexpSubmatches(pattern *regexp.Regexp, body string, maxScans int) [][]string {
	if maxScans <= 0 {
		return nil
	}
	matches := make([][]string, 0, min(maxScans, maxSourceInspectionLinks))
	start := 0
	for scans := 0; scans < maxScans && start <= len(body); scans++ {
		indexes := pattern.FindStringSubmatchIndex(body[start:])
		if indexes == nil {
			break
		}
		match := make([]string, len(indexes)/2)
		for i := 0; i < len(indexes); i += 2 {
			if indexes[i] < 0 || indexes[i+1] < 0 {
				continue
			}
			match[i/2] = body[start+indexes[i] : start+indexes[i+1]]
		}
		matches = append(matches, match)
		advance := indexes[1]
		if advance <= 0 {
			advance = 1
		}
		start += advance
	}
	return matches
}

func appendInspectionLink(links []domain.SourceURLInspectionLink, sourceURL string, rawURL string, text string) []domain.SourceURLInspectionLink {
	normalizedURL, ok := normalizeInspectionLinkURL(sourceURL, rawURL)
	if !ok {
		return links
	}
	for _, existing := range links {
		if existing.URL == normalizedURL {
			return links
		}
	}
	return append(links, domain.SourceURLInspectionLink{
		URL:  normalizedURL,
		Text: sourceTextPreview(text),
	})
}

func normalizeInspectionLinkURL(sourceURL string, rawURL string) (string, bool) {
	rawURL = strings.TrimSpace(rawURL)
	if rawURL == "" || strings.HasPrefix(rawURL, "#") {
		return "", false
	}
	lower := strings.ToLower(rawURL)
	for _, prefix := range []string{"mailto:", "javascript:", "tel:", "data:"} {
		if strings.HasPrefix(lower, prefix) {
			return "", false
		}
	}
	base, err := url.Parse(sourceURL)
	if err != nil {
		return "", false
	}
	parsed, err := url.Parse(rawURL)
	if err != nil {
		return "", false
	}
	resolved := base.ResolveReference(parsed)
	resolved.Fragment = ""
	if resolved.Scheme != "http" && resolved.Scheme != "https" {
		return "", false
	}
	normalized, err := normalizeSourceURL(resolved.String())
	if err != nil {
		return "", false
	}
	return normalizeGitHubMarkdownSourceURL(normalized), true
}

func sourceMetadataFallbackText(metadata domain.SourcePDFMetadata, title string) string {
	parts := []string{"No extractable page text was found."}
	if strings.TrimSpace(title) != "" {
		parts = append(parts, "Title: "+title+".")
	}
	if metadata.Author != "" {
		parts = append(parts, "Author: "+metadata.Author+".")
	}
	if metadata.PublishedDate != "" {
		parts = append(parts, "Published date: "+metadata.PublishedDate+".")
	}
	return strings.Join(parts, " ")
}

func (s *Store) sourceDocumentCitations(ctx context.Context, document domain.Document) ([]domain.Citation, error) {
	chunksByDoc, err := s.loadChunksByDoc(ctx)
	if err != nil {
		return nil, err
	}
	return []domain.Citation{documentCitation(document, chunksByDoc[document.DocID])}, nil
}

func validateIngestedSource(document domain.Document, sourceURL string, sourcePath string, assetPath string, assetAbsPath string) error {
	if document.DocID == "" || document.Path != sourcePath {
		return domain.InternalError("validate ingested source document", fmt.Errorf("created document path %q does not match %q", document.Path, sourcePath))
	}
	sourceType := sourceTypePDF
	if assetPath == "" {
		sourceType = sourceTypeWeb
	}
	requiredMetadata := map[string]string{
		"type":         "source",
		"source_type":  sourceType,
		"modality":     "markdown",
		"source_url":   sourceURL,
		"derived_path": sourcePath,
	}
	if sourceType == sourceTypePDF {
		requiredMetadata["asset_path"] = assetPath
	}
	for key, want := range requiredMetadata {
		if got := strings.TrimSpace(document.Metadata[key]); got != want {
			return domain.InternalError("validate ingested source metadata", fmt.Errorf("metadata %s = %q, want %q", key, got, want))
		}
	}
	if assetAbsPath != "" {
		if _, err := osStat(assetAbsPath); err != nil {
			return domain.InternalError("validate ingested source asset", err)
		}
	} else if strings.TrimSpace(document.Metadata["asset_path"]) != "" {
		return domain.InternalError("validate ingested source asset", errors.New("web source must not record asset_path metadata"))
	}
	return nil
}

func sourceProvenanceDetails(relPath string, frontmatter map[string]string) map[string]string {
	details := map[string]string{"path": relPath}
	for _, key := range []string{
		"source_type",
		"source_url",
		"asset_path",
		"derived_path",
		"sha256",
		"size_bytes",
		"mime_type",
		"page_count",
		"captured_at",
		"source_title",
		"transcript_origin",
		"transcript_policy",
		"language",
		"transcript_sha256",
		"tool",
		"model",
	} {
		if value := strings.TrimSpace(frontmatter[key]); value != "" {
			details[key] = value
		}
	}
	return details
}

func sourceUpdateProvenanceDetails(relPath string, frontmatter map[string]string, previous map[string]string) map[string]string {
	details := sourceProvenanceDetails(relPath, frontmatter)
	for _, key := range []string{"sha256", "page_count", "captured_at", "transcript_sha256"} {
		if value := strings.TrimSpace(previous[key]); value != "" {
			details["previous_"+key] = value
		}
		if value := strings.TrimSpace(frontmatter[key]); value != "" {
			details["new_"+key] = value
		}
	}
	return details
}
