#!/usr/bin/env python3
"""
Wikidata Aerial Heritage Scraper.

Queries Wikidata's public SPARQL endpoint for aerial photography records that
have Wikidata entries. Produces NAPH-compliant Turtle output.

Wikidata is a useful source because:
- Public SPARQL endpoint (no API key required)
- Cross-references to GeoNames, VIAF, library catalogues
- Multilingual labels available
- Records are CC0 (public domain)

Note: Wikidata holds METADATA records about aerial photographs (e.g. famous
historical aerial frames with their own articles). It does NOT hold the
photographs themselves. The scraped output is structured metadata; institutional
URI references point back to Wikidata for canonical authority.

Usage:
    python3 pipeline/scrapers/wikidata.py > out.ttl

Politeness:
- Single SPARQL query (not per-record fetches)
- Standard User-Agent
- Wikidata Query Service explicitly permits this use case
- Output is cached locally; rerun only when refresh is needed
"""

import argparse
import json
import re
import subprocess
import sys
import time
import urllib.parse

WIKIDATA_SPARQL = "https://query.wikidata.org/sparql"

USER_AGENT = "NAPH-Research/1.0 (https://w3id.org/naph; fabio@thetesseractacademy.com)"

# Wikidata SPARQL — looks for photograph items (Q125191) with coordinates AND date.
# This is broader than just aerial photography because Wikidata's coverage of
# explicitly-tagged aerial photographs is sparse. The output is a baseline
# dataset of photographic records with computation-ready metadata that
# institutions can use as a starting point for their own NAPH adoption.
#
# To target only aerial photographs (when more Wikidata coverage exists),
# add: ?item wdt:P186 wd:Q1124313 .
QUERY = """
SELECT DISTINCT ?item ?itemLabel ?coordLocation ?date ?inception WHERE {
  ?item wdt:P31 wd:Q125191 ;
        wdt:P625 ?coordLocation ;
        wdt:P577 ?date .
  OPTIONAL { ?item wdt:P571 ?inception }
  SERVICE wikibase:label { bd:serviceParam wikibase:language "en" }
}
LIMIT 200
"""


def query_wikidata(sparql: str, timeout: int = 30) -> list[dict]:
    """Execute a SPARQL query against Wikidata and return result rows.

    Uses curl via subprocess to leverage system SSL certs reliably on macOS.
    """
    encoded = urllib.parse.urlencode({"query": sparql, "format": "json"})
    url = f"{WIKIDATA_SPARQL}?{encoded}"

    proc = subprocess.run(
        [
            "curl", "-sS", "--max-time", str(timeout),
            "-H", f"User-Agent: {USER_AGENT}",
            "-H", "Accept: application/sparql-results+json",
            url,
        ],
        capture_output=True,
        text=True,
    )

    if proc.returncode != 0:
        raise RuntimeError(f"curl failed: {proc.stderr.strip()}")

    try:
        data = json.loads(proc.stdout)
    except json.JSONDecodeError as e:
        raise RuntimeError(f"Wikidata returned non-JSON: {proc.stdout[:200]}") from e

    bindings = data.get("results", {}).get("bindings", [])
    rows = []
    for b in bindings:
        row = {}
        for k, v in b.items():
            row[k] = v.get("value", "")
        rows.append(row)
    return rows


def parse_coords(point_wkt: str) -> tuple[float, float] | None:
    """Parse Wikidata's POINT(lon lat) coordinate format."""
    match = re.match(r"Point\s*\(\s*([-\d.]+)\s+([-\d.]+)\s*\)", point_wkt)
    if not match:
        return None
    lon = float(match.group(1))
    lat = float(match.group(2))
    return lat, lon


def safe_id(uri: str) -> str:
    """Extract Q-number from Wikidata URI for use as local identifier."""
    match = re.search(r"/Q(\d+)$", uri)
    return f"Q{match.group(1)}" if match else "unknown"


def emit_turtle(rows: list[dict]) -> str:
    """Convert SPARQL result rows into NAPH-compliant Turtle."""
    out = []
    out.append("""@prefix rdf:     <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs:    <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xsd:     <http://www.w3.org/2001/XMLSchema#> .
@prefix skos:    <http://www.w3.org/2004/02/skos/core#> .
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix dctype:  <http://purl.org/dc/dcmitype/> .
@prefix geo:     <http://www.opengis.net/ont/geosparql#> .
@prefix naph:    <https://w3id.org/naph/ontology#> .
@prefix wd:      <http://www.wikidata.org/entity/> .
@prefix ex:      <https://w3id.org/naph/example/wikidata/> .

# =============================================================================
# Generated by pipeline/scrapers/wikidata.py
# Source: Wikidata SPARQL endpoint (https://query.wikidata.org/sparql)
# Licence: CC0 (Wikidata content)
# =============================================================================

ex:WikidataAuthority a naph:CustodialInstitution ;
    rdfs:label "Wikidata aerial heritage records (federated authority)" ;
    rdfs:comment "Records sourced from Wikidata's public SPARQL endpoint. Wikidata is not a custodial institution per se — these records identify aerial photographs that have authoritative Wikidata entries." .

ex:WikidataCollection a naph:Collection ;
    rdfs:label "Wikidata aerial heritage references" ;
    naph:custodian ex:WikidataAuthority .

# Default rights for Wikidata-sourced metadata
ex:WikidataRights a naph:RightsStatement ;
    naph:rightsURI <https://creativecommons.org/publicdomain/zero/1.0/> ;
    naph:rightsLabel "CC0 Public Domain Dedication" ;
    rdfs:comment "Wikidata content is released under CC0. Note: this licenses the metadata, not necessarily the photographs themselves." .

""")

    seen_items = set()
    seen_depicts = set()

    for row in rows:
        item_uri = row.get("item", "")
        if not item_uri or item_uri in seen_items:
            continue
        seen_items.add(item_uri)

        qid = safe_id(item_uri)
        label = row.get("itemLabel", qid)

        # Date — prefer publication date (P577), fall back to inception (P571)
        date_str = row.get("date") or row.get("inception", "")
        iso_date = None
        if date_str:
            # Wikidata dates: "1944-03-28T00:00:00Z" or "+1944-03-28T00:00:00Z"
            match = re.match(r"[\+\-]?(\d{4}-\d{2}-\d{2})", date_str)
            if match:
                iso_date = match.group(1)

        # Coordinates
        coord_str = row.get("coordLocation", "")
        coords = parse_coords(coord_str) if coord_str else None

        safe_label = label.replace('"', '\\"')
        out.append(f"# Wikidata item {qid}: {label}")
        out.append(f"ex:sortie-{qid} a naph:Sortie ;")
        out.append(f'    naph:collectionCode "WIKIDATA" ;')
        out.append(f'    naph:sortieReference "{qid}" ;')
        out.append(f'    rdfs:comment "Synthetic Sortie minted for Wikidata-sourced record (no original sortie metadata)." .')
        out.append(f"")
        out.append(f"ex:photo-{qid} a naph:AerialPhotograph ;")
        out.append(f"    dcterms:type dctype:StillImage ;")
        out.append(f'    rdfs:label "{safe_label}" ;')
        out.append(f'    naph:hasIdentifier "https://w3id.org/naph/example/wikidata/photo/{qid}" ;')
        out.append(f"    skos:exactMatch <{item_uri}> ;")
        out.append(f"    naph:partOfSortie ex:sortie-{qid} ;")
        out.append(f"    naph:belongsToCollection ex:WikidataCollection ;")
        out.append(f"    naph:hasRightsStatement ex:WikidataRights ;")

        if iso_date:
            out.append(f'    naph:capturedOn "{iso_date}"^^xsd:date ;')

        if coords:
            lat, lon = coords
            # Construct a small footprint around the point (1 km box)
            half = 0.005  # ~500m at equator
            wkt = f"POLYGON(({lon-half:.4f} {lat-half:.4f}, {lon+half:.4f} {lat-half:.4f}, {lon+half:.4f} {lat+half:.4f}, {lon-half:.4f} {lat+half:.4f}, {lon-half:.4f} {lat-half:.4f}))"
            out.append(f"    naph:coversArea ex:footprint-{qid} ;")
            out.append(f"    naph:compliesWithTier naph:TierBaseline .")
            out.append(f"")
            out.append(f"ex:footprint-{qid} a naph:GeographicFootprint ;")
            out.append(f'    naph:asWKT "{wkt}"^^geo:wktLiteral .')
        else:
            # No coords → can't claim Baseline
            out[-1] = out[-1].rstrip(" ;") + " ."  # close the photo statement
            out.append(f"# Note: Wikidata item {qid} lacks geographic coordinates — not Baseline-compliant")

        out.append("")

    return "\n".join(out)


def main():
    parser = argparse.ArgumentParser(
        description="Scrape Wikidata for aerial photography heritage records, output NAPH Turtle.",
    )
    parser.add_argument("--limit", type=int, default=50,
                        help="Maximum number of records to fetch (default 50)")
    parser.add_argument("--query-file", default=None,
                        help="Read SPARQL query from file instead of using default")
    args = parser.parse_args()

    if args.query_file:
        with open(args.query_file) as f:
            sparql = f.read()
    else:
        # Substitute LIMIT
        sparql = QUERY.replace("LIMIT 200", f"LIMIT {args.limit}")

    print(f"# Querying Wikidata for up to {args.limit} aerial photography records...", file=sys.stderr)
    print(f"# User-Agent: {USER_AGENT}", file=sys.stderr)

    start = time.perf_counter()
    rows = query_wikidata(sparql)
    elapsed = time.perf_counter() - start

    print(f"# Received {len(rows)} bindings in {elapsed:.2f}s", file=sys.stderr)

    if not rows:
        print("# No results returned from Wikidata.", file=sys.stderr)
        sys.exit(1)

    turtle = emit_turtle(rows)
    print(turtle)


if __name__ == "__main__":
    main()
