#!/usr/bin/env python3
from __future__ import annotations
import csv, json, re
from pathlib import Path
from urllib.parse import urlparse

BASE = Path(__file__).resolve().parent
ROW_FILE = BASE / "dlr_structured_rows.json"
CSV_FILE = BASE / "dlr_structured_rows.csv"
MANIFEST_FILE = BASE / "source_manifest.csv"
RESULT_FILE = BASE / "validation_results.json"
REQUIRED_FIELDS = ["company","period","metric_name","raw_value","normalized_value","unit","value_qualifier","ownership_basis","forecast_or_actual","is_derived","source_url","source_location","source_snippet","snippet_support_status","confidence","caveat"]
MANIFEST_REQUIRED_FIELDS = ["source_id","company","source_url","source_type","period","expected_table_or_section","access_caveats"]
ALLOWED_UNITS = {"MW","USD","percent","months","acres","text"}
ALLOWED_BASIS = {"100% share","Digital Realty share","consolidated","consolidated development projects","unconsolidated/JV basis"}
ALLOWED_CLASS = {"forecast","actual"}
ALLOWED_QUALIFIERS = {"reported","reported_caveat","derived_qoq","derived_sum","range","weighted_average_lag","greater_than","less_than_or_equal","approximate"}
ALLOWED_SNIPPET_STATUS = {"supported","partially_supported","unsupported"}

def load_csv(path):
    with path.open(newline="", encoding="utf-8") as f:
        return list(csv.DictReader(f))

def is_https(url):
    parsed = urlparse(str(url))
    return parsed.scheme == "https" and bool(parsed.netloc)

def normalized_ok(row):
    value, unit, qualifier = row.get("normalized_value"), row.get("unit"), row.get("value_qualifier")
    if unit == "text":
        return isinstance(value, str) and bool(value.strip())
    if qualifier == "range":
        return isinstance(value, list) and len(value) == 2 and all(isinstance(v, (int, float)) for v in value) and value[0] <= value[1]
    return isinstance(value, (int, float)) and not isinstance(value, bool)

def validate():
    rows = json.loads(ROW_FILE.read_text(encoding="utf-8"))
    csv_rows = load_csv(CSV_FILE)
    manifest = load_csv(MANIFEST_FILE)
    checks, row_errors = [], []
    def add(name, passed, detail):
        checks.append({"check_name": name, "status": "pass" if passed else "fail", "detail": detail})
    for idx, row in enumerate(rows, 1):
        missing = [f for f in REQUIRED_FIELDS if f not in row or row[f] in (None, "")]
        if missing: row_errors.append(f"row {idx} missing {missing}")
        if row.get("company") != "DLR": row_errors.append(f"row {idx} company not DLR")
        if not re.fullmatch(r"20\d{2}Q[1-4]", str(row.get("period", ""))): row_errors.append(f"row {idx} period not quarter format")
        if row.get("unit") not in ALLOWED_UNITS: row_errors.append(f"row {idx} unit not allowed: {row.get('unit')}")
        if row.get("ownership_basis") not in ALLOWED_BASIS: row_errors.append(f"row {idx} ownership_basis not allowed: {row.get('ownership_basis')}")
        if row.get("forecast_or_actual") not in ALLOWED_CLASS: row_errors.append(f"row {idx} forecast_or_actual invalid: {row.get('forecast_or_actual')}")
        if not isinstance(row.get("is_derived"), bool): row_errors.append(f"row {idx} is_derived must be boolean")
        if row.get("value_qualifier") not in ALLOWED_QUALIFIERS: row_errors.append(f"row {idx} value_qualifier not allowed: {row.get('value_qualifier')}")
        if row.get("snippet_support_status") not in ALLOWED_SNIPPET_STATUS: row_errors.append(f"row {idx} snippet_support_status invalid")
        if row.get("snippet_support_status") == "supported" and len(str(row.get("source_snippet", "")).strip()) < 20: row_errors.append(f"row {idx} supported snippet too short")
        if not is_https(row.get("source_url", "")): row_errors.append(f"row {idx} source_url not https")
        if not normalized_ok(row): row_errors.append(f"row {idx} normalized_value invalid for {row.get('unit')} / {row.get('value_qualifier')}")
    add("required_fields", not any("missing" in e for e in row_errors), "All structured rows must populate production schema fields.")
    add("row_count", len(rows) >= 20 and len(csv_rows) == len(rows), f"json_rows={len(rows)} csv_rows={len(csv_rows)} minimum=20")
    add("source_urls", not any("source_url" in e for e in row_errors), "Every structured row must use a direct https primary-source URL.")
    add("unit_normalization", not any("unit not allowed" in e or "normalized_value invalid" in e for e in row_errors), "Numeric units must be normalized to MW, USD, percent, months, or acres; text caveat rows must use unit=text.")
    add("ownership_basis", not any("ownership_basis" in e for e in row_errors), "Rows must distinguish 100% share, Digital Realty share, consolidated, consolidated development projects, and unconsolidated/JV basis.")
    add("forecast_or_actual", not any("forecast_or_actual" in e for e in row_errors), "Rows must be marked forecast or actual.")
    add("derived_flag", not any("is_derived" in e for e in row_errors) and sum(1 for r in rows if r.get("is_derived")) >= 2, "Derived rows must be boolean and include QoQ/sum examples.")
    add("snippet_presence", not any("snippet" in e for e in row_errors), "Supported rows must include primary-source snippets.")
    add("snippet_support_status", not any("snippet_support_status" in e for e in row_errors) and all(r.get("snippet_support_status") == "supported" for r in rows), "MVP rows are required to be supported by snippets.")
    add("value_qualifiers", not any("value_qualifier" in e for e in row_errors) and {"approximate","greater_than","range","derived_qoq","derived_sum"}.issubset({r.get("value_qualifier") for r in rows}), "Rows must exercise approximate, greater-than, range, and derived qualifiers.")
    add("actual_vs_forecast_mix", {"actual","forecast"}.issubset({r.get("forecast_or_actual") for r in rows}), "Rows must include actual and forecast observations.")
    add("manifest_required_fields", all(all(src.get(f) for f in MANIFEST_REQUIRED_FIELDS) for src in manifest), "Source manifest must include direct primary URLs, type, period, expected section, and access caveat.")
    add("manifest_primary_urls", all(is_https(src.get("source_url", "")) for src in manifest), "Manifest source URLs must be direct https primary-source URLs.")
    add("manifest_covers_cited_urls", {r["source_url"] for r in rows}.issubset({s["source_url"] for s in manifest}), "Every cited source URL must appear in source_manifest.csv.")
    add("dlr_specific_basis_lessons", {"100% share","Digital Realty share","consolidated","consolidated development projects","unconsolidated/JV basis"}.issubset({r.get("ownership_basis") for r in rows}), "Schema must represent 100% vs DLR share, consolidated capex, and unconsolidated/JV caveat rows.")
    add("independent_sample_verification", all(rows[i]["snippet_support_status"] == "supported" and rows[i]["source_snippet"] for i in [0,3,11,20,21,26]), "Fixed sample rows verified against primary-source snippets included in row data.")
    result = {
        "status": "pass" if all(c["status"] == "pass" for c in checks) else "fail",
        "company": "DLR",
        "period": "2026Q1",
        "row_count": len(rows),
        "csv_row_count": len(csv_rows),
        "manifest_row_count": len(manifest),
        "checks": checks,
        "row_errors": row_errors,
        "ownership_basis_counts": {b: sum(1 for r in rows if r.get("ownership_basis") == b) for b in sorted(ALLOWED_BASIS)},
        "value_qualifier_counts": {q: sum(1 for r in rows if r.get("value_qualifier") == q) for q in sorted({r.get("value_qualifier") for r in rows})},
        "forecast_or_actual_counts": {k: sum(1 for r in rows if r.get("forecast_or_actual") == k) for k in sorted(ALLOWED_CLASS)},
    }
    RESULT_FILE.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    return result

if __name__ == "__main__":
    report = validate()
    if report["status"] != "pass":
        raise SystemExit(1)