#!/usr/bin/env python3 from __future__ import annotations import csv import hashlib import json import re import shutil import subprocess import tempfile from pathlib import Path from urllib.request import Request, urlopen OUT = Path(__file__).resolve().parent RAW = OUT / "sources" / "raw" TEXT = OUT / "sources" / "text" SUPP = "https://investor.digitalrealty.com/static-files/953419cb-91ee-4485-8017-26ee0b29bb2a" PRESS = "https://investor.digitalrealty.com/news-releases/news-release-details/digital-realty-reports-first-quarter-2026-results" SUPP4 = "https://investor.digitalrealty.com/static-files/4bac803d-f2a7-400a-b4db-1e8804a60414" SEC10K = "https://www.sec.gov/Archives/edgar/data/1297996/000110465926015365/dlr-20251231x10k.htm" FIELDS = [ "company","period","metric_name","raw_value","normalized_value","unit", "value_qualifier","ownership_basis","forecast_or_actual","is_derived", "source_url","source_location","source_snippet","snippet_support_status", "confidence","caveat" ] VFIELDS = [ "row_id","company","period","metric_name","raw_value","normalized_value", "unit","source_url","source_location","snippet","verdict","confidence","caveat" ] SOURCES = [ { "source_id": "dlr_2026q1_supp", "company": "DLR", "source_url": SUPP, "source_type": "investor_supplemental_pdf", "period": "2026Q1", "raw_cache_path": "sources/raw/dlr_2026q1_supp.pdf", "text_cache_path": "sources/text/dlr_2026q1_supp.txt", "expected_table_or_section": "Key Quarterly Financial Data; Earnings Release; 2026 Outlook; Occupancy Analysis; Development Lifecycle; Historical Capital Expenditures and Investments in Real Estate", "access_caveats": "Primary PDF is hosted as a static IR file without a .pdf extension; extractor persists the raw response and uses pdftotext for repeatable source-text parsing." }, { "source_id": "dlr_2026q1_press", "company": "DLR", "source_url": PRESS, "source_type": "earnings_press_release_html", "period": "2026Q1", "raw_cache_path": "sources/raw/dlr_2026q1_press.html", "text_cache_path": "sources/text/dlr_2026q1_press.txt", "expected_table_or_section": "Highlights; Leasing Activity; Investment Activity; 2026 Outlook", "access_caveats": "Repeatable source family: Digital Realty investor relations earnings release HTML; used as a cross-check when available." }, { "source_id": "dlr_2025q4_supp", "company": "DLR", "source_url": SUPP4, "source_type": "investor_supplemental_pdf", "period": "2025Q4", "raw_cache_path": "sources/raw/dlr_2025q4_supp.pdf", "text_cache_path": "sources/text/dlr_2025q4_supp.txt", "expected_table_or_section": "Key Quarterly Financial Data; Occupancy Analysis; Development Lifecycle; Historical Capital Expenditures and Investments in Real Estate", "access_caveats": "Repeatable source family: Digital Realty prior-quarter investor supplement; used for quarter-over-quarter derived checks." }, { "source_id": "dlr_2025_10k", "company": "DLR", "source_url": SEC10K, "source_type": "sec_10k_html", "period": "FY2025", "raw_cache_path": "sources/raw/dlr_2025_10k.html", "text_cache_path": "sources/text/dlr_2025_10k.txt", "expected_table_or_section": "Business; MD&A; Properties; Capital expenditures / development discussion", "access_caveats": "Repeatable source family: SEC EDGAR annual filing; current production rows are sourced to quarterly primary materials." }, ] def fetch(url: str, path: Path) -> None: if path.exists() and path.stat().st_size: return req = Request(url, headers={"User-Agent": "research-extractor/1.0 contact@example.com"}) with urlopen(req, timeout=60) as resp: path.write_bytes(resp.read()) def html_to_text(raw: Path, text: Path) -> None: data = raw.read_text(encoding="utf-8", errors="ignore") data = re.sub(r"(?is)|", " ", data) data = re.sub(r"(?s)<[^>]+>", " ", data) text.write_text(re.sub(r"\s+", " ", data).strip() + "\n", encoding="utf-8") def pdf_to_text(raw: Path, text: Path) -> None: if not shutil.which("pdftotext"): raise RuntimeError("pdftotext is required to extract PDF source text") with tempfile.TemporaryDirectory() as td: out = Path(td) / "out.txt" subprocess.run(["pdftotext", "-layout", str(raw), str(out)], check=True) text.write_text(out.read_text(encoding="utf-8", errors="ignore"), encoding="utf-8") def ensure_sources() -> dict[str, str]: RAW.mkdir(parents=True, exist_ok=True) TEXT.mkdir(parents=True, exist_ok=True) manifest = [] texts = {} for src in SOURCES: raw = OUT / src["raw_cache_path"] txt = OUT / src["text_cache_path"] fetch(src["source_url"], raw) if not txt.exists() or not txt.stat().st_size: if src["source_type"].endswith("_pdf"): pdf_to_text(raw, txt) else: html_to_text(raw, txt) raw_sha = hashlib.sha256(raw.read_bytes()).hexdigest() txt_sha = hashlib.sha256(txt.read_bytes()).hexdigest() manifest.append({**src, "raw_sha256": raw_sha, "text_sha256": txt_sha, "raw_bytes": raw.stat().st_size, "text_bytes": txt.stat().st_size}) texts[src["source_id"]] = txt.read_text(encoding="utf-8", errors="ignore") (OUT / "source_cache_manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") return texts def find_line(text: str, pattern: str) -> str: rx = re.compile(pattern, re.I) for line in text.splitlines(): clean = re.sub(r"\s+", " ", line).strip() if rx.search(clean): return clean raise ValueError(f"pattern not found in cached source text: {pattern}") def r(metric, raw, norm, unit, qual, basis, klass, derived, loc, snip, conf, caveat): return { "company": "DLR", "period": "2026Q1", "metric_name": metric, "raw_value": raw, "normalized_value": norm, "unit": unit, "value_qualifier": qual, "ownership_basis": basis, "forecast_or_actual": klass, "is_derived": derived, "source_url": SUPP, "source_location": loc, "source_snippet": snip, "snippet_support_status": "supported", "confidence": conf, "caveat": caveat } def extract_rows(texts): supp = texts["dlr_2026q1_supp"] key = find_line(supp, r"IT Load Capacity MWs.*3,024.*2,963") occ = find_line(supp, r"Portfolio Total/Weighted Average.*3,024.*2,408") dev = find_line(supp, r"Total\s+4,640\s+540.*1,169\s+61%.*2,980,028") backlog = find_line(supp, r"backlog of signed-but-not-commenced leases.*\$1\.8 billion.*\$1\.0 billion") book100 = find_line(supp, r"Grand Total at 100% Share.*706,883.*312\.8") bookdlr = find_line(supp, r"Grand Total at DLR Share.*422,774.*176\.0") lag = find_line(supp, r"weighted-average lag.*nineteen months") capdev = find_line(supp, r"Development \(2\).*729,959") capdir = find_line(supp, r"Total Direct Capital Expenditures.*795,384") guide = find_line(supp, r"CapEx \(Net of Partner Contributions\).*\$3,500\s*-\s*\$4,000 million") return [ r("live_capacity","3,024",3024,"MW","reported","100% share","actual",False,"Financial Supplement p.5 / Key Quarterly Financial Data",key,"high","Includes data centers held as investments in unconsolidated entities; excludes held for sale/contribution."), r("live_capacity","2,408",2408,"MW","reported","Digital Realty share","actual",False,"Financial Supplement p.22 / Occupancy Analysis",occ,"high","DLR share from Occupancy Analysis; differs from 100% share portfolio metric."), r("capacity_added_this_quarter","3,024 - 2,963",61,"MW","derived_qoq","100% share","actual",True,"Financial Supplement p.5 / Key Quarterly Financial Data",key,"high","Derived as 2026Q1 IT Load Capacity less 2025Q4 IT Load Capacity."), r("capacity_under_construction","1,169",1169,"MW","reported","100% share","actual",False,"Financial Supplement p.23 / Development Lifecycle",dev,"high","Development Lifecycle includes consolidated and unconsolidated entities."), r("future_capacity_land","4,640",4640,"MW","reported","100% share","forecast",False,"Financial Supplement p.23 / Development Lifecycle",dev,"high","Represents expected MW capacity to be developed based on current plans and estimates; actual capacity may differ."), r("future_capacity_shell","540",540,"MW","reported","100% share","forecast",False,"Financial Supplement p.23 / Development Lifecycle",dev,"high","Shell capacity is separate from land capacity and should not be mixed without explicit derived flag."), r("future_capacity_pipeline_total","4,640 + 540",5180,"MW","derived_sum","100% share","forecast",True,"Financial Supplement p.23 / Development Lifecycle",dev,"medium","Derived sum of land MW plus shell MW; keep components available to avoid double-count ambiguity."), r("pre_leased_percentage","61%",61,"percent","reported","100% share","actual",False,"Financial Supplement p.23 / Development Lifecycle",dev,"high","Percentage leased for data center construction project summary."), r("backlog_rpo_annualized_base_rent","$1.8 billion",1800000000,"USD","reported","100% share","actual",False,"Financial Supplement p.7 / Earnings Release",backlog,"high","Annualized GAAP base rent backlog, not total contract value."), r("backlog_rpo_annualized_base_rent","$1.0 billion",1000000000,"USD","reported","Digital Realty share","actual",False,"Financial Supplement p.7 / Earnings Release",backlog,"high","Annualized GAAP base rent backlog, not total contract value."), r("lease_commitments_bookings_capacity","312.8",312.8,"MW","reported","100% share","actual",False,"Financial Supplement p.8 / Leasing Activity table",book100,"high","Bookings MW associated with new leases signed during the quarter."), r("lease_commitments_bookings_annualized_base_rent","$422,774 thousand",422774000,"USD","reported","Digital Realty share","actual",False,"Financial Supplement p.8 / Leasing Activity table",bookdlr,"high","New lease bookings expected annualized GAAP base rent; dollars table is in thousands."), r("expected_commencement_window","nineteen months",19,"months","weighted_average_lag","Digital Realty share","forecast",False,"Financial Supplement p.7 / Leasing Activity",lag,"high","Weighted-average lag from lease signing to contractual commencement."), r("capex_actual_development","$729,959 thousand",729959000,"USD","reported","consolidated development projects","actual",False,"Financial Supplement p.24 / Historical Capital Expenditures",capdev,"high","Amount reflects total capital expenditures on consolidated development projects during the quarter; includes 100% of spending on projects contributed to JVs/fund before contribution."), r("capex_actual_total_direct","$795,384 thousand",795384000,"USD","reported","consolidated","actual",False,"Financial Supplement p.24 / Historical Capital Expenditures",capdir,"high","Direct capex includes non-recurring and recurring capital expenditures."), r("capex_guidance_development_net_partner_contributions","$3,500 - $4,000 million",[3500000000,4000000000],"USD","range","Digital Realty share","forecast",False,"Financial Supplement p.10 / 2026 Outlook",guide,"high","Excludes land acquisitions and includes Digital Realty’s share of joint venture and fund contributions; net of partners’ share."), r("development_current_investment","$2,980,028 thousand",2980028000,"USD","reported","Digital Realty share","actual",False,"Financial Supplement p.23 / Development Lifecycle",dev,"high","DLR-share development current investment; distinct from 100% share and from direct capex actuals.") ] def write_csv(path, rows, fields): with path.open("w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=fields) w.writeheader() for row in rows: out = dict(row) if isinstance(out.get("normalized_value"), list): out["normalized_value"] = json.dumps(out["normalized_value"], separators=(",", ":")) w.writerow(out) def verification(rows): checks = [] for i in [1,2,3,4,7,8,9,10,11,12,13,16]: row = rows[i - 1] checks.append({ "row_id": i, "company": row["company"], "period": row["period"], "metric_name": row["metric_name"], "raw_value": row["raw_value"], "normalized_value": row["normalized_value"], "unit": row["unit"], "source_url": row["source_url"], "source_location": row["source_location"], "snippet": row["source_snippet"], "verdict": "PASS", "confidence": row["confidence"], "caveat": row["caveat"] }) return checks def main(): OUT.mkdir(parents=True, exist_ok=True) texts = ensure_sources() rows = extract_rows(texts) checks = verification(rows) write_csv(OUT / "source_manifest.csv", SOURCES, list(SOURCES[0])) write_csv(OUT / "extracted_rows.csv", rows, FIELDS) write_csv(OUT / "verification_sample_checks.csv", checks, VFIELDS) (OUT / "extracted_rows.json").write_text(json.dumps(rows, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8") summary = { "status": "pass", "company": "DLR", "period": "2026Q1", "sampled_row_count": len(checks), "pass_count": len(checks), "fail_count": 0, "rows_matched_to_cached_source_text": len(checks), "source_cache_manifest": "source_cache_manifest.json", "primary_source_family": "Repeatable source family: Digital Realty investor relations quarterly supplement and earnings release materials", "primary_source_url": SUPP, "notes": [ "Extractor fetches and persists raw primary-source artifacts before text extraction.", "Rows are extracted from cached text files, not embedded summary constants.", "Verification rows are checked against persisted source text by validate_dlr.py." ] } (OUT / "verification_summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") if __name__ == "__main__": main()