#!/usr/bin/env python3
"""Inventory and download every public MACStats Excel source.

The archive page exposes one ZIP per report edition.  The current compiled
page exposes one publication page per exhibit, and each publication page
contains a direct XLS/XLSX link.  This script records both structures in a
single manifest before downloading anything, then verifies every file with a
SHA-256 checksum.
"""

from __future__ import annotations

import argparse
import concurrent.futures
import csv
import hashlib
import html
import http.client
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import asdict, dataclass
from html.parser import HTMLParser
from pathlib import Path
from typing import Iterable


ARCHIVE_URL = "https://www.macpac.gov/publication/macstats-archive/"
CURRENT_URL = "https://www.macpac.gov/publication/macstats-compiled/"
USER_AGENT = "Mozilla/5.0 (compatible; MACStatsTidy/1.0; research data pipeline)"


class LinkParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__()
        self.links: list[dict[str, str]] = []
        self._href: str | None = None
        self._text: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag.lower() == "a":
            self._href = dict(attrs).get("href")
            self._text = []

    def handle_data(self, data: str) -> None:
        if self._href is not None:
            self._text.append(data)

    def handle_endtag(self, tag: str) -> None:
        if tag.lower() == "a" and self._href is not None:
            text = " ".join(" ".join(self._text).split())
            self.links.append({"text": html.unescape(text), "href": self._href})
            self._href = None
            self._text = []


@dataclass
class SourceRecord:
    source_kind: str
    report_edition: str
    publication_date: str
    exhibit_number: str
    exhibit_title: str
    landing_page_url: str
    download_url: str
    local_path: str
    file_name: str
    excel_available: bool = True
    file_size_bytes: int = 0
    sha256: str = ""
    http_last_modified: str = ""


def fetch(url: str, attempts: int = 4) -> tuple[bytes, dict[str, str]]:
    parsed = urllib.parse.urlsplit(url)
    url = urllib.parse.urlunsplit(
        (parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path, safe="/%:@"), parsed.query, parsed.fragment)
    )
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    last_error: Exception | None = None
    for attempt in range(attempts):
        try:
            with urllib.request.urlopen(req, timeout=90) as response:
                body = response.read()
                headers = {k.lower(): v for k, v in response.headers.items()}
                return body, headers
        except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException) as exc:
            last_error = exc
            if attempt + 1 < attempts:
                time.sleep(2**attempt)
    raise RuntimeError(f"Failed to download {url}: {last_error}")


def parse_links(page: bytes) -> list[dict[str, str]]:
    parser = LinkParser()
    parser.feed(page.decode("utf-8", errors="replace"))
    return parser.links


def strip_tags(page: bytes) -> str:
    text = re.sub(r"<script\b[^>]*>.*?</script>", " ", page.decode("utf-8", errors="replace"), flags=re.I | re.S)
    text = re.sub(r"<style\b[^>]*>.*?</style>", " ", text, flags=re.I | re.S)
    text = re.sub(r"<[^>]+>", " ", text)
    return " ".join(html.unescape(text).split())


def clean_file_name(url: str) -> str:
    return urllib.parse.unquote(Path(urllib.parse.urlparse(url).path).name)


def safe_segment(value: str) -> str:
    value = value.lower().strip()
    value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
    return value or "unknown"


def inventory_archive(root: Path) -> list[SourceRecord]:
    page, _ = fetch(ARCHIVE_URL)
    records: list[SourceRecord] = []
    for link in parse_links(page):
        href = urllib.parse.urljoin(ARCHIVE_URL, link["href"])
        if not href.lower().split("?")[0].endswith(".zip"):
            continue
        file_name = clean_file_name(href)
        match = re.search(r"(20\d{2})(?:[-_ ](0[1-9]|1[0-2]))?", file_name)
        if not match:
            raise ValueError(f"Cannot determine report edition from {file_name}")
        year, month = match.group(1), match.group(2)
        if month:
            report_edition = f"{year}-{month}"
            publication_date = report_edition
        else:
            report_edition = f"{year}-12"
            publication_date = report_edition
        relative = Path("archive") / report_edition / file_name
        records.append(
            SourceRecord(
                source_kind="archive_zip",
                report_edition=report_edition,
                publication_date=publication_date,
                exhibit_number="",
                exhibit_title="",
                landing_page_url=ARCHIVE_URL,
                download_url=href,
                local_path=str(root / relative),
                file_name=file_name,
            )
        )
    return sorted(records, key=lambda r: r.report_edition)


def parse_publication_date(page_text: str) -> str:
    match = re.search(r"Published In\s+([A-Z][a-z]+\s+20\d{2})", page_text)
    return match.group(1) if match else ""


def inventory_current(root: Path) -> list[SourceRecord]:
    page, _ = fetch(CURRENT_URL)
    records: list[SourceRecord] = []
    exhibit_links = []
    for link in parse_links(page):
        match = re.match(r"EXHIBIT\s+(\d+)\.\s*(.+)", link["text"], flags=re.I)
        if match:
            exhibit_links.append((int(match.group(1)), match.group(2).strip(), urllib.parse.urljoin(CURRENT_URL, link["href"])))

    def inspect_exhibit(item: tuple[int, str, str]) -> SourceRecord:
        exhibit_no, compiled_title, landing_url = item
        exhibit_page, _ = fetch(landing_url)
        page_text = strip_tags(exhibit_page)
        publication_date = parse_publication_date(page_text)
        download_links = [
            urllib.parse.urljoin(landing_url, link["href"])
            for link in parse_links(exhibit_page)
            if re.search(r"\.xlsx?(?:\?|$)", link["href"], flags=re.I)
        ]
        if len(download_links) > 1:
            raise RuntimeError(f"Expected at most one Excel link for exhibit {exhibit_no}, found {len(download_links)}: {landing_url}")
        download_url = download_links[0] if download_links else ""
        file_name = clean_file_name(download_url) if download_url else ""
        edition_match = re.search(r"(20\d{2})", publication_date)
        report_year = edition_match.group(1) if edition_match else "current"
        report_edition = f"{report_year}-02" if report_year != "current" else "current"
        relative = Path("current") / report_edition / f"exhibit-{exhibit_no:02d}" / file_name if file_name else Path()
        return SourceRecord(
            source_kind="current_exhibit",
            report_edition=report_edition,
            publication_date=publication_date,
            exhibit_number=str(exhibit_no),
            exhibit_title=compiled_title,
            landing_page_url=landing_url,
            download_url=download_url,
            local_path=str(root / relative) if file_name else "",
            file_name=file_name,
            excel_available=bool(download_url),
        )

    with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
        future_map = {pool.submit(inspect_exhibit, item): item[0] for item in sorted(exhibit_links)}
        for completed, future in enumerate(concurrent.futures.as_completed(future_map), start=1):
            record = future.result()
            records.append(record)
            print(f"Inventoried current exhibit {record.exhibit_number} ({completed}/{len(future_map)})", flush=True)
    return sorted(records, key=lambda r: int(r.exhibit_number))


def download_record(record: SourceRecord, project_root: Path, force: bool = False) -> None:
    if not record.excel_available:
        return
    path = Path(record.local_path)
    if not path.is_absolute():
        path = project_root / path
    path.parent.mkdir(parents=True, exist_ok=True)
    headers: dict[str, str] = {}
    if force or not path.exists() or path.stat().st_size == 0:
        body, headers = fetch(record.download_url)
        tmp = path.with_suffix(path.suffix + ".part")
        tmp.write_bytes(body)
        tmp.replace(path)
    body = path.read_bytes()
    record.file_size_bytes = len(body)
    record.sha256 = hashlib.sha256(body).hexdigest()
    record.http_last_modified = headers.get("last-modified", "")


def write_manifest(records: Iterable[SourceRecord], csv_path: Path, json_path: Path) -> None:
    rows = [asdict(record) for record in records]
    csv_path.parent.mkdir(parents=True, exist_ok=True)
    with csv_path.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)
    json_path.write_text(json.dumps(rows, indent=2), encoding="utf-8")


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument("--project-root", type=Path, default=Path(__file__).resolve().parents[1])
    parser.add_argument("--inventory-only", action="store_true")
    parser.add_argument("--force", action="store_true")
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    download_root = Path("data") / "raw" / "downloads"
    records = inventory_archive(download_root) + inventory_current(download_root)
    if not args.inventory_only:
        downloadable = [record for record in records if record.excel_available]
        with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
            future_map = {pool.submit(download_record, record, args.project_root, args.force): record for record in downloadable}
            for index, future in enumerate(concurrent.futures.as_completed(future_map), start=1):
                record = future_map[future]
                future.result()
                print(f"Downloaded [{index:02d}/{len(downloadable):02d}] {record.report_edition} {record.file_name}", flush=True)
    manifest_dir = args.project_root / "data" / "raw"
    write_manifest(records, manifest_dir / "source_manifest.csv", manifest_dir / "source_manifest.json")
    print(f"Wrote {len(records)} source records")


if __name__ == "__main__":
    main()
