#!/usr/bin/env python3
"""Analyze recorded consumer-UI links offline; never contacts an API or website."""
import argparse
import csv
import hashlib
import json
import re
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit


def normalized_url(value):
    parts = urlsplit(value)
    if parts.scheme not in ('http', 'https') or not parts.hostname:
        return None
    if parts.hostname in ('chatgpt.com', 'claude.ai', 'support.claude.com', 'privacy.anthropic.com'):
        return None
    query = [(key, val) for key, val in parse_qsl(parts.query, keep_blank_values=True)
             if not key.lower().startswith('utm_') and key.lower() not in ('gclid', 'fbclid')]
    return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), parts.path or '/', urlencode(query), ''))


def write_csv(path, rows, fields):
    with path.open('w', newline='') as handle:
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()
        writer.writerows(rows)


def analyze(raw_dir, protocol_path, output, require_complete):
    protocol = json.loads(protocol_path.read_text())
    prompts = protocol['prompts']
    expected = [(row, engine, repeat) for row in prompts for engine in ('chatgpt', 'claude') for repeat in (1, 2)]
    runs, links, missing, sets = [], [], [], {}
    for row, engine, repeat in expected:
        run_id = f"{row['prompt_id']}-{engine}-r{repeat}"
        path = raw_dir / f'{run_id}.json'
        if not path.exists():
            missing.append(run_id)
            continue
        raw = json.loads(path.read_text())
        if row['prompt'] not in raw['text'] or len(raw['text']) < len(row['prompt']) + 200:
            raise ValueError(f'Answer missing or mismatched: {run_id}')
        urls = {}
        groups = 0
        for item in raw['links']:
            if not item['text'].strip():
                continue  # Image-only/UI anchors are not text citation links.
            url = normalized_url(item['url'])
            if not url:
                continue
            groups += bool(re.search(r'\+\d+\s*$', item['text']))
            urls.setdefault(url, item['text'].strip())
        sets[(row['prompt_id'], engine, repeat)] = set(urls)
        domains = {urlsplit(url).hostname.removeprefix('www.') for url in urls}
        runs.append(dict(run_id=run_id, prompt_id=row['prompt_id'], category=row['category'], engine=engine,
                         repeat=repeat, captured_at_utc=raw['capturedAt'], visible_unique_urls=len(urls),
                         visible_unique_domains=len(domains), grouped_chips=groups,
                         archive_sha256=hashlib.sha256(path.read_bytes()).hexdigest()))
        for url, label in urls.items():
            links.append(dict(run_id=run_id, prompt_id=row['prompt_id'], category=row['category'], engine=engine,
                              repeat=repeat, url=url, domain=urlsplit(url).hostname.removeprefix('www.'),
                              displayed_label=label, verification='Observed link; destination and claim support not systematically audited'))
    if require_complete and missing:
        raise ValueError(f'{len(missing)} answers not collected: {missing}')
    overlap = []
    for row in prompts:
        for engine in ('chatgpt', 'claude'):
            first, second = sets.get((row['prompt_id'], engine, 1)), sets.get((row['prompt_id'], engine, 2))
            if first is None or second is None:
                continue
            union = first | second
            shared = first & second
            overlap.append(dict(prompt_id=row['prompt_id'], category=row['category'], engine=engine,
                                first_urls=len(first), second_urls=len(second), shared_urls=len(shared),
                                union_urls=len(union), jaccard=round(len(shared) / len(union), 6) if union else None))
    totals = {}
    for engine in ('chatgpt', 'claude'):
        subset = [row for row in runs if row['engine'] == engine]
        source_rows = [row for row in links if row['engine'] == engine]
        pairs = [row for row in overlap if row['engine'] == engine and row['jaccard'] is not None]
        totals[engine] = dict(answers=len(subset), answers_with_visible_links=sum(row['visible_unique_urls'] > 0 for row in subset),
                              answer_url_occurrences=len(source_rows), unique_urls=len({row['url'] for row in source_rows}),
                              unique_domains=len({row['domain'] for row in source_rows}),
                              repeated_prompt_pairs=len(pairs), pairs_with_identical_url_sets=sum(row['jaccard'] == 1 for row in pairs),
                              pairs_with_no_shared_urls=sum(row['jaccard'] == 0 for row in pairs),
                              mean_jaccard=round(sum(row['jaccard'] for row in pairs) / len(pairs), 6) if pairs else None)
    output.mkdir(parents=True, exist_ok=True)
    write_csv(output/'runs.csv', runs, ['run_id','prompt_id','category','engine','repeat','captured_at_utc','visible_unique_urls','visible_unique_domains','grouped_chips','archive_sha256'])
    write_csv(output/'visible-links.csv', links, ['run_id','prompt_id','category','engine','repeat','url','domain','displayed_label','verification'])
    write_csv(output/'repeat-overlap.csv', overlap, ['prompt_id','category','engine','first_urls','second_urls','shared_urls','union_urls','jaccard'])
    write_csv(output/'prompts.csv', [dict(prompt_id=row['prompt_id'],category=row['category'],prompt=row['prompt']+'\n\n'+protocol['suffix']) for row in prompts], ['prompt_id','category','prompt'])
    summary = dict(protocol=protocol['version'], expected_answers=len(expected), collected_answers=len(runs), missing=missing,
                   api_spend_usd=0, totals=totals, definition='Unique visible text-link URL per answer; repeated across answers counts again. Tracking parameters and fragments removed. Collapsed additional citations and image sources excluded.')
    (output/'summary.json').write_text(json.dumps(summary, indent=2)+'\n')
    print(json.dumps(summary, indent=2))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--raw-dir', type=Path, required=True)
    parser.add_argument('--protocol', type=Path, default=Path('docs/research/dtc-citation-study/consumer-protocol.json'))
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--require-complete', action='store_true')
    args = parser.parse_args()
    analyze(args.raw_dir, args.protocol, args.output, args.require_complete)