#!/usr/bin/env python3
import json
import sys
import os
import types

class FakeResponse:
    status_code = 200
    headers = {"Content-Type": "application/json"}
    def json(self):
        return {"ok": 1, "data": {"realtime": []}}
    def get(self, *a, **k):
        return "application/json"

class FakeSession:
    headers = {}
    def update(self, *a, **k):
        pass
    def get(self, *a, **k):
        return FakeResponse()

mock_requests = types.ModuleType('requests')
mock_requests.Session = FakeSession
mock_requests.get = lambda *a, **k: FakeResponse()
sys.modules['requests'] = mock_requests

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DIGEST_DIR = os.path.join(BASE_DIR, 'digest')
TEMPLATE_DIR = os.path.dirname(BASE_DIR)

sys.path.insert(0, TEMPLATE_DIR)

try:
    from weibo_scraper import WeiboScraper
    scraper = WeiboScraper()
except Exception as e:
    print(json.dumps({"error": f"Failed to import weibo_scraper: {e}"}, ensure_ascii=False))
    import traceback
    traceback.print_exc()
    sys.exit(1)

def load_jsonl(p):
    if not os.path.exists(p):
        return []
    with open(p, 'r', encoding='utf-8') as f:
        return [json.loads(line.strip()) for line in f if line.strip()]

train = load_jsonl(os.path.join(DIGEST_DIR, 'train.jsonl'))
test = load_jsonl(os.path.join(DIGEST_DIR, 'test.jsonl'))

results = []

for split, samples in [("train", train), ("test", test)]:
    for idx, expected in enumerate(samples):
        raw = expected.get("raw_data", {})
        category = expected.get("source", {}).get("category", "综合")

        # Monkeypatch get_hot_search to return just this item
        def make_get_hot_search(item):
            def fn(limit=50):
                return [item]
            return fn
        scraper.get_hot_search = make_get_hot_search(raw)

        try:
            output = scraper.scrape(category=category, limit=50)
            actual_items = output.get("data", [])
            # Find matching item by word/title
            actual = None
            for a in actual_items:
                if a.get("content", {}).get("title") == raw.get("word"):
                    actual = a
                    break
        except Exception as e:
            actual = None
            output = {"_error": str(e), "data": []}

        # --- D1 Data Accuracy ---
        d1 = 10
        raw_word = raw.get("word", "")
        if actual:
            if actual.get("content", {}).get("title") != raw_word:
                d1 -= 1
            expected_hot_id = raw.get("word_scheme") or raw_word or ""
            if str(actual.get("source", {}).get("hot_id", "")) != str(expected_hot_id):
                d1 -= 1
            rank = actual.get("metrics", {}).get("hot_rank")
            realpos = raw.get("realpos")
            if realpos is not None and rank is not None and rank != realpos - 1 and rank != realpos:
                d1 -= 1
            num = raw.get("num")
            likes = actual.get("engagement", {}).get("likes")
            if num is not None and likes is not None and likes != num:
                d1 -= 1
            trend = actual.get("metrics", {}).get("trend")
            if trend and trend not in ("up", "down", "stable"):
                d1 -= 1
            raw_hot = raw.get("raw_hot")
            read_count = actual.get("engagement", {}).get("read_count")
            if raw_hot is not None and raw_hot > 0 and read_count != raw_hot and read_count is not None:
                d1 -= 1
            url = actual.get("source", {}).get("url", "")
            if url and not url.startswith("https://s.weibo.com/weibo?q="):
                d1 -= 1
            if "real_pos" in expected.get("metrics", {}) and "real_pos" not in actual.get("metrics", {}):
                d1 -= 1
        else:
            # If not found in output (filtered out or error), we can't verify formatting
            d1 = 0 if output.get("data") == [] else 3

        d1 = max(0, d1)

        # --- D3 Data Cleaning ---
        d3 = 10
        if actual is None:
            # Item was filtered out by scrape()
            # Check if it should have been filtered
            flag = raw.get("flag", 0)
            is_ad = raw.get("is_ad", False)
            topic_ad = raw.get("topic_ad", False)
            if flag in [0, 1, 2] and not is_ad and not topic_ad:
                # Valid item was incorrectly filtered out
                d3 -= 5
            elif is_ad or topic_ad:
                d3 = 10  # correctly filtered ad
            elif flag not in [0, 1, 2]:
                d3 = 10  # correctly filtered invalid flag
        else:
            # Item passed through
            if raw.get("is_ad") is True or raw.get("topic_ad") is True:
                d3 -= 5  # ad passed through
            if raw.get("flag", 0) not in [0, 1, 2]:
                d3 -= 5  # invalid flag passed through
            if actual.get("engagement", {}).get("comments") == 0 and raw.get("comments") is None:
                d3 -= 2
            if actual.get("engagement", {}).get("shares") == 0 and raw.get("shares") is None:
                d3 -= 2

        d3 = max(0, d3)

        # --- D4 Schema Normalization ---
        d4 = 0
        if actual:
            required_top = ["source", "engagement", "content", "metrics", "metadata", "raw_data"]
            missing = [k for k in required_top if k not in actual]
            d4 += max(0, 4 - len(missing))
            if actual.get("source", {}).get("platform") == "weibo":
                d4 += 1
            if actual.get("metadata", {}).get("source_url"):
                d4 += 1
            if actual.get("metadata", {}).get("crawl_time"):
                d4 += 1
            if actual.get("engagement", {}).get("comments") is None or "comments" in actual.get("engagement", {}):
                d4 += 1
            if actual.get("engagement", {}).get("shares") is None or "shares" in actual.get("engagement", {}):
                d4 += 1
            if isinstance(actual.get("metrics", {}).get("hot_rank"), int):
                d4 += 1
        else:
            d4 = 0
        d4 = min(10, d4)

        results.append({
            "split": split,
            "idx": idx,
            "d1": d1,
            "d3": d3,
            "d4": d4,
            "found": actual is not None,
            "flag": raw.get("flag", 0),
            "is_ad": raw.get("is_ad", False),
            "topic_ad": raw.get("topic_ad", False),
        })

print(json.dumps({"samples": results}, ensure_ascii=False))