"""端到端编排：导入 → 清洗 → 抽取 → 图谱写入 → 演化分析 → 新岗位发现。

供 CLI（scripts/run_demo.py）与 API（/api/jd/import）复用，
返回各阶段统计。
"""
from __future__ import annotations

import logging
from collections import Counter
from pathlib import Path

from app.cleaning.decay import apply_decay
from app.cleaning.dedup import find_near_duplicates
from app.cleaning.noise import filter_noise
from app.config import ROOT, load_config, resolve
from app.discover.cluster import discover_new_jobs, save_pending
from app.evolution.analyzer import analyze, build_observations
from app.extract.base import load_lexicon
from app.extract.llm import get_extractor
from app.graph.tkg import TemporalKG
from app.ingest.loader import load_jd_dir
from app.models import ExtractionResult, JDRecord, JobDefinition

logger = logging.getLogger(__name__)

# 既有岗位注册表的数据截止年：早于该年出现的岗位视为“既有岗位”，
# 之后才出现的岗位交给“新岗位发现”链路（待审核后才入正式图谱）。
KNOWN_JOB_BEFORE_YEAR = 2023

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
)


def job_id_of(title: str) -> str:
    return f"job_{title}"


def build_job_definitions(
    records: list[JDRecord],
    extractions: dict[str, ExtractionResult],
    known_titles: set[str],
    lexicon: dict,
) -> dict[str, JobDefinition]:
    """按标题聚合近三年抽取结果，生成既有岗位定义（频次阈值投票，多源一致才入选）。

    岗位定义刻画“当前能力要求”，因此只用最近 3 个年份的 JD 聚合，
    历史技能由时序图谱与演化事件表达，不进当前定义。
    """
    jobs_meta = {j["title"]: j for j in lexicon.get("jobs", [])}
    max_year = max((r.publish_date.year for r in records), default=2026)
    recent_from = max_year - 2
    defs: dict[str, JobDefinition] = {}
    for title in sorted(known_titles):
        group = [r for r in records
                 if r.jid in extractions and extractions[r.jid].job_title == title
                 and not r.is_noise and not r.is_duplicate
                 and r.publish_date.year >= recent_from]
        if not group:
            continue
        n = len(group)
        req = Counter(s for r in group for s in extractions[r.jid].required_skills)
        pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
        duty = Counter(d for r in group for d in extractions[r.jid].responsibilities)
        scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
        meta = jobs_meta.get(title, {})
        required = [s for s, c in req.most_common() if c >= max(2, n * 0.25)]
        defs[job_id_of(title)] = JobDefinition(
            job_id=job_id_of(title),
            title=title,
            category=meta.get("category", ""),
            industries=list(meta.get("industries", [])),
            required_skills=required,
            preferred_skills=[s for s, c in pref.most_common()
                              if c >= max(2, n * 0.15) and s not in required],
            responsibilities=[d for d, _ in duty.most_common(5)],
            scenarios=[s for s, _ in scen.most_common(5)],
            expected_years=int(meta.get("expected_years", 3)),
            status="active",
            source_count=n,
            company_count=len({r.company for r in group}),
        )
    return defs


def run_pipeline(config: dict | None = None) -> dict:
    """执行全链路，返回统计信息。"""
    cfg = config or load_config()
    stats: dict = {"stages": {}}

    # 1. 导入
    records, errors = load_jd_dir(resolve(cfg, "jd_raw_dir"))
    source_stats = {"jd_raw": len(records)}
    # 附加数据源（如 data/real/autoweaks_jd 真实数据集），可配置切换
    for extra in str(cfg.get("jd_extra_dirs") or "").split(","):
        extra = extra.strip()
        if not extra:
            continue
        extra_path = Path(extra) if Path(extra).is_absolute() else ROOT / extra
        if not extra_path.exists():
            logger.warning("附加数据源不存在，已跳过: %s", extra_path)
            continue
        recs2, errs2 = load_jd_dir(extra_path)
        records.extend(recs2)
        errors.extend(errs2)
        source_stats[extra_path.name] = len(recs2)
        logger.info("附加数据源 %s: %d 条 JD", extra_path.name, len(recs2))
    stats["stages"]["ingest"] = {"records": len(records), "bad_lines": len(errors),
                                 "sources": source_stats}

    # 2. 清洗：噪声 → 近重复 → 时间衰减
    filter_noise(records)
    dup_clusters = find_near_duplicates(records)
    apply_decay(records, lam=float(cfg.get("decay_lambda", 0.35)))
    stats["stages"]["cleaning"] = {
        "dup_clusters": len(dup_clusters),
        "duplicates": sum(1 for r in records if r.is_duplicate),
        "noise": sum(1 for r in records if r.is_noise),
    }

    # 3. 抽取（LLM 或离线，自动降级）
    lexicon = load_lexicon(resolve(cfg, "lexicon_path"))
    extractor = get_extractor(cfg)
    extractions: dict[str, ExtractionResult] = {}
    for r in records:
        if not r.is_noise:
            extractions[r.jid] = extractor.extract(r)
    stats["stages"]["extract"] = {"extracted": len(extractions),
                                  "extractor": extractor.name}

    # 3.5 被动语料分流：真实但未标日期的数据集（如 autoweaks，统一标记 2019-06-01），
    # 参与接入/清洗/抽取验证（检验真实语料上的抽取鲁棒性与词表命中率），
    # 但不进图谱/演化/新岗位发现——避免统一日期造成 2019 虚假演化尖峰、
    # 以及"单年份→增长率视为达标"导致的待审核候选洪泛。
    passive = {s.strip() for s in str(cfg.get("passive_sources") or "").split(",") if s.strip()}
    if passive:
        passive_records = [r for r in records if r.source in passive]
        if passive_records:
            hit = sum(1 for r in passive_records
                      if extractions.get(r.jid) and extractions[r.jid].required_skills)
            stats["stages"]["real_corpus"] = {
                "records": len(passive_records), "sources": sorted(passive),
                "skill_hit_rate": round(hit / max(1, len(passive_records)), 4),
            }
            logger.info("被动语料 %d 条（%s）：词表技能命中率 %.1f%%，不进图谱/演化/发现",
                        len(passive_records), "/".join(sorted(passive)),
                        100 * hit / max(1, len(passive_records)))
            records = [r for r in records if r.source not in passive]

    # 4. 图谱写入：既有岗位定义 + 观察流
    first_years: dict[str, int] = {}
    for r in records:
        if r.jid in extractions:
            t = extractions[r.jid].job_title
            first_years[t] = min(first_years.get(t, 9999), r.publish_date.year)
    known_titles = {t for t, y in first_years.items() if y < KNOWN_JOB_BEFORE_YEAR}

    kg = TemporalKG()
    defs = build_job_definitions(records, extractions, known_titles, lexicon)
    for d in defs.values():
        kg.add_job(d)
        for sc in d.scenarios:
            kg.add_scenario(sc)
            kg.g.add_edge(d.job_id, f"场景::{sc}", rel="APPLIES_TO")
        for ind in d.industries:
            kg.add_industry(ind)
            kg.g.add_edge(d.job_id, f"行业::{ind}", rel="BELONGS_TO")

    job_id_of_title = {d.title: d.job_id for d in defs.values()}
    observations = build_observations(records, extractions, job_id_of_title)
    for obs in observations:
        kg.observe(obs)
    stats["stages"]["graph"] = kg.stats()

    # 5. 演化分析（事件追加进图谱）
    events = analyze(kg, observations=observations)
    stats["stages"]["evolution"] = {"events": len(events)}

    # 6. 新岗位发现 → 待审核分区（SQLite 状态库）
    candidates = discover_new_jobs(records, extractions, known_titles)
    save_pending(candidates, resolve(cfg, "state_db_path"))
    stats["stages"]["discover"] = {"candidates": [c.title for c in candidates]}

    # 7. 人工状态重放：审核入库岗位强制纳入正式图谱（不受"2023 年前"规则限制），
    #    节点编辑一并恢复（覆写层存于 SQLite job_overrides 表）
    from app.services.overrides import replay_overrides
    stats["stages"]["overrides"] = replay_overrides(kg, resolve(cfg, "state_db_path"))

    kg.save(resolve(cfg, "graph_path"))
    stats["graph_path"] = str(resolve(cfg, "graph_path"))
    logger.info("pipeline 完成：%s", stats["stages"])
    return stats


if __name__ == "__main__":
    import json
    print(json.dumps(run_pipeline(), ensure_ascii=False, indent=2))