CareerNexus 职联：岗位能力图谱构建与动态演化分析平台（挑战杯 XH-202621）

- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
2026-08-03 13:15:35 +08:00
commit a10442d78c
78 changed files with 12860 additions and 0 deletions
@@ -0,0 +1,251 @@
+"""新岗位发现：纯 Python TF-IDF（字符 n-gram）向量化 + 凝聚聚类。
+
+流程：无法归入既有岗位的 JD → TF-IDF 向量 → 余弦相似度凝聚聚类 →
+簇同时满足四项指标（支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度）→
+候选新岗位 → 进入“待审核”分区，人工 accept/modify/reject 后才写入正式图谱，审核留痕。
+阈值与《项目方案》第七章对齐：≥15 条 JD / ≥5 家企业 / 增长率 ≥50% / 差异度 ≥0.4。
+"""
+from __future__ import annotations
+
+import logging
+import math
+from collections import Counter
+from pathlib import Path
+
+from app.graph.tkg import now_str, quarter_of
+from app.models import ExtractionResult, JDRecord, JobDefinition, ReviewRecord
+
+logger = logging.getLogger(__name__)
+
+# 候选新岗位阈值（与方案文档一致）
+MIN_CLUSTER_SIZE = 15     # 支撑 JD 数
+MIN_COMPANIES = 5         # 独立企业数（防单一企业造词）
+MIN_GROWTH = 1.5          # 近期增长率（最近一年 JD 数 / 前一年，≥50% 增长）
+MIN_DISTINCTNESS = 0.4    # 与既有岗位差异度：1 - 与最近既有岗位质心的余弦相似度
+MERGE_SIM = 0.50          # 凝聚合并的质心余弦相似度阈值
+
+
+# ---------- TF-IDF（字符 2/3-gram） ----------
+def _char_ngrams(text: str, ns=(2, 3)) -> list[str]:
+    text = "".join(text.split())
+    out: list[str] = []
+    for n in ns:
+        out.extend(text[i:i + n] for i in range(max(0, len(text) - n + 1)))
+    return out
+
+
+class TfidfVectorizer:
+    """极简 TF-IDF：字符 n-gram 词表 + IDF 加权 + L2 归一。"""
+
+    def __init__(self):
+        self.idf: dict[str, float] = {}
+
+    def fit(self, docs: list[str]) -> "TfidfVectorizer":
+        df: Counter = Counter()
+        for d in docs:
+            df.update(set(_char_ngrams(d)))
+        n = max(1, len(docs))
+        self.idf = {t: math.log((n + 1) / (c + 1)) + 1.0 for t, c in df.items()}
+        return self
+
+    def transform(self, doc: str) -> dict[str, float]:
+        tf: Counter = Counter(_char_ngrams(doc))
+        vec = {t: (c / max(1, sum(tf.values()))) * self.idf.get(t, 1.0)
+               for t, c in tf.items()}
+        norm = math.sqrt(sum(v * v for v in vec.values())) or 1.0
+        return {t: v / norm for t, v in vec.items()}
+
+
+def cosine(a: dict[str, float], b: dict[str, float]) -> float:
+    """稀疏向量余弦相似度。"""
+    if len(a) > len(b):
+        a, b = b, a
+    return sum(v * b.get(t, 0.0) for t, v in a.items())
+
+
+def _centroid(vecs: list[dict[str, float]]) -> dict[str, float]:
+    c: dict[str, float] = {}
+    for v in vecs:
+        for t, x in v.items():
+            c[t] = c.get(t, 0.0) + x
+    n = max(1, len(vecs))
+    return {t: x / n for t, x in c.items()}
+
+
+def agglomerative(vecs: list[dict[str, float]], merge_sim: float = MERGE_SIM) -> list[list[int]]:
+    """凝聚聚类（平均链接的贪心实现）：每轮合并质心最相似的一对簇。"""
+    clusters: list[list[int]] = [[i] for i in range(len(vecs))]
+    centroids = [_centroid([v]) for v in vecs]
+    while True:
+        best, pair = merge_sim, None
+        for i in range(len(clusters)):
+            for j in range(i + 1, len(clusters)):
+                s = cosine(centroids[i], centroids[j])
+                if s > best:
+                    best, pair = s, (i, j)
+        if pair is None:
+            return clusters
+        i, j = pair
+        clusters[i].extend(clusters[j])
+        centroids[i] = _centroid([vecs[k] for k in clusters[i]])
+        del clusters[j], centroids[j]
+
+
+# ---------- 候选生成与审核 ----------
+def discover_new_jobs(
+    records: list[JDRecord],
+    extractions: dict[str, ExtractionResult],
+    known_titles: set[str],
+    min_size: int = MIN_CLUSTER_SIZE,
+    min_companies: int = MIN_COMPANIES,
+    min_growth: float = MIN_GROWTH,
+    min_distinctness: float = MIN_DISTINCTNESS,
+) -> list[JobDefinition]:
+    """从未知岗位 JD 中聚类发现候选新岗位（status=pending）。
+
+    四指标判定：支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度。
+    """
+    unknown = [r for r in records
+               if not r.is_noise and not r.is_duplicate
+               and r.jid in extractions
+               and extractions[r.jid].job_title not in known_titles]
+    if len(unknown) < min_size:
+        logger.info("未知岗位 JD 不足 %d 条，跳过新岗位发现", min_size)
+        return []
+
+    # 岗位名称是强信号：重复十次提升其在 TF-IDF 中的权重；
+    # 再叠加抽取出的必备技能（重复三次），保证长文本、多模板句式的 JD 仍能按岗位聚拢
+    def doc_of(r: JDRecord) -> str:
+        e = extractions[r.jid]
+        return f"{r.job_title} " * 10 + " ".join(e.required_skills * 3) + " " + r.text
+
+    docs = [doc_of(r) for r in unknown]
+    vectorizer = TfidfVectorizer().fit(docs)
+    vecs = [vectorizer.transform(d) for d in docs]
+    clusters = agglomerative(vecs)
+
+    # 既有岗位质心（差异度基准）：按已知标题聚合并向量化
+    known_docs: dict[str, list[str]] = {}
+    for r in records:
+        e = extractions.get(r.jid)
+        if e and e.job_title in known_titles and not r.is_noise and not r.is_duplicate:
+            known_docs.setdefault(e.job_title, []).append(doc_of(r))
+    known_centroids = {t: _centroid([vectorizer.transform(d) for d in ds])
+                       for t, ds in known_docs.items()}
+
+    candidates: list[JobDefinition] = []
+    for members in clusters:
+        if len(members) < min_size:
+            continue
+        group = [unknown[i] for i in members]
+        companies = {r.company for r in group}
+        if len(companies) < min_companies:
+            continue
+        years = Counter(r.publish_date.year for r in group)
+        latest_year = max(years)
+        prev_year_n = years.get(latest_year - 1, 0)
+        # 前一年没有基数视为“从无到有”，增长率视为达标
+        growth = years[latest_year] / prev_year_n if prev_year_n else float("inf")
+        if growth < min_growth:
+            continue
+        # 差异度：与最近既有岗位质心的距离，过近说明只是既有岗位的变体
+        cvec = _centroid([vecs[i] for i in members])
+        max_sim = max((cosine(cvec, kc) for kc in known_centroids.values()), default=0.0)
+        distinctness = 1.0 - max_sim
+        if distinctness < min_distinctness:
+            logger.info("簇与既有岗位差异度不足（%.2f < %.2f），不列为新岗位", distinctness, min_distinctness)
+            continue
+        # 簇内最常见的抽取标题作为候选岗位名；技能取簇内高频
+        titles = Counter(extractions[r.jid].job_title for r in group)
+        title = titles.most_common(1)[0][0]
+        req = Counter(s for r in group for s in extractions[r.jid].required_skills)
+        pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
+        scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
+        ind = Counter(r.industry for r in group if r.industry)
+        n = len(group)
+        candidates.append(JobDefinition(
+            job_id=f"cand_{title}",
+            title=title,
+            category="新岗位候选",
+            industries=[s for s, _ in ind.most_common(2)],
+            required_skills=[s for s, c in req.most_common() if c >= n * 0.4],
+            preferred_skills=[s for s, c in pref.most_common() if c >= n * 0.25],
+            scenarios=[s for s, _ in scen.most_common(5)],
+            status="pending",
+            source_count=n,
+            company_count=len(companies),
+            support_source_ids=[r.source_id for r in group],
+            support_windows=sorted({quarter_of(r.publish_date) for r in group}),
+        ))
+        logger.info("候选新岗位：%s（支撑 %d 条 JD / %d 家企业 / 增长率 %.2f / 差异度 %.2f）",
+                    title, n, len(companies), growth, distinctness)
+    return candidates
+
+
+# ---------- 待审核分区与审核留痕（SQLite 状态库，见 app/storage/db.py） ----------
+def save_pending(candidates: list[JobDefinition], path: str | Path) -> None:
+    """整体覆写待审核分区。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。"""
+    from app.storage import db
+    keep = set()
+    for c in candidates:
+        db.save_candidate(path, c.model_dump())
+        keep.add(c.job_id)
+    for old in db.list_candidates(path):
+        if old["job_id"] not in keep:
+            db.delete_candidate(path, old["job_id"])
+
+
+def load_pending(path: str | Path) -> list[JobDefinition]:
+    """读取全部待审核候选。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。"""
+    from app.storage import db
+    return [JobDefinition(**d) for d in db.list_candidates(path)]
+
+
+def apply_review(
+    pending_path: str | Path,
+    audit_path: str | Path,
+    candidate_id: str,
+    action: str,
+    operator: str,
+    reason: str = "",
+    modify: dict | None = None,
+) -> tuple[JobDefinition | None, ReviewRecord]:
+    """人工审核：accept/modify/reject。返回 (通过的岗位定义或 None, 审核留痕)。
+
+    ``pending_path`` 为 SQLite 状态库路径；``audit_path`` 为旧 audit.jsonl 路径，
+    仅在状态库首次打开时作为一次性迁移种子使用（原文件保留）。
+    """
+    from app.storage import db
+    db.connect(pending_path, legacy_audit=audit_path).close()   # 首次打开触发迁移（幂等）
+    candidates = load_pending(pending_path)
+    target = next((c for c in candidates if c.job_id == candidate_id), None)
+    if target is None:
+        raise KeyError(f"候选不存在：{candidate_id}")
+    before = target.model_dump()
+    accepted: JobDefinition | None = None
+    after: dict = {}
+    if action == "accept":
+        target.status = "active"
+        accepted = target
+        after = target.model_dump()
+    elif action == "modify":
+        if modify:
+            target = target.model_copy(update=modify)
+        target.status = "active"
+        accepted = target
+        after = target.model_dump()
+    elif action == "reject":
+        target.status = "rejected"
+        after = target.model_dump()
+        # 驳回：若该候选曾入库（覆写层有记录），一并清理，避免重建后复活
+        db.delete_override(pending_path, candidate_id)
+    else:
+        raise ValueError(f"未知审核动作：{action}")
+
+    record = ReviewRecord(candidate_id=candidate_id, action=action, operator=operator,
+                          reason=reason, before=before, after=after, time=now_str())
+    db.append_audit(pending_path, record.model_dump())
+    # 已审核的候选从待审核分区移除
+    db.delete_candidate(pending_path, candidate_id)
+    logger.info("审核完成：%s -> %s（操作人 %s）", candidate_id, action, operator)
+    return accepted, record