"""新岗位发现：纯 Python TF-IDF（字符 n-gram）向量化 + 凝聚聚类。 流程：无法归入既有岗位的 JD → TF-IDF 向量 → 余弦相似度凝聚聚类 → 簇同时满足四项指标（支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度）→ 候选新岗位 → 进入“待审核”分区，人工 accept/modify/reject 后才写入正式图谱，审核留痕。 阈值与《项目方案》第七章对齐：≥15 条 JD / ≥5 家企业 / 增长率 ≥50% / 差异度 ≥0.4。 """ from __future__ import annotations import logging import math from collections import Counter from pathlib import Path from app.graph.tkg import now_str, quarter_of from app.models import ExtractionResult, JDRecord, JobDefinition, ReviewRecord logger = logging.getLogger(__name__) # 候选新岗位阈值（与方案文档一致） MIN_CLUSTER_SIZE = 15 # 支撑 JD 数 MIN_COMPANIES = 5 # 独立企业数（防单一企业造词） MIN_GROWTH = 1.5 # 近期增长率（最近一年 JD 数 / 前一年，≥50% 增长） MIN_DISTINCTNESS = 0.4 # 与既有岗位差异度：1 - 与最近既有岗位质心的余弦相似度 MERGE_SIM = 0.50 # 凝聚合并的质心余弦相似度阈值 # ---------- TF-IDF（字符 2/3-gram） ---------- def _char_ngrams(text: str, ns=(2, 3)) -> list[str]: text = "".join(text.split()) out: list[str] = [] for n in ns: out.extend(text[i:i + n] for i in range(max(0, len(text) - n + 1))) return out class TfidfVectorizer: """极简 TF-IDF：字符 n-gram 词表 + IDF 加权 + L2 归一。""" def __init__(self): self.idf: dict[str, float] = {} def fit(self, docs: list[str]) -> "TfidfVectorizer": df: Counter = Counter() for d in docs: df.update(set(_char_ngrams(d))) n = max(1, len(docs)) self.idf = {t: math.log((n + 1) / (c + 1)) + 1.0 for t, c in df.items()} return self def transform(self, doc: str) -> dict[str, float]: tf: Counter = Counter(_char_ngrams(doc)) vec = {t: (c / max(1, sum(tf.values()))) * self.idf.get(t, 1.0) for t, c in tf.items()} norm = math.sqrt(sum(v * v for v in vec.values())) or 1.0 return {t: v / norm for t, v in vec.items()} def cosine(a: dict[str, float], b: dict[str, float]) -> float: """稀疏向量余弦相似度。""" if len(a) > len(b): a, b = b, a return sum(v * b.get(t, 0.0) for t, v in a.items()) def _centroid(vecs: list[dict[str, float]]) -> dict[str, float]: c: dict[str, float] = {} for v in vecs: for t, x in v.items(): c[t] = c.get(t, 0.0) + x n = max(1, len(vecs)) return {t: x / n for t, x in c.items()} def agglomerative(vecs: list[dict[str, float]], merge_sim: float = MERGE_SIM) -> list[list[int]]: """凝聚聚类（平均链接的贪心实现）：每轮合并质心最相似的一对簇。""" clusters: list[list[int]] = [[i] for i in range(len(vecs))] centroids = [_centroid([v]) for v in vecs] while True: best, pair = merge_sim, None for i in range(len(clusters)): for j in range(i + 1, len(clusters)): s = cosine(centroids[i], centroids[j]) if s > best: best, pair = s, (i, j) if pair is None: return clusters i, j = pair clusters[i].extend(clusters[j]) centroids[i] = _centroid([vecs[k] for k in clusters[i]]) del clusters[j], centroids[j] # ---------- 候选生成与审核 ---------- def discover_new_jobs( records: list[JDRecord], extractions: dict[str, ExtractionResult], known_titles: set[str], min_size: int = MIN_CLUSTER_SIZE, min_companies: int = MIN_COMPANIES, min_growth: float = MIN_GROWTH, min_distinctness: float = MIN_DISTINCTNESS, ) -> list[JobDefinition]: """从未知岗位 JD 中聚类发现候选新岗位（status=pending）。 四指标判定：支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度。 """ unknown = [r for r in records if not r.is_noise and not r.is_duplicate and r.jid in extractions and extractions[r.jid].job_title not in known_titles] if len(unknown) < min_size: logger.info("未知岗位 JD 不足 %d 条，跳过新岗位发现", min_size) return [] # 岗位名称是强信号：重复十次提升其在 TF-IDF 中的权重； # 再叠加抽取出的必备技能（重复三次），保证长文本、多模板句式的 JD 仍能按岗位聚拢 def doc_of(r: JDRecord) -> str: e = extractions[r.jid] return f"{r.job_title} " * 10 + " ".join(e.required_skills * 3) + " " + r.text docs = [doc_of(r) for r in unknown] vectorizer = TfidfVectorizer().fit(docs) vecs = [vectorizer.transform(d) for d in docs] clusters = agglomerative(vecs) # 既有岗位质心（差异度基准）：按已知标题聚合并向量化 known_docs: dict[str, list[str]] = {} for r in records: e = extractions.get(r.jid) if e and e.job_title in known_titles and not r.is_noise and not r.is_duplicate: known_docs.setdefault(e.job_title, []).append(doc_of(r)) known_centroids = {t: _centroid([vectorizer.transform(d) for d in ds]) for t, ds in known_docs.items()} candidates: list[JobDefinition] = [] for members in clusters: if len(members) < min_size: continue group = [unknown[i] for i in members] companies = {r.company for r in group} if len(companies) < min_companies: continue years = Counter(r.publish_date.year for r in group) latest_year = max(years) prev_year_n = years.get(latest_year - 1, 0) # 前一年没有基数视为“从无到有”，增长率视为达标 growth = years[latest_year] / prev_year_n if prev_year_n else float("inf") if growth < min_growth: continue # 差异度：与最近既有岗位质心的距离，过近说明只是既有岗位的变体 cvec = _centroid([vecs[i] for i in members]) max_sim = max((cosine(cvec, kc) for kc in known_centroids.values()), default=0.0) distinctness = 1.0 - max_sim if distinctness < min_distinctness: logger.info("簇与既有岗位差异度不足（%.2f < %.2f），不列为新岗位", distinctness, min_distinctness) continue # 簇内最常见的抽取标题作为候选岗位名；技能取簇内高频 titles = Counter(extractions[r.jid].job_title for r in group) title = titles.most_common(1)[0][0] req = Counter(s for r in group for s in extractions[r.jid].required_skills) pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills) scen = Counter(s for r in group for s in extractions[r.jid].scenarios) ind = Counter(r.industry for r in group if r.industry) n = len(group) candidates.append(JobDefinition( job_id=f"cand_{title}", title=title, category="新岗位候选", industries=[s for s, _ in ind.most_common(2)], required_skills=[s for s, c in req.most_common() if c >= n * 0.4], preferred_skills=[s for s, c in pref.most_common() if c >= n * 0.25], scenarios=[s for s, _ in scen.most_common(5)], status="pending", source_count=n, company_count=len(companies), support_source_ids=[r.source_id for r in group], support_windows=sorted({quarter_of(r.publish_date) for r in group}), )) logger.info("候选新岗位：%s（支撑 %d 条 JD / %d 家企业 / 增长率 %.2f / 差异度 %.2f）", title, n, len(companies), growth, distinctness) return candidates # ---------- 待审核分区与审核留痕（SQLite 状态库，见 app/storage/db.py） ---------- def save_pending(candidates: list[JobDefinition], path: str | Path) -> None: """整体覆写待审核分区。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。""" from app.storage import db keep = set() for c in candidates: db.save_candidate(path, c.model_dump()) keep.add(c.job_id) for old in db.list_candidates(path): if old["job_id"] not in keep: db.delete_candidate(path, old["job_id"]) def load_pending(path: str | Path) -> list[JobDefinition]: """读取全部待审核候选。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。""" from app.storage import db return [JobDefinition(**d) for d in db.list_candidates(path)] def apply_review( pending_path: str | Path, audit_path: str | Path, candidate_id: str, action: str, operator: str, reason: str = "", modify: dict | None = None, ) -> tuple[JobDefinition | None, ReviewRecord]: """人工审核：accept/modify/reject。返回 (通过的岗位定义或 None, 审核留痕)。 ``pending_path`` 为 SQLite 状态库路径；``audit_path`` 为旧 audit.jsonl 路径， 仅在状态库首次打开时作为一次性迁移种子使用（原文件保留）。 """ from app.storage import db db.connect(pending_path, legacy_audit=audit_path).close() # 首次打开触发迁移（幂等） candidates = load_pending(pending_path) target = next((c for c in candidates if c.job_id == candidate_id), None) if target is None: raise KeyError(f"候选不存在：{candidate_id}") before = target.model_dump() accepted: JobDefinition | None = None after: dict = {} if action == "accept": target.status = "active" accepted = target after = target.model_dump() elif action == "modify": if modify: target = target.model_copy(update=modify) target.status = "active" accepted = target after = target.model_dump() elif action == "reject": target.status = "rejected" after = target.model_dump() # 驳回：若该候选曾入库（覆写层有记录），一并清理，避免重建后复活 db.delete_override(pending_path, candidate_id) else: raise ValueError(f"未知审核动作：{action}") record = ReviewRecord(candidate_id=candidate_id, action=action, operator=operator, reason=reason, before=before, after=after, time=now_str()) db.append_audit(pending_path, record.model_dump()) # 已审核的候选从待审核分区移除 db.delete_candidate(pending_path, candidate_id) logger.info("审核完成：%s -> %s（操作人 %s）", candidate_id, action, operator) return accepted, record