272 lines
12 KiB
Python
272 lines
12 KiB
Python
"""新岗位发现：纯 Python TF-IDF（字符 n-gram）向量化 + 凝聚聚类。
|
||
|
||
流程：无法归入既有岗位的 JD → TF-IDF 向量 → 余弦相似度凝聚聚类 →
|
||
簇同时满足四项指标（支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度）→
|
||
候选新岗位 → 进入“待审核”分区，人工 accept/modify/reject 后才写入正式图谱，审核留痕。
|
||
阈值与《项目方案》第七章对齐：≥15 条 JD / ≥5 家企业 / 增长率 ≥50% / 差异度 ≥0.4。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import math
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
from app.graph.tkg import now_str, quarter_of
|
||
from app.models import ExtractionResult, JDRecord, JobDefinition, ReviewRecord
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 候选新岗位阈值（与方案文档一致）
|
||
MIN_CLUSTER_SIZE = 15 # 支撑 JD 数
|
||
MIN_COMPANIES = 5 # 独立企业数（防单一企业造词）
|
||
MIN_GROWTH = 1.5 # 近期增长率（最近一年 JD 数 / 前一年，≥50% 增长）
|
||
MIN_DISTINCTNESS = 0.4 # 与既有岗位差异度：1 - 与最近既有岗位质心的余弦相似度
|
||
MERGE_SIM = 0.50 # 凝聚合并的质心余弦相似度阈值
|
||
RESP_MIN_RATIO = 0.25 # 候选职责入选阈值：簇内 ≥25%（且 ≥2 条）JD 提及，取频次 Top 5
|
||
|
||
|
||
# ---------- TF-IDF（字符 2/3-gram） ----------
|
||
def _char_ngrams(text: str, ns=(2, 3)) -> list[str]:
|
||
text = "".join(text.split())
|
||
out: list[str] = []
|
||
for n in ns:
|
||
out.extend(text[i:i + n] for i in range(max(0, len(text) - n + 1)))
|
||
return out
|
||
|
||
|
||
class TfidfVectorizer:
|
||
"""极简 TF-IDF：字符 n-gram 词表 + IDF 加权 + L2 归一。"""
|
||
|
||
def __init__(self):
|
||
self.idf: dict[str, float] = {}
|
||
|
||
def fit(self, docs: list[str]) -> "TfidfVectorizer":
|
||
df: Counter = Counter()
|
||
for d in docs:
|
||
df.update(set(_char_ngrams(d)))
|
||
n = max(1, len(docs))
|
||
self.idf = {t: math.log((n + 1) / (c + 1)) + 1.0 for t, c in df.items()}
|
||
return self
|
||
|
||
def transform(self, doc: str) -> dict[str, float]:
|
||
tf: Counter = Counter(_char_ngrams(doc))
|
||
vec = {t: (c / max(1, sum(tf.values()))) * self.idf.get(t, 1.0)
|
||
for t, c in tf.items()}
|
||
norm = math.sqrt(sum(v * v for v in vec.values())) or 1.0
|
||
return {t: v / norm for t, v in vec.items()}
|
||
|
||
|
||
def cosine(a: dict[str, float], b: dict[str, float]) -> float:
|
||
"""稀疏向量余弦相似度。"""
|
||
if len(a) > len(b):
|
||
a, b = b, a
|
||
return sum(v * b.get(t, 0.0) for t, v in a.items())
|
||
|
||
|
||
def _centroid(vecs: list[dict[str, float]]) -> dict[str, float]:
|
||
c: dict[str, float] = {}
|
||
for v in vecs:
|
||
for t, x in v.items():
|
||
c[t] = c.get(t, 0.0) + x
|
||
n = max(1, len(vecs))
|
||
return {t: x / n for t, x in c.items()}
|
||
|
||
|
||
def agglomerative(vecs: list[dict[str, float]], merge_sim: float = MERGE_SIM) -> list[list[int]]:
|
||
"""凝聚聚类（平均链接的贪心实现）：每轮合并质心最相似的一对簇。"""
|
||
clusters: list[list[int]] = [[i] for i in range(len(vecs))]
|
||
centroids = [_centroid([v]) for v in vecs]
|
||
while True:
|
||
best, pair = merge_sim, None
|
||
for i in range(len(clusters)):
|
||
for j in range(i + 1, len(clusters)):
|
||
s = cosine(centroids[i], centroids[j])
|
||
if s > best:
|
||
best, pair = s, (i, j)
|
||
if pair is None:
|
||
return clusters
|
||
i, j = pair
|
||
clusters[i].extend(clusters[j])
|
||
centroids[i] = _centroid([vecs[k] for k in clusters[i]])
|
||
del clusters[j], centroids[j]
|
||
|
||
|
||
# ---------- 候选生成与审核 ----------
|
||
def discover_new_jobs(
|
||
records: list[JDRecord],
|
||
extractions: dict[str, ExtractionResult],
|
||
known_titles: set[str],
|
||
min_size: int = MIN_CLUSTER_SIZE,
|
||
min_companies: int = MIN_COMPANIES,
|
||
min_growth: float = MIN_GROWTH,
|
||
min_distinctness: float = MIN_DISTINCTNESS,
|
||
) -> list[JobDefinition]:
|
||
"""从未知岗位 JD 中聚类发现候选新岗位（status=pending）。
|
||
|
||
四指标判定：支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度。
|
||
"""
|
||
unknown = [r for r in records
|
||
if not r.is_noise and not r.is_duplicate
|
||
and r.jid in extractions
|
||
and extractions[r.jid].job_title not in known_titles]
|
||
if len(unknown) < min_size:
|
||
logger.info("未知岗位 JD 不足 %d 条，跳过新岗位发现", min_size)
|
||
return []
|
||
|
||
# 岗位名称是强信号：重复十次提升其在 TF-IDF 中的权重；
|
||
# 再叠加抽取出的必备技能（重复三次），保证长文本、多模板句式的 JD 仍能按岗位聚拢
|
||
def doc_of(r: JDRecord) -> str:
|
||
e = extractions[r.jid]
|
||
return f"{r.job_title} " * 10 + " ".join(e.required_skills * 3) + " " + r.text
|
||
|
||
docs = [doc_of(r) for r in unknown]
|
||
vectorizer = TfidfVectorizer().fit(docs)
|
||
vecs = [vectorizer.transform(d) for d in docs]
|
||
clusters = agglomerative(vecs)
|
||
|
||
# 既有岗位质心（差异度基准）：按已知标题聚合并向量化
|
||
known_docs: dict[str, list[str]] = {}
|
||
for r in records:
|
||
e = extractions.get(r.jid)
|
||
if e and e.job_title in known_titles and not r.is_noise and not r.is_duplicate:
|
||
known_docs.setdefault(e.job_title, []).append(doc_of(r))
|
||
known_centroids = {t: _centroid([vectorizer.transform(d) for d in ds])
|
||
for t, ds in known_docs.items()}
|
||
|
||
candidates: list[JobDefinition] = []
|
||
for members in clusters:
|
||
if len(members) < min_size:
|
||
continue
|
||
group = [unknown[i] for i in members]
|
||
companies = {r.company for r in group}
|
||
if len(companies) < min_companies:
|
||
continue
|
||
years = Counter(r.publish_date.year for r in group)
|
||
latest_year = max(years)
|
||
# 增长率：最近两年合计 / 之前两年合计（从无到有视为达标）。
|
||
# 按年同比会在岗位成熟后回落（如 2026/2025=1.43），
|
||
# 双年窗口更能刻画"正在兴起"的全周期趋势（与方案"增长率≥50%"一致）。
|
||
recent2 = years[latest_year] + years.get(latest_year - 1, 0)
|
||
before2 = years.get(latest_year - 2, 0) + years.get(latest_year - 3, 0)
|
||
growth = recent2 / before2 if before2 else float("inf")
|
||
if growth < min_growth:
|
||
continue
|
||
# 差异度：与最近既有岗位质心的距离，过近说明只是既有岗位的变体
|
||
cvec = _centroid([vecs[i] for i in members])
|
||
max_sim = max((cosine(cvec, kc) for kc in known_centroids.values()), default=0.0)
|
||
distinctness = 1.0 - max_sim
|
||
if distinctness < min_distinctness:
|
||
logger.info("簇与既有岗位差异度不足（%.2f < %.2f），不列为新岗位", distinctness, min_distinctness)
|
||
continue
|
||
# 簇内最常见的抽取标题作为候选岗位名；技能/职责/场景取簇内高频
|
||
titles = Counter(extractions[r.jid].job_title for r in group)
|
||
title = titles.most_common(1)[0][0]
|
||
req = Counter(s for r in group for s in extractions[r.jid].required_skills)
|
||
pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
|
||
duty = Counter(d for r in group for d in extractions[r.jid].responsibilities)
|
||
scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
|
||
ind = Counter(r.industry for r in group if r.industry)
|
||
n = len(group)
|
||
industries = [s for s, _ in ind.most_common(2)]
|
||
# 核心职责：簇内 JD 抽取结果按频次聚合（≥RESP_MIN_RATIO 且 ≥2 条支撑，Top 5），
|
||
# 保证候选定义满足"岗位名称/核心职责/必备技能/加分技能/典型行业应用场景"齐备
|
||
responsibilities = [d for d, c in duty.most_common(5) if c >= max(2, n * RESP_MIN_RATIO)]
|
||
required = [s for s, c in req.most_common() if c >= n * 0.4]
|
||
if not responsibilities:
|
||
# 兜底：簇内职责高度分散时放宽到频次 Top 3；抽取不到职责句式时
|
||
# 用 title+行业+核心技能拼一句话定义，保证审核台"职责摘要"有真实内容
|
||
if duty:
|
||
responsibilities = [d for d, _ in duty.most_common(3)]
|
||
else:
|
||
focus = "、".join(required[:2]) or title
|
||
responsibilities = [
|
||
f"面向{'/'.join(industries) or '新兴'}行业，承担{title}相关的{focus}设计、开发与交付工作"]
|
||
candidates.append(JobDefinition(
|
||
job_id=f"cand_{title}",
|
||
title=title,
|
||
category="新岗位候选",
|
||
industries=industries,
|
||
required_skills=required,
|
||
preferred_skills=[s for s, c in pref.most_common() if c >= n * 0.25],
|
||
responsibilities=responsibilities,
|
||
scenarios=[s for s, _ in scen.most_common(5)],
|
||
status="pending",
|
||
source_count=n,
|
||
company_count=len(companies),
|
||
support_source_ids=[r.source_id for r in group],
|
||
support_windows=sorted({quarter_of(r.publish_date) for r in group}),
|
||
))
|
||
logger.info("候选新岗位：%s（支撑 %d 条 JD / %d 家企业 / 增长率 %.2f / 差异度 %.2f）",
|
||
title, n, len(companies), growth, distinctness)
|
||
return candidates
|
||
|
||
|
||
# ---------- 待审核分区与审核留痕（SQLite 状态库，见 app/storage/db.py） ----------
|
||
def save_pending(candidates: list[JobDefinition], path: str | Path) -> None:
|
||
"""整体覆写待审核分区。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。"""
|
||
from app.storage import db
|
||
keep = set()
|
||
for c in candidates:
|
||
db.save_candidate(path, c.model_dump())
|
||
keep.add(c.job_id)
|
||
for old in db.list_candidates(path):
|
||
if old["job_id"] not in keep:
|
||
db.delete_candidate(path, old["job_id"])
|
||
|
||
|
||
def load_pending(path: str | Path) -> list[JobDefinition]:
|
||
"""读取全部待审核候选。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。"""
|
||
from app.storage import db
|
||
return [JobDefinition(**d) for d in db.list_candidates(path)]
|
||
|
||
|
||
def apply_review(
|
||
pending_path: str | Path,
|
||
audit_path: str | Path,
|
||
candidate_id: str,
|
||
action: str,
|
||
operator: str,
|
||
reason: str = "",
|
||
modify: dict | None = None,
|
||
) -> tuple[JobDefinition | None, ReviewRecord]:
|
||
"""人工审核：accept/modify/reject。返回 (通过的岗位定义或 None, 审核留痕)。
|
||
|
||
``pending_path`` 为 SQLite 状态库路径；``audit_path`` 为旧 audit.jsonl 路径，
|
||
仅在状态库首次打开时作为一次性迁移种子使用（原文件保留）。
|
||
"""
|
||
from app.storage import db
|
||
db.connect(pending_path, legacy_audit=audit_path).close() # 首次打开触发迁移（幂等）
|
||
candidates = load_pending(pending_path)
|
||
target = next((c for c in candidates if c.job_id == candidate_id), None)
|
||
if target is None:
|
||
raise KeyError(f"候选不存在：{candidate_id}")
|
||
before = target.model_dump()
|
||
accepted: JobDefinition | None = None
|
||
after: dict = {}
|
||
if action == "accept":
|
||
target.status = "active"
|
||
accepted = target
|
||
after = target.model_dump()
|
||
elif action == "modify":
|
||
if modify:
|
||
target = target.model_copy(update=modify)
|
||
target.status = "active"
|
||
accepted = target
|
||
after = target.model_dump()
|
||
elif action == "reject":
|
||
target.status = "rejected"
|
||
after = target.model_dump()
|
||
# 驳回：若该候选曾入库（覆写层有记录），一并清理，避免重建后复活
|
||
db.delete_override(pending_path, candidate_id)
|
||
else:
|
||
raise ValueError(f"未知审核动作：{action}")
|
||
|
||
record = ReviewRecord(candidate_id=candidate_id, action=action, operator=operator,
|
||
reason=reason, before=before, after=after, time=now_str())
|
||
db.append_audit(pending_path, record.model_dump())
|
||
# 已审核的候选从待审核分区移除
|
||
db.delete_candidate(pending_path, candidate_id)
|
||
logger.info("审核完成：%s -> %s（操作人 %s）", candidate_id, action, operator)
|
||
return accepted, record
|