diff --git a/app/cleaning/dedup.py b/app/cleaning/dedup.py index 5322578..a120b6d 100644 --- a/app/cleaning/dedup.py +++ b/app/cleaning/dedup.py @@ -13,7 +13,7 @@ from app.models import JDRecord logger = logging.getLogger(__name__) -NUM_HASHES = 128 # MinHash 哈希函数个数（>=64，越高估计越稳） +NUM_HASHES = 256 # MinHash 哈希函数个数（越高估计越稳；128 时边沿样本偏差可达 ±0.1 误杀植入簇） SHINGLE_N = 5 # 字符 5-gram shingles LSH_BANDS = 16 # 16 bands × 8 rows（严格分带，抑制模板化文本误配） _PRIME = (1 << 61) - 1 # 梅森素数，用于 (a*x+b) mod p 哈希族 diff --git a/app/discover/cluster.py b/app/discover/cluster.py index 82e1262..a52c844 100644 --- a/app/discover/cluster.py +++ b/app/discover/cluster.py @@ -23,6 +23,7 @@ MIN_COMPANIES = 5 # 独立企业数（防单一企业造词） MIN_GROWTH = 1.5 # 近期增长率（最近一年 JD 数 / 前一年，≥50% 增长） MIN_DISTINCTNESS = 0.4 # 与既有岗位差异度：1 - 与最近既有岗位质心的余弦相似度 MERGE_SIM = 0.50 # 凝聚合并的质心余弦相似度阈值 +RESP_MIN_RATIO = 0.25 # 候选职责入选阈值：簇内 ≥25%（且 ≥2 条）JD 提及，取频次 Top 5 # ---------- TF-IDF（字符 2/3-gram） ---------- @@ -158,21 +159,37 @@ def discover_new_jobs( if distinctness < min_distinctness: logger.info("簇与既有岗位差异度不足（%.2f < %.2f），不列为新岗位", distinctness, min_distinctness) continue - # 簇内最常见的抽取标题作为候选岗位名；技能取簇内高频 + # 簇内最常见的抽取标题作为候选岗位名；技能/职责/场景取簇内高频 titles = Counter(extractions[r.jid].job_title for r in group) title = titles.most_common(1)[0][0] req = Counter(s for r in group for s in extractions[r.jid].required_skills) pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills) + duty = Counter(d for r in group for d in extractions[r.jid].responsibilities) scen = Counter(s for r in group for s in extractions[r.jid].scenarios) ind = Counter(r.industry for r in group if r.industry) n = len(group) + industries = [s for s, _ in ind.most_common(2)] + # 核心职责：簇内 JD 抽取结果按频次聚合（≥RESP_MIN_RATIO 且 ≥2 条支撑，Top 5）， + # 保证候选定义满足"岗位名称/核心职责/必备技能/加分技能/典型行业应用场景"齐备 + responsibilities = [d for d, c in duty.most_common(5) if c >= max(2, n * RESP_MIN_RATIO)] + required = [s for s, c in req.most_common() if c >= n * 0.4] + if not responsibilities: + # 兜底：簇内职责高度分散时放宽到频次 Top 3；抽取不到职责句式时 + # 用 title+行业+核心技能拼一句话定义，保证审核台"职责摘要"有真实内容 + if duty: + responsibilities = [d for d, _ in duty.most_common(3)] + else: + focus = "、".join(required[:2]) or title + responsibilities = [ + f"面向{'/'.join(industries) or '新兴'}行业，承担{title}相关的{focus}设计、开发与交付工作"] candidates.append(JobDefinition( job_id=f"cand_{title}", title=title, category="新岗位候选", - industries=[s for s, _ in ind.most_common(2)], - required_skills=[s for s, c in req.most_common() if c >= n * 0.4], + industries=industries, + required_skills=required, preferred_skills=[s for s, c in pref.most_common() if c >= n * 0.25], + responsibilities=responsibilities, scenarios=[s for s, _ in scen.most_common(5)], status="pending", source_count=n, diff --git a/reports/metrics.json b/reports/metrics.json index 7676cc5..6da61ed 100644 --- a/reports/metrics.json +++ b/reports/metrics.json @@ -85,7 +85,7 @@ { "job": "Java开发工程师", "expected": true, - "score": 76.67, + "score": 75.42, "ok": true }, { @@ -97,7 +97,7 @@ { "job": "Python开发工程师", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -109,7 +109,7 @@ { "job": "前端工程师", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -121,7 +121,7 @@ { "job": "数据工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -133,7 +133,7 @@ { "job": "算法工程师", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -145,7 +145,7 @@ { "job": "数据分析师", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -157,7 +157,7 @@ { "job": "测试开发工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -169,7 +169,7 @@ { "job": "运维工程师（SRE）", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -181,7 +181,7 @@ { "job": "嵌入式工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -193,7 +193,7 @@ { "job": "物联网工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -205,7 +205,7 @@ { "job": "NLP工程师", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -217,7 +217,7 @@ { "job": "CV工程师", "expected": true, - "score": 75.0, + "score": 73.75, "ok": true }, { @@ -229,7 +229,7 @@ { "job": "云原生工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -241,7 +241,7 @@ { "job": "AI产品经理", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -253,7 +253,7 @@ { "job": "Go开发工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -265,7 +265,7 @@ { "job": "大数据开发工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -277,7 +277,7 @@ { "job": "机器学习工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -289,7 +289,7 @@ { "job": "智能机器人算法工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -301,7 +301,7 @@ { "job": "数据治理工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -313,7 +313,7 @@ { "job": "边缘计算工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -325,7 +325,7 @@ { "job": "物联网安全工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -337,7 +337,7 @@ { "job": "智能系统架构师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -349,7 +349,7 @@ { "job": "隐私计算工程师", "expected": true, - "score": 75.0, + "score": 73.0, "ok": true }, { @@ -361,31 +361,31 @@ { "job": "前端工程师", "expected": true, - "score": 70.0, + "score": 67.5, "ok": true }, { "job": "数据分析师", "expected": true, - "score": 70.0, + "score": 67.5, "ok": true }, { "job": "运维工程师（SRE）", "expected": true, - "score": 70.0, + "score": 67.5, "ok": true }, { "job": "前端工程师", "expected": true, - "score": 90.0, + "score": 87.5, "ok": true }, { "job": "数据分析师", "expected": true, - "score": 80.0, + "score": 77.5, "ok": true }, { diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 0ae8aa4..2cec374 100644 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -415,6 +415,81 @@ ARCHETYPES = [ COMPANY_PRE = ["星澜", "云启", "数联", "智汇", "蓝湾", "极光", "恒宇", "锐思", "博远", "辰安", "飞鱼", "墨白", "青藤", "远望", "砺石", "潮汐", "峰行", "微光", "若谷", "启航"] COMPANY_SUF = ["科技", "网络", "信息", "数据", "智能", "软件", "互联", "数字"] + +# ---------------------------------------------------------------- 近期演化信号（2024–2026Q3） +# 评审反馈修复：此前仅 6 个岗位有深度演化剧本，其余岗位 2023Q1 后事件空档、 +# 且 2026Q4 属未来窗口。这里为每个既有岗位补 3-6 个轻量近期信号（贴合"AI 渗透" +# 真实趋势：2024+ 多数岗位出现 AI 辅助/大模型加分项，2025+ 部分技能减弱/退出）： +# add_pref: [(起始年, 技能)] 该年起作为加分项写入 JD（触发"新增"事件） +# fade: (起始年, 技能) 该年起从 JD 技能清单消失（触发"退出"事件） +# weaken: (起始年, 技能, 保留概率) 高量级岗位该年起提及率下降（触发"减弱"事件） +RECENT_SIGNALS = { + # —— 2024 年时间轴已换代的岗位：只补 2025/2026 信号 —— + "Java开发工程师": {"add_pref": [(2025, "RAG")], "weaken": (2026, "MySQL", 0.4)}, + "Python开发工程师": {"add_pref": [(2025, "向量数据库")], "weaken": (2026, "Redis", 0.4)}, + "前端工程师": {"add_pref": [(2025, "大模型应用开发")], "fade": (2026, "微前端")}, + "算法工程师": {"add_pref": [(2025, "RAG")], "fade": (2026, "强化学习")}, + "数据分析师": {"add_pref": [(2025, "提示词工程")], "weaken": (2026, "Excel", 0.4)}, + "运维工程师（SRE）": {"add_pref": [(2025, "AI辅助编程")], "weaken": (2026, "Docker", 0.4)}, + "NLP工程师": {"add_pref": [(2025, "AI辅助编程")], "fade": (2026, "LangChain")}, + "CV工程师": {"add_pref": [(2025, "AI辅助编程")], "fade": (2026, "模型微调")}, + # —— 其余岗位：2024 起补 AI 加分项 + 2026 技能退出 —— + "数据工程师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "weaken": (2026, "Spark", 0.4)}, + "测试开发工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "大模型应用开发")], + "weaken": (2026, "性能测试", 0.4)}, + "嵌入式工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "边缘推理")], + "fade": (2026, "STM32")}, + "物联网工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "大模型应用开发")], + "fade": (2026, "LoRa")}, + "云原生工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "大模型应用开发")], + "fade": (2026, "Service Mesh")}, + "AI产品经理": {"add_pref": [(2024, "AI辅助编程"), (2025, "RAG")], + "fade": (2026, "用户增长")}, + "Go开发工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "大模型应用开发")], + "fade": (2026, "Kafka")}, + "大数据开发工程师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "fade": (2026, "Hive")}, + "机器学习工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "RAG")], + "fade": (2026, "XGBoost")}, + "智能机器人算法工程师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "fade": (2026, "OpenCV")}, + "数据治理工程师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "fade": (2026, "Hive")}, + "边缘计算工程师": {"add_pref": [(2024, "AI辅助编程"), (2025, "大模型应用开发")], + "fade": (2026, "KubeEdge")}, + "物联网安全工程师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "fade": (2026, "嵌入式开发")}, + "智能系统架构师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "fade": (2026, "Service Mesh")}, + "隐私计算工程师": {"add_pref": [(2024, "大模型应用开发"), (2025, "AI辅助编程")], + "fade": (2026, "Spark")}, +} + +# 数据收敛到当前季度：今天是 2026-08（2026Q3），2026 年 JD 只落在 1-9 月， +# 不生成 2026Q4 等未来窗口（含近重复副本与噪声记录的日期上限）。 +CURRENT_YEAR_MAX_MONTH = 9 + + +def apply_recent_signals(a: dict, year: int, req: list[str], pref: list[str], + ) -> tuple[list[str], list[str]]: + """按 RECENT_SIGNALS 对当年技能清单施加近期演化信号（先定结构再渲染，gold 自动一致）。""" + sig = RECENT_SIGNALS.get(a["title"]) + if not sig: + return req, pref + for y0, s in sig.get("add_pref", ()): + if year >= y0 and s not in req and s not in pref: + pref.append(s) + fade = sig.get("fade") + if fade and year >= fade[0]: + req = [x for x in req if x != fade[1]] + pref = [x for x in pref if x != fade[1]] + wk = sig.get("weaken") + if wk and year >= wk[0] and rng.random() > wk[2]: + req = [x for x in req if x != wk[1]] + pref = [x for x in pref if x != wk[1]] + return req, pref + # 15 城：一线 + 新一线（成都/武汉/西安/苏州/长沙等） CITIES = ["北京", "上海", "深圳", "杭州", "成都", "广州", "南京", "武汉", "西安", "苏州", "长沙", "重庆", "天津", "青岛", "合肥"] @@ -569,11 +644,13 @@ def make_jd(a: dict, year: int, jid: str, month: int | None = None) -> tuple[dic if len(req) > 4 and rng.random() < 0.15: # 轻微抖动：偶尔少写一项必备 req = req[:-1] pref = pref[: rng.choice([2, 2, 3])] + req, pref = apply_recent_signals(a, year, req, pref) duties = pick_duties(a, year) scens = rng.sample(a["scenarios"], k=min(2, len(a["scenarios"]))) years = rng.randint(max(1, a["years"] - 1), a["years"] + 2) text = build_text(a, company, year, years, duties, req, pref, scens) - month = month or rng.randint(1, 12) + # 2026 年只取 1-9 月：数据收敛到当前季度 2026Q3，不出现未来窗口 + month = month or rng.randint(1, CURRENT_YEAR_MAX_MONTH if year >= 2026 else 12) day = rng.randint(1, 28) record = { "jid": jid, @@ -612,7 +689,7 @@ def make_dup_copy(rec: dict, jid: str) -> dict: "text": text, "publish_date": min( date.fromisoformat(rec["publish_date"]) + timedelta(days=rng.randint(10, 120)), - date(2026, 12, 28)).isoformat(), + date(2026, CURRENT_YEAR_MAX_MONTH, 28)).isoformat(), }) return copy @@ -659,10 +736,11 @@ RESUME_LEGACY_MISMATCH = ( def _cand(title: str, req: list[str], pref: list[str], scens: list[str], - src: int, comp: int, status: str = "pending") -> dict: + src: int, comp: int, status: str = "pending", resp: list[str] | None = None) -> dict: """构造一条待审核候选的快照（审核留痕种子用）。""" return {"job_id": f"cand_{title}", "title": title, "category": "新岗位候选", - "required_skills": req, "preferred_skills": pref, "responsibilities": [], + "required_skills": req, "preferred_skills": pref, + "responsibilities": list(resp or []), "scenarios": scens, "expected_years": 3, "status": status, "source_count": src, "company_count": comp} @@ -673,20 +751,30 @@ AUDIT_SEED = [ "reason": "岗位簇证据充分：40+ 条 JD、独立企业数达标，2024 年起持续高增长，技能结构清晰，准予入库。", "before": _cand("大模型应用工程师", ["提示词工程", "Python", "LangChain", "大模型应用开发", "NLP"], - ["模型微调", "向量数据库", "RAG"], ["教育", "医疗", "金融"], 42, 38), + ["模型微调", "向量数据库", "RAG"], ["教育", "医疗", "金融"], 42, 38, + resp=["负责智能助手/企业知识库等大模型应用的开发与落地", + "设计提示词与流程编排，建设应用评测体系", + "优化推理成本与延迟，跟进生态工具"]), "after": _cand("大模型应用工程师", ["提示词工程", "Python", "LangChain", "大模型应用开发", "NLP"], - ["模型微调", "向量数据库", "RAG"], ["教育", "医疗", "金融"], 42, 38, "active"), + ["模型微调", "向量数据库", "RAG"], ["教育", "医疗", "金融"], 42, 38, "active", + resp=["负责智能助手/企业知识库等大模型应用的开发与落地", + "设计提示词与流程编排，建设应用评测体系", + "优化推理成本与延迟，跟进生态工具"]), "time": "2026-07-18 10:24:31"}, {"candidate_id": "cand_RAG工程师", "action": "modify", "operator": "李强（评审专家）", "reason": "簇内 JD 普遍要求检索引擎实操经验，补充 Elasticsearch 为必备技能后入库。", "before": _cand("RAG工程师", ["Python", "RAG", "向量数据库", "大模型应用开发", "LangChain"], - ["模型微调", "知识图谱"], ["金融", "政务", "教育"], 28, 22), + ["模型微调", "知识图谱"], ["金融", "政务", "教育"], 28, 22, + resp=["负责检索链路搭建与召回重排优化", + "建设知识库加工流水线，评估问答质量"]), "after": _cand("RAG工程师", ["Python", "RAG", "向量数据库", "大模型应用开发", "LangChain", "Elasticsearch"], - ["模型微调", "知识图谱"], ["金融", "政务", "教育"], 28, 22, "active"), + ["模型微调", "知识图谱"], ["金融", "政务", "教育"], 28, 22, "active", + resp=["负责检索链路搭建与召回重排优化", + "建设知识库加工流水线，评估问答质量"]), "time": "2026-07-19 15:02:11"}, {"candidate_id": "cand_元宇宙社区运营", "action": "reject", "operator": "王蕾（评审专家）", @@ -710,10 +798,14 @@ AUDIT_SEED = [ "reason": "2024 年起多家企业独立设岗，职责与传统运营岗差异显著，予以入库观察。", "before": _cand("提示词工程师", ["提示词工程", "大模型应用开发", "Python", "数据分析"], - ["LangChain", "RAG", "AI产品"], ["教育", "电商", "广告"], 18, 14), + ["LangChain", "RAG", "AI产品"], ["教育", "电商", "广告"], 18, 14, + resp=["负责提示词设计与模板库沉淀", + "评估输出质量，编写最佳实践文档"]), "after": _cand("提示词工程师", ["提示词工程", "大模型应用开发", "Python", "数据分析"], - ["LangChain", "RAG", "AI产品"], ["教育", "电商", "广告"], 18, 14, "active"), + ["LangChain", "RAG", "AI产品"], ["教育", "电商", "广告"], 18, 14, "active", + resp=["负责提示词设计与模板库沉淀", + "评估输出质量，编写最佳实践文档"]), "time": "2026-07-25 11:30:44"}, {"kind": "node_update", "node_id": "job_运维工程师（SRE）", "operator": "李强（评审专家）", @@ -817,8 +909,10 @@ def main() -> None: for year, count in sorted(a["volumes"].items()): for i in range(count): seq += 1 - # 月份年内均匀铺开：季度窗口计数只随剧本（时间轴/量级）变化 - rec, _ = make_jd(a, year, f"J{seq:05d}", month=i * 12 // count + 1) + # 月份年内均匀铺开：季度窗口计数只随剧本（时间轴/量级）变化； + # 2026 年收敛到 1-9 月（当前季度 2026Q3，不生成未来窗口） + span = CURRENT_YEAR_MAX_MONTH if year >= 2026 else 12 + rec, _ = make_jd(a, year, f"J{seq:05d}", month=i * span // count + 1) raw_by_year[year].append(rec) # 3. 植入噪声（约 3%）：一半混排尾注（清洗时剔除行），一半整篇广告（整条标记） @@ -833,12 +927,13 @@ def main() -> None: seq += 1 jid = f"J{seq:05d}" year = rng.randint(2020, 2026) + max_month = CURRENT_YEAR_MAX_MONTH if year >= 2026 else 12 raw_by_year[year].append({ "jid": jid, "job_title": rng.choice(ARCHETYPES)["title"], "company": rng.choice(COMPANY_PRE) + rng.choice(COMPANY_SUF), "city": rng.choice(CITIES), "salary": "面议", "industry": rng.choice(INDUSTRIES), - "publish_date": date(year, rng.randint(1, 12), rng.randint(1, 28)).isoformat(), + "publish_date": date(year, rng.randint(1, max_month), rng.randint(1, 28)).isoformat(), "source": rng.choice(SOURCES), "url": f"https://jobs.example.com/{jid}", "text": full, }) diff --git a/webui/app.js b/webui/app.js index 3febcb9..a043d44 100644 --- a/webui/app.js +++ b/webui/app.js @@ -608,10 +608,17 @@ async function renderReviewDesk(focusId) { const t = candidateTrust(c, CN.jobs); const wins = (c.support_windows || []).slice().sort(); const span = wins.length ? `${wins[0]} ~ ${wins[wins.length - 1]}` : "—"; + const resp = c.responsibilities || []; + const summary = resp[0] + || `面向${(c.industries || []).join("/") || "新兴"}领域的「${c.title}」岗位（职责描述待人工补充）`; return `
候选：${esc(c.title)} 可信等级：${t.level}
-
职责摘要：${esc((c.responsibilities || [])[0] || `负责${c.title}相关的方案设计、开发与交付（职责描述待人工补充）`)}
+
职责摘要：${esc(summary)}
+ ${resp.length > 1 ? `
核心职责：${resp.slice(0, 5).map((d) => `${esc(d.replace(/；\s*$/, ""))}`).join(" ")}
` : ""} +
应用场景：${(c.scenarios || []).length + ? c.scenarios.map((s) => `${esc(s)}`).join(" ") + : "—"}
${t.dots.map((d) => ``).join("")} ${t.okCount}/4 指标达标（JD / 企业 / 增长 / 差异）