RoyirinRosmontis
a10442d78c
- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
361 lines
12 KiB
Python
361 lines
12 KiB
Python
"""SQLite 应用状态库：审核留痕 / 待审核候选 / 异步任务（默认启用，零新依赖）。
|
||
|
||
- 数据库文件默认 ``data/careernexus.db``（配置键 ``state_db_path``），仅用标准库 sqlite3；
|
||
- WAL 模式、外键开；``init_schema()`` 幂等建表，重复调用无副作用；
|
||
- 一次性迁移：打开连接时若旧文件存在、对应表为空且尚未迁移过，把
|
||
``data/review/audit.jsonl`` 与 ``data/review/pending.json`` 导入
|
||
（原文件保留不删除，迁移标记写入 meta 表并记日志）。
|
||
|
||
留痕存取约定：审核记录（candidate_id/action）把 action 存进 kind 列、
|
||
candidate_id 存进 node_id 列；节点编辑记录（kind=node_update）原样存取。
|
||
``get_audit`` 按此约定还原出与旧 audit.jsonl 行完全一致的字典结构。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import sqlite3
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
SCHEMA = """\
|
||
CREATE TABLE IF NOT EXISTS audit_log (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
ts TEXT,
|
||
node_id TEXT,
|
||
kind TEXT,
|
||
operator TEXT,
|
||
reason TEXT,
|
||
before_json TEXT,
|
||
after_json TEXT
|
||
);
|
||
CREATE TABLE IF NOT EXISTS review_candidates (
|
||
job_id TEXT PRIMARY KEY,
|
||
payload_json TEXT,
|
||
updated_ts TEXT
|
||
);
|
||
CREATE TABLE IF NOT EXISTS tasks (
|
||
task_id TEXT PRIMARY KEY,
|
||
kind TEXT,
|
||
status TEXT,
|
||
payload_json TEXT,
|
||
created_ts TEXT,
|
||
updated_ts TEXT
|
||
);
|
||
CREATE TABLE IF NOT EXISTS job_overrides (
|
||
name TEXT PRIMARY KEY,
|
||
payload_json TEXT,
|
||
updated_ts TEXT
|
||
);
|
||
CREATE TABLE IF NOT EXISTS meta (
|
||
key TEXT PRIMARY KEY,
|
||
value TEXT
|
||
);
|
||
"""
|
||
|
||
REVIEW_ACTIONS = {"accept", "modify", "reject"}
|
||
|
||
|
||
def _now() -> str:
|
||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
# ---------- 连接与建表 ----------
|
||
def init_schema(conn: sqlite3.Connection) -> None:
|
||
"""幂等建表（CREATE TABLE IF NOT EXISTS），可反复调用。"""
|
||
conn.executescript(SCHEMA)
|
||
conn.commit()
|
||
|
||
|
||
def connect(
|
||
db_path: str | Path,
|
||
legacy_audit: str | Path | None = None,
|
||
legacy_pending: str | Path | None = None,
|
||
) -> sqlite3.Connection:
|
||
"""打开（必要时创建）状态库：WAL + 外键 + 幂等建表 + 一次性旧文件迁移。"""
|
||
path = Path(db_path)
|
||
if str(path.parent) not in ("", "."):
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
conn = sqlite3.connect(str(path))
|
||
conn.row_factory = sqlite3.Row
|
||
conn.execute("PRAGMA journal_mode=WAL")
|
||
conn.execute("PRAGMA foreign_keys=ON")
|
||
init_schema(conn)
|
||
migrate_legacy(conn, legacy_audit=legacy_audit, legacy_pending=legacy_pending)
|
||
return conn
|
||
|
||
|
||
# ---------- 一次性迁移（旧 JSONL/JSON -> SQLite，原文件保留） ----------
|
||
def _migration_done(conn: sqlite3.Connection, key: str) -> bool:
|
||
row = conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
||
return row is not None
|
||
|
||
|
||
def _mark_migrated(conn: sqlite3.Connection, key: str) -> None:
|
||
conn.execute("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)", (key, _now()))
|
||
|
||
|
||
def _table_empty(conn: sqlite3.Connection, table: str) -> bool:
|
||
return conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0
|
||
|
||
|
||
def migrate_legacy(
|
||
conn: sqlite3.Connection,
|
||
legacy_audit: str | Path | None = None,
|
||
legacy_pending: str | Path | None = None,
|
||
) -> None:
|
||
"""把旧状态文件导入 SQLite：仅当对应表为空且未迁移过时执行，原文件保留。"""
|
||
if legacy_audit is not None:
|
||
_migrate_audit(conn, Path(legacy_audit))
|
||
if legacy_pending is not None:
|
||
_migrate_pending(conn, Path(legacy_pending))
|
||
conn.commit()
|
||
|
||
|
||
def _migrate_audit(conn: sqlite3.Connection, path: Path) -> None:
|
||
key = "migrated_audit"
|
||
if _migration_done(conn, key):
|
||
return
|
||
_mark_migrated(conn, key)
|
||
if not path.exists() or not _table_empty(conn, "audit_log"):
|
||
return
|
||
n = 0
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
if not line.strip():
|
||
continue
|
||
try:
|
||
record = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
_insert_audit(conn, record)
|
||
n += 1
|
||
logger.info("状态迁移：%s -> audit_log（%d 条，原文件保留）", path, n)
|
||
|
||
|
||
def _migrate_pending(conn: sqlite3.Connection, path: Path) -> None:
|
||
key = "migrated_pending"
|
||
if _migration_done(conn, key):
|
||
return
|
||
_mark_migrated(conn, key)
|
||
if not path.exists() or not _table_empty(conn, "review_candidates"):
|
||
return
|
||
try:
|
||
candidates = json.loads(path.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError:
|
||
return
|
||
for payload in candidates:
|
||
_upsert_candidate(conn, payload)
|
||
logger.info("状态迁移：%s -> review_candidates（%d 条，原文件保留）", path, len(candidates))
|
||
|
||
|
||
# ---------- 审核留痕 ----------
|
||
def _insert_audit(conn: sqlite3.Connection, record: dict) -> None:
|
||
kind = record.get("kind") or record.get("action") or ""
|
||
node_id = record.get("node_id") or record.get("candidate_id") or ""
|
||
conn.execute(
|
||
"INSERT INTO audit_log(ts, node_id, kind, operator, reason, before_json, after_json)"
|
||
" VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||
(
|
||
record.get("time") or _now(),
|
||
node_id,
|
||
kind,
|
||
record.get("operator", ""),
|
||
record.get("reason", ""),
|
||
json.dumps(record.get("before") or {}, ensure_ascii=False),
|
||
json.dumps(record.get("after") or {}, ensure_ascii=False),
|
||
),
|
||
)
|
||
|
||
|
||
def append_audit(db_path: str | Path, record: dict) -> None:
|
||
"""追加一条留痕（审核记录或 kind=node_update 的节点编辑记录）。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
_insert_audit(conn, record)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _audit_row_to_record(row: sqlite3.Row) -> dict:
|
||
"""把 audit_log 行还原为旧 audit.jsonl 行的字典结构（/api/audit 兼容）。"""
|
||
before = json.loads(row["before_json"] or "{}")
|
||
after = json.loads(row["after_json"] or "{}")
|
||
if row["kind"] in REVIEW_ACTIONS:
|
||
return {"candidate_id": row["node_id"], "action": row["kind"],
|
||
"operator": row["operator"], "reason": row["reason"],
|
||
"before": before, "after": after, "time": row["ts"]}
|
||
return {"kind": row["kind"], "node_id": row["node_id"],
|
||
"operator": row["operator"], "reason": row["reason"],
|
||
"before": before, "after": after, "time": row["ts"]}
|
||
|
||
|
||
def get_audit(db_path: str | Path, node_id: str | None = None, limit: int = 50) -> list[dict]:
|
||
"""读取留痕（新→旧排序），可按节点/候选 id 过滤。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
if node_id:
|
||
rows = conn.execute(
|
||
"SELECT * FROM audit_log WHERE node_id = ? ORDER BY id DESC LIMIT ?",
|
||
(node_id, limit)).fetchall()
|
||
else:
|
||
rows = conn.execute(
|
||
"SELECT * FROM audit_log ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
|
||
return [_audit_row_to_record(r) for r in rows]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
# ---------- 待审核候选 ----------
|
||
def _upsert_candidate(conn: sqlite3.Connection, payload: dict) -> None:
|
||
conn.execute(
|
||
"INSERT INTO review_candidates(job_id, payload_json, updated_ts) VALUES (?, ?, ?)"
|
||
" ON CONFLICT(job_id) DO UPDATE SET payload_json = excluded.payload_json,"
|
||
" updated_ts = excluded.updated_ts",
|
||
(payload["job_id"], json.dumps(payload, ensure_ascii=False), _now()))
|
||
|
||
|
||
def save_candidate(db_path: str | Path, payload: dict) -> None:
|
||
"""新增或更新一个待审核候选（按 job_id upsert）。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
_upsert_candidate(conn, payload)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def get_candidate(db_path: str | Path, job_id: str) -> dict | None:
|
||
conn = connect(db_path)
|
||
try:
|
||
row = conn.execute(
|
||
"SELECT payload_json FROM review_candidates WHERE job_id = ?",
|
||
(job_id,)).fetchone()
|
||
return json.loads(row["payload_json"]) if row else None
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def list_candidates(db_path: str | Path) -> list[dict]:
|
||
"""全部待审核候选（按写入顺序）。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT payload_json FROM review_candidates ORDER BY rowid").fetchall()
|
||
return [json.loads(r["payload_json"]) for r in rows]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def delete_candidate(db_path: str | Path, job_id: str) -> None:
|
||
conn = connect(db_path)
|
||
try:
|
||
conn.execute("DELETE FROM review_candidates WHERE job_id = ?", (job_id,))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
# ---------- 异步任务 ----------
|
||
def upsert_task(
|
||
db_path: str | Path,
|
||
task_id: str,
|
||
kind: str,
|
||
status: str,
|
||
payload: dict,
|
||
created_ts: str | None = None,
|
||
) -> None:
|
||
"""登记或更新任务状态（payload 为内存中任务 dict 的 JSON 快照）。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(task_id, kind, status, payload_json, created_ts, updated_ts)"
|
||
" VALUES (?, ?, ?, ?, ?, ?)"
|
||
" ON CONFLICT(task_id) DO UPDATE SET status = excluded.status,"
|
||
" payload_json = excluded.payload_json, updated_ts = excluded.updated_ts",
|
||
(task_id, kind, status, json.dumps(payload, ensure_ascii=False),
|
||
created_ts or _now(), _now()))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _task_row(row: sqlite3.Row) -> dict:
|
||
return {"task_id": row["task_id"], "kind": row["kind"], "status": row["status"],
|
||
"payload": json.loads(row["payload_json"] or "{}"),
|
||
"created_ts": row["created_ts"], "updated_ts": row["updated_ts"]}
|
||
|
||
|
||
def get_task(db_path: str | Path, task_id: str) -> dict | None:
|
||
conn = connect(db_path)
|
||
try:
|
||
row = conn.execute("SELECT * FROM tasks WHERE task_id = ?", (task_id,)).fetchone()
|
||
return _task_row(row) if row else None
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def list_tasks(db_path: str | Path) -> list[dict]:
|
||
"""全部任务（按创建时间升序），供重启后恢复内存任务表。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
rows = conn.execute("SELECT * FROM tasks ORDER BY created_ts, rowid").fetchall()
|
||
return [_task_row(r) for r in rows]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
# ---------- 人工状态覆写层（跨图谱重建持久化） ----------
|
||
def _upsert_override(conn: sqlite3.Connection, name: str, payload: dict) -> None:
|
||
conn.execute(
|
||
"INSERT INTO job_overrides(name, payload_json, updated_ts) VALUES (?, ?, ?)"
|
||
" ON CONFLICT(name) DO UPDATE SET payload_json = excluded.payload_json,"
|
||
" updated_ts = excluded.updated_ts",
|
||
(name, json.dumps(payload, ensure_ascii=False), _now()))
|
||
|
||
|
||
def save_override(db_path: str | Path, name: str, payload: dict) -> None:
|
||
"""新增或更新一条人工覆写（按 name upsert）。
|
||
|
||
payload 约定：{"kind": "job_accept", "job": <JobDefinition dump>}
|
||
或 {"kind": "node_update", "node_id": ..., "updates": {...}}。
|
||
"""
|
||
conn = connect(db_path)
|
||
try:
|
||
_upsert_override(conn, name, payload)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def get_override(db_path: str | Path, name: str) -> dict | None:
|
||
conn = connect(db_path)
|
||
try:
|
||
row = conn.execute(
|
||
"SELECT payload_json FROM job_overrides WHERE name = ?", (name,)).fetchone()
|
||
return json.loads(row["payload_json"]) if row else None
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def list_overrides(db_path: str | Path) -> list[dict]:
|
||
"""全部人工覆写（按写入顺序），pipeline 重建后按序重放。"""
|
||
conn = connect(db_path)
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT payload_json FROM job_overrides ORDER BY rowid").fetchall()
|
||
return [json.loads(r["payload_json"]) for r in rows]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def delete_override(db_path: str | Path, name: str) -> None:
|
||
conn = connect(db_path)
|
||
try:
|
||
conn.execute("DELETE FROM job_overrides WHERE name = ?", (name,))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|