element.chen/ai-g3sb-backman2.0
1
0
Fork 0
Code Pull Requests Activity

加入日志管理

Browse Source
This commit is contained in:
陈辅元
2026-04-16 10:53:10 +08:00
parent 284e02f41c
commit 695356a496
19 changed files with 582 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/__pycache__/main.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/agents/__pycache__/orchestrator.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/agents/orchestrator.py
+49 -6
View File
@@ -182,7 +182,11 @@ class Text2SQLOrchestrator:
indexer.ensure_index_for_schema(self.schema_manager)
# 检索
logger.info(f"[Orchestrator] 开始向量检索: query='{question[:50]}...'")
logger.info(
"[Orchestrator] 开始向量检索: query_chars=%s query_preview=%r",
len(question or ""),
(question or "")[:200] + ("…" if len(question or "") > 200 else ""),
)
results = indexer.search(
query=question,
top_k=top_k,
@@ -190,7 +194,15 @@ class Text2SQLOrchestrator:
)
candidate_tables = [r["table_name"] for r in results]
logger.debug(f"粗筛候选表：{candidate_tables[:10]}...（共{len(candidate_tables)}个）")
scored = [
(r["table_name"], round(float(r.get("score", 0.0)), 4))
for r in results[: min(25, len(results))]
]
logger.info(
"[Orchestrator] 向量粗筛: 命中=%s 张（阈值内），表名+分: %s",
len(candidate_tables),
scored,
)
return candidate_tables
except Exception as e:
@@ -239,7 +251,13 @@ class Text2SQLOrchestrator:
# 限制数量
relevant_tables = relevant_tables[:max_tables]
logger.info(f"LLM精筛选中表：{relevant_tables}")
rs = (reasoning or "").strip()
logger.info(
"LLM精筛选中表：%s | reasoning_chars=%s reasoning_preview=%r",
relevant_tables,
len(rs),
rs[:600] + ("…" if len(rs) > 600 else ""),
)
return relevant_tables, reasoning
_BROKER_KEYWORDS_CN = ("对手方", "经纪商", "券商", "對手方")
@@ -361,7 +379,12 @@ class Text2SQLOrchestrator:
for i, ex in enumerate(examples)
])
schema_str = f"参考以下相似示例的SQL编写风格：\n\n{examples_prompt}\n\n【当前Schema】\n{schema_str}"
logger.debug(f"已注入 {len(examples)} 个few-shot示例: {[ex.qid for ex in examples]}")
logger.info(
"已注入 %s 个 few-shot 示例: qid=%s question_zh_preview=%r",
len(examples),
[ex.qid for ex in examples],
[((ex.question_zh or "")[:80] + "…") if len(ex.question_zh or "") > 80 else (ex.question_zh or "") for ex in examples],
)
except Exception as e:
logger.warning(f"Few-shot检索失败: {e}")
@@ -419,7 +442,9 @@ class Text2SQLOrchestrator:
sql = normalize_sql_for_dialect(sql, dialect)
logger.debug(f"生成的SQL：{sql[:200]}...")
lim = 12000
body = sql if len(sql) <= lim else sql[:lim] + "\n…(日志已截断)"
logger.info("生成的SQL（chars=%s）:\n%s", len(sql), body)
return sql
def _validate_sql(
@@ -562,6 +587,19 @@ class Text2SQLOrchestrator:
empty_feedback = (empty_feedback or prefix) + follow
is_valid = len(errors) == 0
logger.info(
"[validate] 程序+探针+LLM 汇总: valid=%s err_count=%s warn_count=%s "
"db_execution_status=%s sql_chars=%s",
is_valid,
len(errors),
len(warnings),
db_execution_status,
len(sql or ""),
)
if errors:
logger.info("[validate] errors 预览: %s", errors[:5])
if warnings:
logger.info("[validate] warnings: %s", warnings[:5])
return is_valid, errors, warnings, db_execution_status, empty_feedback
def generate(
@@ -608,7 +646,12 @@ class Text2SQLOrchestrator:
dc_raw = (dialog_context or "").strip()
retrieval_question = self._merge_dialog_for_model(dc_raw, question, 4000)
linker_question = self._merge_dialog_for_model(dc_raw, question, 6000)
logger.info(f"[GEN] 开始生成SQL：{question[:50]}...")
logger.info(
"[GEN] 开始生成SQL: question_chars=%s preview=%r dialog_context_chars=%s",
len(question or ""),
(question or "")[:300] + ("…" if len(question or "") > 300 else ""),
len(dc_raw) if dc_raw else 0,
)
attempt = 0
last_sql = None
backend/llm/__pycache__/deepseek_client.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/schema/__pycache__/indexer.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/schema/indexer.py
+9 -21
View File
@@ -22,12 +22,8 @@ class SchemaIndexer:
功能：
1. 为表结构构建向量索引
2. 基于问题的表检索
<<<<<<< HEAD
3. 默认使用 Chroma ``PersistentClient``，数据落在 ``persist_dir``（与 ``VECTOR_DB_PATH`` 一致）；
仅当环境变量 ``SCHEMA_INDEXER_EPHEMERAL=true`` 或构造参数 ``use_ephemeral=True`` 时使用内存客户端。
=======
3. Chroma 使用持久化（PersistentClient），向量与元数据写入 ``persist_dir``，进程重启后可复用
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
"""
def __init__(
@@ -43,11 +39,7 @@ class SchemaIndexer:
Args:
embedder: Embedding模型实例
<<<<<<< HEAD
persist_dir: Chroma 持久化目录（默认与编排器 ``vector_db_path`` / ``VECTOR_DB_PATH`` 一致）
=======
persist_dir: Chroma 持久化根目录（磁盘路径）
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
collection_name: 集合名称
use_ephemeral: 为 True 时使用内存 Chroma；为 None 时读环境变量 SCHEMA_INDEXER_EPHEMERAL
"""
@@ -56,17 +48,15 @@ class SchemaIndexer:
self.persist_dir.mkdir(parents=True, exist_ok=True)
self.collection_name = collection_name
<<<<<<< HEAD
env_ephemeral = os.getenv("SCHEMA_INDEXER_EPHEMERAL", "").lower() in (
"1",
"true",
"yes",
=======
)
# 持久化 Chroma：数据落盘至 persist_dir
self.client = chromadb.PersistentClient(
path=str(self.persist_dir),
settings=ChromaSettings(anonymized_telemetry=False),
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
)
self._chroma_ephemeral = bool(use_ephemeral) if use_ephemeral is not None else env_ephemeral
@@ -90,13 +80,8 @@ class SchemaIndexer:
)
logger.info(
<<<<<<< HEAD
f"[OK] 初始化SchemaIndexer({backend_desc}): collection={self.collection_name}, "
f"count={self.collection.count()}"
=======
f"[OK] 初始化SchemaIndexer(Chroma持久化): collection={self.collection_name}, "
f"path={self.persist_dir}"
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
)
def ensure_index_for_schema(
@@ -255,7 +240,14 @@ class SchemaIndexer:
"rank": idx + 1,
})
logger.debug(f"检索 '{query[:50]}...' -> 找到{len(formatted)}个相关表（阈值={score_threshold}）")
top = [(x["table_name"], round(float(x.get("score", 0.0)), 4)) for x in formatted[:15]]
logger.info(
"Schema 向量检索: query_chars=%s 命中=%s（阈值=%s）top=%s",
len(query or ""),
len(formatted),
score_threshold,
top,
)
return formatted
def search_by_table_names(self, table_names: List[str]) -> List[Dict]:
@@ -332,9 +324,5 @@ class SchemaIndexer:
"total_columns": total_columns,
"avg_columns": total_columns / count if count > 0 else 0,
"persist_dir": str(self.persist_dir),
<<<<<<< HEAD
"chroma_mode": "memory" if self._chroma_ephemeral else "persistent",
=======
"chroma_mode": "persistent",
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
}
backend/utils/__pycache__/dialog_classifier.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/utils/__pycache__/fewshot_selector.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/utils/__pycache__/repo_logging.cpython-312.pyc
BIN
View File
Binary file not shown.
backend/utils/dialog_classifier.py
+14 -10
View File
@@ -136,28 +136,28 @@ def _classify_dialog_rules(
)
if _SQL_OR_QUERY_HINT_RE.search(t):
logger.debug("[dialog] intent=text2sql (query/business hint)")
logger.info("[dialog] intent=text2sql (query/business hint) preview=%r", user_text[:120])
return DialogClassifyResult(DialogIntent.TEXT2SQL, None)
core = _strip_trailing_punct(t)
if core.casefold() in _CHITCHAT_KEYS:
logger.debug("[dialog] intent=conversation (chitchat phrase)")
logger.info("[dialog] intent=conversation (chitchat phrase) core=%r", core)
return DialogClassifyResult(
DialogIntent.CONVERSATION, reply_suggestion=DEFAULT_CONVERSATION_REPLY
)
if _META_QUESTION_RE.search(t):
logger.debug("[dialog] intent=conversation (meta question)")
logger.info("[dialog] intent=conversation (meta question) preview=%r", user_text[:120])
return DialogClassifyResult(
DialogIntent.CONVERSATION,
reply_suggestion=DEFAULT_CONVERSATION_REPLY,
)
if last_turn_was_data_query:
logger.debug("[dialog] intent=text2sql (follow-up after data query)")
logger.info("[dialog] intent=text2sql (follow-up after data query) preview=%r", user_text[:120])
return DialogClassifyResult(DialogIntent.TEXT2SQL, None)
logger.debug("[dialog] intent=text2sql (default)")
logger.info("[dialog] intent=text2sql (default rules) preview=%r", user_text[:120])
return DialogClassifyResult(DialogIntent.TEXT2SQL, None)
@@ -208,11 +208,15 @@ def _classify_dialog_llm(
"若上一版 SQL 或结果不符合预期，请具体说明：希望增加/修改哪些条件、"
"时间范围或统计维度，以便重新生成。"
)
logger.info("[dialog] intent=conversation (LLM)")
logger.info(
"[dialog] intent=conversation (LLM) reply_chars=%s reply_preview=%r",
len(reply),
reply[:300] + ("…" if len(reply) > 300 else ""),
)
return DialogClassifyResult(DialogIntent.CONVERSATION, reply_suggestion=reply)
if intent_s == "text2sql":
logger.info("[dialog] intent=text2sql (LLM)")
logger.info("[dialog] intent=text2sql (LLM) user_preview=%r", user_text[:200])
return DialogClassifyResult(DialogIntent.TEXT2SQL, None)
raise ValueError(f"unexpected intent field: {intent_s!r}")
@@ -254,18 +258,18 @@ def classify_dialog(
)
if _SQL_OR_QUERY_HINT_RE.search(t):
logger.debug("[dialog] intent=text2sql (query/business hint, hybrid fast)")
logger.info("[dialog] intent=text2sql (hybrid fast: query hint) preview=%r", user_text[:120])
return DialogClassifyResult(DialogIntent.TEXT2SQL, None)
core = _strip_trailing_punct(t)
if core.casefold() in _CHITCHAT_KEYS:
logger.debug("[dialog] intent=conversation (chitchat phrase, hybrid fast)")
logger.info("[dialog] intent=conversation (hybrid fast: chitchat) core=%r", core)
return DialogClassifyResult(
DialogIntent.CONVERSATION, reply_suggestion=DEFAULT_CONVERSATION_REPLY
)
if _META_QUESTION_RE.search(t):
logger.debug("[dialog] intent=conversation (meta question, hybrid fast)")
logger.info("[dialog] intent=conversation (hybrid fast: meta) preview=%r", user_text[:120])
return DialogClassifyResult(
DialogIntent.CONVERSATION,
reply_suggestion=DEFAULT_CONVERSATION_REPLY,
backend/utils/repo_logging.py
+80
View File
@@ -0,0 +1,80 @@
"""
Text2SQL API 仓库级日志：控制台 + 单一磁盘文件（每次进程启动清空 logs/*.log 后重写）。
由 ``api_server`` 在导入早期调用；与 ``uvicorn.run(..., log_config=False)`` 配合，
使 uvicorn / FastAPI 的日志经 root 统一落到文件。
"""
from __future__ import annotations
import logging
from pathlib import Path
def configure_text2sql_api_logging(repo_root: Path) -> Path:
"""
配置 root logger：控制台（INFO，简短格式）+ ``logs/text2sql_api.log``（INFO，含文件名/行号/函数）。
每次调用会删除 ``logs`` 目录下所有 ``*.log``，再新建 ``text2sql_api.log``（覆盖写），
保证一次运行仅保留一个日志文件。
Returns:
主日志文件绝对路径。
"""
log_dir = (repo_root / "logs").resolve()
log_dir.mkdir(parents=True, exist_ok=True)
for p in log_dir.glob("*.log"):
try:
p.unlink()
except OSError:
pass
log_path = log_dir / "text2sql_api.log"
root = logging.getLogger()
root.setLevel(logging.INFO)
for h in list(root.handlers):
root.removeHandler(h)
try:
h.close()
except Exception:
pass
fmt_console = logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
fmt_file = logging.Formatter(
"%(asctime)s %(levelname)-7s [%(name)s] %(filename)s:%(lineno)d %(funcName)s() | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
fh = logging.FileHandler(log_path, mode="w", encoding="utf-8")
fh.setLevel(logging.INFO)
fh.setFormatter(fmt_file)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(fmt_console)
root.addHandler(fh)
root.addHandler(ch)
# 依赖库降噪（避免 httpx 每条请求刷屏；Chroma 遥测与新版 posthog 不兼容时会打 ERROR）
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.WARNING)
for _name in (
"chromadb.telemetry",
"chromadb.telemetry.product.posthog",
"posthog",
):
logging.getLogger(_name).disabled = True
# Uvicorn 默认自带 handler；与 log_config=False 合用时改为只往 root 冒泡
for name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
lg = logging.getLogger(name)
lg.handlers.clear()
lg.propagate = True
logging.getLogger(__name__).info("日志文件: %s", log_path)
return log_path