element.chen/ai-g3sb-backman2.0
1
0
Fork 0
Code Pull Requests Activity

Merge branch 'main' of http://192.168.3.110:3000/element.chen/ai-g3sb-backman2.0

Browse Source
This commit is contained in:
陈辅元
2026-04-16 09:50:26 +08:00
parent df33657099 9369563942
commit 284e02f41c
7 changed files with 224 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/.gitignore
Generated
+8
View File
@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
backend/schema/indexer.py
+25
View File
@@ -22,8 +22,12 @@ class SchemaIndexer:
功能：
1. 为表结构构建向量索引
2. 基于问题的表检索
<<<<<<< HEAD
3. 默认使用 Chroma ``PersistentClient``，数据落在 ``persist_dir``（与 ``VECTOR_DB_PATH`` 一致）；
仅当环境变量 ``SCHEMA_INDEXER_EPHEMERAL=true`` 或构造参数 ``use_ephemeral=True`` 时使用内存客户端。
=======
3. Chroma 使用持久化（PersistentClient），向量与元数据写入 ``persist_dir``，进程重启后可复用
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
"""
def __init__(
@@ -39,18 +43,30 @@ class SchemaIndexer:
Args:
embedder: Embedding模型实例
<<<<<<< HEAD
persist_dir: Chroma 持久化目录（默认与编排器 ``vector_db_path`` / ``VECTOR_DB_PATH`` 一致）
=======
persist_dir: Chroma 持久化根目录（磁盘路径）
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
collection_name: 集合名称
use_ephemeral: 为 True 时使用内存 Chroma；为 None 时读环境变量 SCHEMA_INDEXER_EPHEMERAL
"""
self.embedder = embedder
self.persist_dir = Path(persist_dir)
self.persist_dir.mkdir(parents=True, exist_ok=True)
self.collection_name = collection_name
<<<<<<< HEAD
env_ephemeral = os.getenv("SCHEMA_INDEXER_EPHEMERAL", "").lower() in (
"1",
"true",
"yes",
=======
# 持久化 Chroma：数据落盘至 persist_dir
self.client = chromadb.PersistentClient(
path=str(self.persist_dir),
settings=ChromaSettings(anonymized_telemetry=False),
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
)
self._chroma_ephemeral = bool(use_ephemeral) if use_ephemeral is not None else env_ephemeral
@@ -74,8 +90,13 @@ class SchemaIndexer:
)
logger.info(
<<<<<<< HEAD
f"[OK] 初始化SchemaIndexer({backend_desc}): collection={self.collection_name}, "
f"count={self.collection.count()}"
=======
f"[OK] 初始化SchemaIndexer(Chroma持久化): collection={self.collection_name}, "
f"path={self.persist_dir}"
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
)
def ensure_index_for_schema(
@@ -311,5 +332,9 @@ class SchemaIndexer:
"total_columns": total_columns,
"avg_columns": total_columns / count if count > 0 else 0,
"persist_dir": str(self.persist_dir),
<<<<<<< HEAD
"chroma_mode": "memory" if self._chroma_ephemeral else "persistent",
=======
"chroma_mode": "persistent",
>>>>>>> 9369563942d8de442508613191670123d5ae9d08
}
deploy/msvcp140.dll
BIN
View File
Binary file not shown.
deploy/vcruntime140.dll
BIN
View File
Binary file not shown.
deploy/vcruntime140_1.dll
BIN
View File
Binary file not shown.
main.spec
+102 -8
View File
@@ -8,6 +8,35 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules, coll
import sys
_REPO_DIR = Path(os.getcwd()).resolve()
def _strip_msvc_runtime_binaries(bin_list):
"""
从 ``collect_dynamic_libs`` 等额外收集的 binaries 中移除 MSVC 运行库 DLL。
chromadb / hnswlib 等 wheel 常自带 ``MSVCP140.dll`` 等；与 PyInstaller 已为
``python.exe`` 解析到的 CRT 一并打入单文件包时，易在运行期混用多份 CRT
导致 ``MSVCP140.dll`` 内 ``0xc0000005``。统一依赖解释器自带的 CRT 集更安全。
"""
skip = frozenset({
'msvcp140.dll',
'msvcp140_1.dll',
'msvcp140_2.dll',
'msvcp140_atomic_wait.dll',
'msvcp140_codecvt_ids.dll',
'vcruntime140.dll',
'vcruntime140_1.dll',
'concrt140.dll',
})
out = []
for item in bin_list:
dest = item[0]
base = Path(dest).name.lower()
if base in skip:
continue
out.append(item)
return out
# 根目录执行：pyinstaller --clean main.spec
#
# ---------------------------------------------------------------------------
@@ -45,6 +74,62 @@ try:
except Exception:
pass
binaries = _strip_msvc_runtime_binaries(binaries)
def _dedupe_msvc_runtime_in_analysis_binaries(toc):
"""
对 ``Analysis`` 产出的 ``a.binaries`` 去重：同名 MSVC 运行库 DLL 只保留一份。
PyInstaller 会从 Python 安装目录、numpy、torch、chromadb 等多个依赖各自打入
同名 ``MSVCP140.dll`` 等；单文件解压到 ``_MEI*`` 后若出现多份或加载顺序混乱，
易在 ``MSVCP140.dll`` 内触发 ``0xc0000005``。优先保留 **源路径位于**
``sys.base_prefix`` 下的条目（与解释器主 CRT 一致）。
"""
vc_names = frozenset({
'msvcp140.dll',
'msvcp140_1.dll',
'msvcp140_2.dll',
'msvcp140_atomic_wait.dll',
'msvcp140_codecvt_ids.dll',
'vcruntime140.dll',
'vcruntime140_1.dll',
'concrt140.dll',
})
try:
base_root = str(Path(sys.base_prefix).resolve()).lower()
except OSError:
base_root = str(Path(sys.base_prefix)).lower()
rest = []
vc_groups = {}
for item in toc:
dest = item[0]
bname = Path(dest).name.lower()
if bname not in vc_names:
rest.append(item)
continue
vc_groups.setdefault(bname, []).append(item)
picked = []
for bname, items in vc_groups.items():
if len(items) == 1:
picked.append(items[0])
continue
def _score(it):
try:
sp = str(Path(it[1]).resolve()).lower()
except OSError:
sp = str(it[1]).lower()
under = 0 if sp.startswith(base_root) else 1
return (under, len(sp))
items_sorted = sorted(items, key=_score)
picked.append(items_sorted[0])
return rest + picked
# 收集所有需要的隐藏导入
hiddenimports = [
# Web 框架
@@ -57,7 +142,7 @@ hiddenimports = [
'uvicorn.protocols.http.auto',
'uvicorn.protocols.websockets.auto',
'uvicorn.loops.auto',
# Backend 模块（重要！）
'main', # backend/main.py
'backend.main',
@@ -90,7 +175,7 @@ hiddenimports = [
'backend.utils.question_locale',
'backend.utils.sql_parser',
'backend.utils.validators',
# 数据库相关
'pymssql',
'sqlalchemy',
@@ -100,27 +185,32 @@ hiddenimports = [
'sqlglot',
'sqlglot.dialects',
'sqlparse',
# 向量数据库 - 使用 collect_submodules 自动收集所有子模块
# Embedding 相关
'sentencepiece',
'accelerate',
'safetensors',
# LLM API
'openai',
# 数据处理
'numpy',
'jieba',
# Pydantic
'pydantic',
'pydantic_settings',
'dotenv',
# 工具库
'tenacity',
'structlog',
'tqdm',
'camel_ai',
# 标准库中可能被遗漏的
'email.mime.text',
'email.mime.multipart',
@@ -146,11 +236,15 @@ a = Analysis(
hiddenimports=hiddenimports,
hookspath=[],
hooksconfig={},
runtime_hooks=[],
runtime_hooks=[str(_REPO_DIR / 'packaging' / 'pyi_rth_crt_path.py')],
excludes=excludes,
noarchive=False,
optimize=0,
)
# 去掉 Analysis 自动收集到的重复 MSVC 运行库（比仅过滤手动 binaries 更关键）
a.binaries = _dedupe_msvc_runtime_in_analysis_binaries(list(a.binaries))
pyz = PYZ(a.pure)
exe = EXE(
packaging/pyi_rth_crt_path.py
+89
View File
@@ -0,0 +1,89 @@
"""
PyInstaller 运行时钩子：固定 MSVC 运行库加载顺序，再注册 DLL 搜索路径。
打包阶段在 strip 掉各 wheel 自带 CRT 后，会 **仅嵌入** 构建所用 Python ``DLLs``
下的 ``vcruntime140.dll`` / ``vcruntime140_1.dll`` / ``msvcp140.dll`` 至 ``_MEIPASS``。
本钩子在导入 chroma/torch 等扩展 **之前**，用 ``ctypes.WinDLL(绝对路径)`` 从 **同一目录**
（优先 exe 旁若用户自行放置了完整三件套，否则 ``_MEIPASS``）预加载，避免进程落到
目标机 ``System32`` 里较旧版本（如 14.29）再与 native 扩展不兼容触发 ``0xc0000005``。
随后再 ``add_dll_directory`` exe 目录与 ``_MEIPASS``，并调整 ``PATH``。不使用 ``SetDllDirectoryW``。
"""
def _preload_msvc_crt_once():
"""若 exe 目录或 ``_MEIPASS`` 下存在 ``msvcp140.dll``，则按依赖顺序预加载该目录内 CRT。"""
import ctypes
import sys
from pathlib import Path
if sys.platform != "win32" or not getattr(sys, "frozen", False):
return
order = ("vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll")
search_dirs = []
try:
exe_parent = Path(sys.executable).resolve().parent
search_dirs.append(exe_parent)
except OSError:
pass
meipass = getattr(sys, "_MEIPASS", None)
if meipass:
search_dirs.append(Path(meipass))
for d in search_dirs:
if not d.is_dir():
continue
if not (d / "msvcp140.dll").is_file():
continue
for name in order:
p = d / name
if p.is_file():
try:
ctypes.WinDLL(str(p))
except OSError:
pass
return
def _apply_dll_search_paths():
"""将 exe 目录与 ``_MEIPASS`` 注册为 DLL 目录，并置于 PATH 最前（exe 目录优先）。"""
import os
import sys
from pathlib import Path
if not getattr(sys, "frozen", False):
return
dirs = []
try:
exe_dir = str(Path(sys.executable).resolve().parent)
if exe_dir:
dirs.append(exe_dir)
except OSError:
pass
meipass = getattr(sys, "_MEIPASS", None)
if meipass:
dirs.append(meipass)
for d in dirs:
if not d:
continue
try:
if hasattr(os, "add_dll_directory"):
os.add_dll_directory(d)
except OSError:
pass
path = os.environ.get("PATH", "")
prefix = os.pathsep.join([d for d in dirs if d])
if not prefix:
return
if path.startswith(prefix):
return
os.environ["PATH"] = f"{prefix}{os.pathsep}{path}"
_preload_msvc_crt_once()
_apply_dll_search_paths()