Merge branch 'main' of http://192.168.3.110:3000/element.chen/ai-g3sb-backman2.0

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

backend/schema/indexer.py

@@ -22,8 +22,12 @@ class SchemaIndexer:
     功能：
     1. 为表结构构建向量索引
     2. 基于问题的表检索
+<<<<<<< HEAD
     3. 默认使用 Chroma ``PersistentClient``，数据落在 ``persist_dir``（与 ``VECTOR_DB_PATH`` 一致）；
        仅当环境变量 ``SCHEMA_INDEXER_EPHEMERAL=true`` 或构造参数 ``use_ephemeral=True`` 时使用内存客户端。
+=======
+    3. Chroma 使用持久化（PersistentClient），向量与元数据写入 ``persist_dir``，进程重启后可复用
+>>>>>>> 9369563942d8de442508613191670123d5ae9d08
     """
 
     def __init__(
@@ -39,18 +43,30 @@ class SchemaIndexer:
 
         Args:
             embedder: Embedding模型实例
+<<<<<<< HEAD
             persist_dir: Chroma 持久化目录（默认与编排器 ``vector_db_path`` / ``VECTOR_DB_PATH`` 一致）
+=======
+            persist_dir: Chroma 持久化根目录（磁盘路径）
+>>>>>>> 9369563942d8de442508613191670123d5ae9d08
             collection_name: 集合名称
             use_ephemeral: 为 True 时使用内存 Chroma；为 None 时读环境变量 SCHEMA_INDEXER_EPHEMERAL
         """
         self.embedder = embedder
         self.persist_dir = Path(persist_dir)
+        self.persist_dir.mkdir(parents=True, exist_ok=True)
         self.collection_name = collection_name
 
+<<<<<<< HEAD
         env_ephemeral = os.getenv("SCHEMA_INDEXER_EPHEMERAL", "").lower() in (
             "1",
             "true",
             "yes",
+=======
+        # 持久化 Chroma：数据落盘至 persist_dir
+        self.client = chromadb.PersistentClient(
+            path=str(self.persist_dir),
+            settings=ChromaSettings(anonymized_telemetry=False),
+>>>>>>> 9369563942d8de442508613191670123d5ae9d08
         )
         self._chroma_ephemeral = bool(use_ephemeral) if use_ephemeral is not None else env_ephemeral
 
@@ -74,8 +90,13 @@ class SchemaIndexer:
         )
 
         logger.info(
+<<<<<<< HEAD
             f"[OK] 初始化SchemaIndexer({backend_desc}): collection={self.collection_name}, "
             f"count={self.collection.count()}"
+=======
+            f"[OK] 初始化SchemaIndexer(Chroma持久化): collection={self.collection_name}, "
+            f"path={self.persist_dir}"
+>>>>>>> 9369563942d8de442508613191670123d5ae9d08
         )
 
     def ensure_index_for_schema(
@@ -311,5 +332,9 @@ class SchemaIndexer:
             "total_columns": total_columns,
             "avg_columns": total_columns / count if count > 0 else 0,
             "persist_dir": str(self.persist_dir),
+<<<<<<< HEAD
             "chroma_mode": "memory" if self._chroma_ephemeral else "persistent",
+=======
+            "chroma_mode": "persistent",
+>>>>>>> 9369563942d8de442508613191670123d5ae9d08
         }

main.spec

@@ -8,6 +8,35 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules, coll
 import sys
 _REPO_DIR = Path(os.getcwd()).resolve()
 
+
+def _strip_msvc_runtime_binaries(bin_list):
+    """
+    从 ``collect_dynamic_libs`` 等额外收集的 binaries 中移除 MSVC 运行库 DLL。
+
+    chromadb / hnswlib 等 wheel 常自带 ``MSVCP140.dll`` 等；与 PyInstaller 已为
+    ``python.exe`` 解析到的 CRT 一并打入单文件包时，易在运行期混用多份 CRT
+    导致 ``MSVCP140.dll`` 内 ``0xc0000005``。统一依赖解释器自带的 CRT 集更安全。
+    """
+    skip = frozenset({
+        'msvcp140.dll',
+        'msvcp140_1.dll',
+        'msvcp140_2.dll',
+        'msvcp140_atomic_wait.dll',
+        'msvcp140_codecvt_ids.dll',
+        'vcruntime140.dll',
+        'vcruntime140_1.dll',
+        'concrt140.dll',
+    })
+    out = []
+    for item in bin_list:
+        dest = item[0]
+        base = Path(dest).name.lower()
+        if base in skip:
+            continue
+        out.append(item)
+    return out
+
+
 # 根目录执行：pyinstaller --clean main.spec
 #
 # ---------------------------------------------------------------------------
@@ -45,6 +74,62 @@ try:
 except Exception:
     pass
 
+binaries = _strip_msvc_runtime_binaries(binaries)
+
+
+def _dedupe_msvc_runtime_in_analysis_binaries(toc):
+    """
+    对 ``Analysis`` 产出的 ``a.binaries`` 去重：同名 MSVC 运行库 DLL 只保留一份。
+
+    PyInstaller 会从 Python 安装目录、numpy、torch、chromadb 等多个依赖各自打入
+    同名 ``MSVCP140.dll`` 等；单文件解压到 ``_MEI*`` 后若出现多份或加载顺序混乱，
+    易在 ``MSVCP140.dll`` 内触发 ``0xc0000005``。优先保留 **源路径位于**
+    ``sys.base_prefix`` 下的条目（与解释器主 CRT 一致）。
+    """
+    vc_names = frozenset({
+        'msvcp140.dll',
+        'msvcp140_1.dll',
+        'msvcp140_2.dll',
+        'msvcp140_atomic_wait.dll',
+        'msvcp140_codecvt_ids.dll',
+        'vcruntime140.dll',
+        'vcruntime140_1.dll',
+        'concrt140.dll',
+    })
+    try:
+        base_root = str(Path(sys.base_prefix).resolve()).lower()
+    except OSError:
+        base_root = str(Path(sys.base_prefix)).lower()
+
+    rest = []
+    vc_groups = {}
+    for item in toc:
+        dest = item[0]
+        bname = Path(dest).name.lower()
+        if bname not in vc_names:
+            rest.append(item)
+            continue
+        vc_groups.setdefault(bname, []).append(item)
+
+    picked = []
+    for bname, items in vc_groups.items():
+        if len(items) == 1:
+            picked.append(items[0])
+            continue
+
+        def _score(it):
+            try:
+                sp = str(Path(it[1]).resolve()).lower()
+            except OSError:
+                sp = str(it[1]).lower()
+            under = 0 if sp.startswith(base_root) else 1
+            return (under, len(sp))
+
+        items_sorted = sorted(items, key=_score)
+        picked.append(items_sorted[0])
+
+    return rest + picked
+
 # 收集所有需要的隐藏导入
 hiddenimports = [
     # Web 框架
@@ -103,6 +188,11 @@ hiddenimports = [
 
     # 向量数据库 - 使用 collect_submodules 自动收集所有子模块
 
+    # Embedding 相关
+    'sentencepiece',
+    'accelerate',
+    'safetensors',
+
     # LLM API
     'openai',
 
@@ -146,11 +236,15 @@ a = Analysis(
     hiddenimports=hiddenimports,
     hookspath=[],
     hooksconfig={},
-    runtime_hooks=[],
+    runtime_hooks=[str(_REPO_DIR / 'packaging' / 'pyi_rth_crt_path.py')],
     excludes=excludes,
     noarchive=False,
     optimize=0,
 )
+
+# 去掉 Analysis 自动收集到的重复 MSVC 运行库（比仅过滤手动 binaries 更关键）
+a.binaries = _dedupe_msvc_runtime_in_analysis_binaries(list(a.binaries))
+
 pyz = PYZ(a.pure)
 
 exe = EXE(

packaging/pyi_rth_crt_path.py

@@ -0,0 +1,89 @@
+"""
+PyInstaller 运行时钩子：固定 MSVC 运行库加载顺序，再注册 DLL 搜索路径。
+
+打包阶段在 strip 掉各 wheel 自带 CRT 后，会 **仅嵌入** 构建所用 Python ``DLLs``
+下的 ``vcruntime140.dll`` / ``vcruntime140_1.dll`` / ``msvcp140.dll`` 至 ``_MEIPASS``。
+本钩子在导入 chroma/torch 等扩展 **之前**，用 ``ctypes.WinDLL(绝对路径)`` 从 **同一目录**
+（优先 exe 旁若用户自行放置了完整三件套，否则 ``_MEIPASS``）预加载，避免进程落到
+目标机 ``System32`` 里较旧版本（如 14.29）再与 native 扩展不兼容触发 ``0xc0000005``。
+
+随后再 ``add_dll_directory`` exe 目录与 ``_MEIPASS``，并调整 ``PATH``。不使用 ``SetDllDirectoryW``。
+"""
+
+
+def _preload_msvc_crt_once():
+    """若 exe 目录或 ``_MEIPASS`` 下存在 ``msvcp140.dll``，则按依赖顺序预加载该目录内 CRT。"""
+    import ctypes
+    import sys
+    from pathlib import Path
+
+    if sys.platform != "win32" or not getattr(sys, "frozen", False):
+        return
+
+    order = ("vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll")
+    search_dirs = []
+    try:
+        exe_parent = Path(sys.executable).resolve().parent
+        search_dirs.append(exe_parent)
+    except OSError:
+        pass
+    meipass = getattr(sys, "_MEIPASS", None)
+    if meipass:
+        search_dirs.append(Path(meipass))
+
+    for d in search_dirs:
+        if not d.is_dir():
+            continue
+        if not (d / "msvcp140.dll").is_file():
+            continue
+        for name in order:
+            p = d / name
+            if p.is_file():
+                try:
+                    ctypes.WinDLL(str(p))
+                except OSError:
+                    pass
+        return
+
+
+def _apply_dll_search_paths():
+    """将 exe 目录与 ``_MEIPASS`` 注册为 DLL 目录，并置于 PATH 最前（exe 目录优先）。"""
+    import os
+    import sys
+    from pathlib import Path
+
+    if not getattr(sys, "frozen", False):
+        return
+
+    dirs = []
+    try:
+        exe_dir = str(Path(sys.executable).resolve().parent)
+        if exe_dir:
+            dirs.append(exe_dir)
+    except OSError:
+        pass
+
+    meipass = getattr(sys, "_MEIPASS", None)
+    if meipass:
+        dirs.append(meipass)
+
+    for d in dirs:
+        if not d:
+            continue
+        try:
+            if hasattr(os, "add_dll_directory"):
+                os.add_dll_directory(d)
+        except OSError:
+            pass
+
+    path = os.environ.get("PATH", "")
+    prefix = os.pathsep.join([d for d in dirs if d])
+    if not prefix:
+        return
+    if path.startswith(prefix):
+        return
+    os.environ["PATH"] = f"{prefix}{os.pathsep}{path}"
+
+
+_preload_msvc_crt_once()
+_apply_dll_search_paths()