diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/backend/schema/indexer.py b/backend/schema/indexer.py index 21d34b6..3613ef4 100644 --- a/backend/schema/indexer.py +++ b/backend/schema/indexer.py @@ -22,8 +22,12 @@ class SchemaIndexer: 功能： 1. 为表结构构建向量索引 2. 基于问题的表检索 +<<<<<<< HEAD 3. 默认使用 Chroma ``PersistentClient``，数据落在 ``persist_dir``（与 ``VECTOR_DB_PATH`` 一致）； 仅当环境变量 ``SCHEMA_INDEXER_EPHEMERAL=true`` 或构造参数 ``use_ephemeral=True`` 时使用内存客户端。 +======= + 3. Chroma 使用持久化（PersistentClient），向量与元数据写入 ``persist_dir``，进程重启后可复用 +>>>>>>> 9369563942d8de442508613191670123d5ae9d08 """ def __init__( @@ -39,18 +43,30 @@ class SchemaIndexer: Args: embedder: Embedding模型实例 +<<<<<<< HEAD persist_dir: Chroma 持久化目录（默认与编排器 ``vector_db_path`` / ``VECTOR_DB_PATH`` 一致） +======= + persist_dir: Chroma 持久化根目录（磁盘路径） +>>>>>>> 9369563942d8de442508613191670123d5ae9d08 collection_name: 集合名称 use_ephemeral: 为 True 时使用内存 Chroma；为 None 时读环境变量 SCHEMA_INDEXER_EPHEMERAL """ self.embedder = embedder self.persist_dir = Path(persist_dir) + self.persist_dir.mkdir(parents=True, exist_ok=True) self.collection_name = collection_name +<<<<<<< HEAD env_ephemeral = os.getenv("SCHEMA_INDEXER_EPHEMERAL", "").lower() in ( "1", "true", "yes", +======= + # 持久化 Chroma：数据落盘至 persist_dir + self.client = chromadb.PersistentClient( + path=str(self.persist_dir), + settings=ChromaSettings(anonymized_telemetry=False), +>>>>>>> 9369563942d8de442508613191670123d5ae9d08 ) self._chroma_ephemeral = bool(use_ephemeral) if use_ephemeral is not None else env_ephemeral @@ -74,8 +90,13 @@ class SchemaIndexer: ) logger.info( +<<<<<<< HEAD f"[OK] 初始化SchemaIndexer({backend_desc}): collection={self.collection_name}, " f"count={self.collection.count()}" +======= + f"[OK] 初始化SchemaIndexer(Chroma持久化): collection={self.collection_name}, " + f"path={self.persist_dir}" +>>>>>>> 9369563942d8de442508613191670123d5ae9d08 ) def ensure_index_for_schema( @@ -311,5 +332,9 @@ class SchemaIndexer: "total_columns": total_columns, "avg_columns": total_columns / count if count > 0 else 0, "persist_dir": str(self.persist_dir), +<<<<<<< HEAD "chroma_mode": "memory" if self._chroma_ephemeral else "persistent", +======= + "chroma_mode": "persistent", +>>>>>>> 9369563942d8de442508613191670123d5ae9d08 } diff --git a/deploy/msvcp140.dll b/deploy/msvcp140.dll new file mode 100644 index 0000000..bea9c37 Binary files /dev/null and b/deploy/msvcp140.dll differ diff --git a/deploy/vcruntime140.dll b/deploy/vcruntime140.dll new file mode 100644 index 0000000..a9ed5c4 Binary files /dev/null and b/deploy/vcruntime140.dll differ diff --git a/deploy/vcruntime140_1.dll b/deploy/vcruntime140_1.dll new file mode 100644 index 0000000..cb34ab6 Binary files /dev/null and b/deploy/vcruntime140_1.dll differ diff --git a/main.spec b/main.spec index 855febe..e4ba35d 100644 --- a/main.spec +++ b/main.spec @@ -8,6 +8,35 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules, coll import sys _REPO_DIR = Path(os.getcwd()).resolve() + +def _strip_msvc_runtime_binaries(bin_list): + """ + 从 ``collect_dynamic_libs`` 等额外收集的 binaries 中移除 MSVC 运行库 DLL。 + + chromadb / hnswlib 等 wheel 常自带 ``MSVCP140.dll`` 等；与 PyInstaller 已为 + ``python.exe`` 解析到的 CRT 一并打入单文件包时，易在运行期混用多份 CRT + 导致 ``MSVCP140.dll`` 内 ``0xc0000005``。统一依赖解释器自带的 CRT 集更安全。 + """ + skip = frozenset({ + 'msvcp140.dll', + 'msvcp140_1.dll', + 'msvcp140_2.dll', + 'msvcp140_atomic_wait.dll', + 'msvcp140_codecvt_ids.dll', + 'vcruntime140.dll', + 'vcruntime140_1.dll', + 'concrt140.dll', + }) + out = [] + for item in bin_list: + dest = item[0] + base = Path(dest).name.lower() + if base in skip: + continue + out.append(item) + return out + + # 根目录执行：pyinstaller --clean main.spec # # --------------------------------------------------------------------------- @@ -45,6 +74,62 @@ try: except Exception: pass +binaries = _strip_msvc_runtime_binaries(binaries) + + +def _dedupe_msvc_runtime_in_analysis_binaries(toc): + """ + 对 ``Analysis`` 产出的 ``a.binaries`` 去重：同名 MSVC 运行库 DLL 只保留一份。 + + PyInstaller 会从 Python 安装目录、numpy、torch、chromadb 等多个依赖各自打入 + 同名 ``MSVCP140.dll`` 等；单文件解压到 ``_MEI*`` 后若出现多份或加载顺序混乱， + 易在 ``MSVCP140.dll`` 内触发 ``0xc0000005``。优先保留 **源路径位于** + ``sys.base_prefix`` 下的条目（与解释器主 CRT 一致）。 + """ + vc_names = frozenset({ + 'msvcp140.dll', + 'msvcp140_1.dll', + 'msvcp140_2.dll', + 'msvcp140_atomic_wait.dll', + 'msvcp140_codecvt_ids.dll', + 'vcruntime140.dll', + 'vcruntime140_1.dll', + 'concrt140.dll', + }) + try: + base_root = str(Path(sys.base_prefix).resolve()).lower() + except OSError: + base_root = str(Path(sys.base_prefix)).lower() + + rest = [] + vc_groups = {} + for item in toc: + dest = item[0] + bname = Path(dest).name.lower() + if bname not in vc_names: + rest.append(item) + continue + vc_groups.setdefault(bname, []).append(item) + + picked = [] + for bname, items in vc_groups.items(): + if len(items) == 1: + picked.append(items[0]) + continue + + def _score(it): + try: + sp = str(Path(it[1]).resolve()).lower() + except OSError: + sp = str(it[1]).lower() + under = 0 if sp.startswith(base_root) else 1 + return (under, len(sp)) + + items_sorted = sorted(items, key=_score) + picked.append(items_sorted[0]) + + return rest + picked + # 收集所有需要的隐藏导入 hiddenimports = [ # Web 框架 @@ -57,7 +142,7 @@ hiddenimports = [ 'uvicorn.protocols.http.auto', 'uvicorn.protocols.websockets.auto', 'uvicorn.loops.auto', - + # Backend 模块（重要！） 'main', # backend/main.py 'backend.main', @@ -90,7 +175,7 @@ hiddenimports = [ 'backend.utils.question_locale', 'backend.utils.sql_parser', 'backend.utils.validators', - + # 数据库相关 'pymssql', 'sqlalchemy', @@ -100,27 +185,32 @@ hiddenimports = [ 'sqlglot', 'sqlglot.dialects', 'sqlparse', - + # 向量数据库 - 使用 collect_submodules 自动收集所有子模块 + # Embedding 相关 + 'sentencepiece', + 'accelerate', + 'safetensors', + # LLM API 'openai', - + # 数据处理 'numpy', 'jieba', - + # Pydantic 'pydantic', 'pydantic_settings', 'dotenv', - + # 工具库 'tenacity', 'structlog', 'tqdm', 'camel_ai', - + # 标准库中可能被遗漏的 'email.mime.text', 'email.mime.multipart', @@ -146,11 +236,15 @@ a = Analysis( hiddenimports=hiddenimports, hookspath=[], hooksconfig={}, - runtime_hooks=[], + runtime_hooks=[str(_REPO_DIR / 'packaging' / 'pyi_rth_crt_path.py')], excludes=excludes, noarchive=False, optimize=0, ) + +# 去掉 Analysis 自动收集到的重复 MSVC 运行库（比仅过滤手动 binaries 更关键） +a.binaries = _dedupe_msvc_runtime_in_analysis_binaries(list(a.binaries)) + pyz = PYZ(a.pure) exe = EXE( diff --git a/packaging/pyi_rth_crt_path.py b/packaging/pyi_rth_crt_path.py new file mode 100644 index 0000000..cc104b4 --- /dev/null +++ b/packaging/pyi_rth_crt_path.py @@ -0,0 +1,89 @@ +""" +PyInstaller 运行时钩子：固定 MSVC 运行库加载顺序，再注册 DLL 搜索路径。 + +打包阶段在 strip 掉各 wheel 自带 CRT 后，会 **仅嵌入** 构建所用 Python ``DLLs`` +下的 ``vcruntime140.dll`` / ``vcruntime140_1.dll`` / ``msvcp140.dll`` 至 ``_MEIPASS``。 +本钩子在导入 chroma/torch 等扩展 **之前**，用 ``ctypes.WinDLL(绝对路径)`` 从 **同一目录** +（优先 exe 旁若用户自行放置了完整三件套，否则 ``_MEIPASS``）预加载，避免进程落到 +目标机 ``System32`` 里较旧版本（如 14.29）再与 native 扩展不兼容触发 ``0xc0000005``。 + +随后再 ``add_dll_directory`` exe 目录与 ``_MEIPASS``，并调整 ``PATH``。不使用 ``SetDllDirectoryW``。 +""" + + +def _preload_msvc_crt_once(): + """若 exe 目录或 ``_MEIPASS`` 下存在 ``msvcp140.dll``，则按依赖顺序预加载该目录内 CRT。""" + import ctypes + import sys + from pathlib import Path + + if sys.platform != "win32" or not getattr(sys, "frozen", False): + return + + order = ("vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll") + search_dirs = [] + try: + exe_parent = Path(sys.executable).resolve().parent + search_dirs.append(exe_parent) + except OSError: + pass + meipass = getattr(sys, "_MEIPASS", None) + if meipass: + search_dirs.append(Path(meipass)) + + for d in search_dirs: + if not d.is_dir(): + continue + if not (d / "msvcp140.dll").is_file(): + continue + for name in order: + p = d / name + if p.is_file(): + try: + ctypes.WinDLL(str(p)) + except OSError: + pass + return + + +def _apply_dll_search_paths(): + """将 exe 目录与 ``_MEIPASS`` 注册为 DLL 目录，并置于 PATH 最前（exe 目录优先）。""" + import os + import sys + from pathlib import Path + + if not getattr(sys, "frozen", False): + return + + dirs = [] + try: + exe_dir = str(Path(sys.executable).resolve().parent) + if exe_dir: + dirs.append(exe_dir) + except OSError: + pass + + meipass = getattr(sys, "_MEIPASS", None) + if meipass: + dirs.append(meipass) + + for d in dirs: + if not d: + continue + try: + if hasattr(os, "add_dll_directory"): + os.add_dll_directory(d) + except OSError: + pass + + path = os.environ.get("PATH", "") + prefix = os.pathsep.join([d for d in dirs if d]) + if not prefix: + return + if path.startswith(prefix): + return + os.environ["PATH"] = f"{prefix}{os.pathsep}{path}" + + +_preload_msvc_crt_once() +_apply_dll_search_paths()