diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/backend/schema/indexer.py b/backend/schema/indexer.py index 6121750..ae75463 100644 --- a/backend/schema/indexer.py +++ b/backend/schema/indexer.py @@ -20,7 +20,7 @@ class SchemaIndexer: 功能： 1. 为表结构构建向量索引 2. 基于问题的表检索 - 3. Chroma 使用内存模式（EphemeralClient），进程退出后不保留；``persist_dir`` 仅作配置/统计引用 + 3. Chroma 使用持久化（PersistentClient），向量与元数据写入 ``persist_dir``，进程重启后可复用 """ def __init__( @@ -34,15 +34,17 @@ class SchemaIndexer: Args: embedder: Embedding模型实例 - persist_dir: 历史配置中的向量库路径（仅展示与统计，不落盘） + persist_dir: Chroma 持久化根目录（磁盘路径） collection_name: 集合名称 """ self.embedder = embedder self.persist_dir = Path(persist_dir) + self.persist_dir.mkdir(parents=True, exist_ok=True) self.collection_name = collection_name - # 内存 Chroma：不落盘，每次进程需重新 build_index - self.client = chromadb.EphemeralClient( + # 持久化 Chroma：数据落盘至 persist_dir + self.client = chromadb.PersistentClient( + path=str(self.persist_dir), settings=ChromaSettings(anonymized_telemetry=False), ) @@ -53,8 +55,8 @@ class SchemaIndexer: ) logger.info( - f"[OK] 初始化SchemaIndexer(Chroma内存): collection={self.collection_name}, " - f"配置路径引用={self.persist_dir}" + f"[OK] 初始化SchemaIndexer(Chroma持久化): collection={self.collection_name}, " + f"path={self.persist_dir}" ) def build_index( @@ -250,5 +252,5 @@ class SchemaIndexer: "total_columns": total_columns, "avg_columns": total_columns / count if count > 0 else 0, "persist_dir": str(self.persist_dir), - "chroma_mode": "memory", + "chroma_mode": "persistent", } diff --git a/deploy/msvcp140.dll b/deploy/msvcp140.dll new file mode 100644 index 0000000..bea9c37 Binary files /dev/null and b/deploy/msvcp140.dll differ diff --git a/deploy/vcruntime140.dll b/deploy/vcruntime140.dll new file mode 100644 index 0000000..a9ed5c4 Binary files /dev/null and b/deploy/vcruntime140.dll differ diff --git a/deploy/vcruntime140_1.dll b/deploy/vcruntime140_1.dll new file mode 100644 index 0000000..cb34ab6 Binary files /dev/null and b/deploy/vcruntime140_1.dll differ diff --git a/main.spec b/main.spec index 7aae955..32312b0 100644 --- a/main.spec +++ b/main.spec @@ -8,6 +8,35 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules, coll import sys _REPO_DIR = Path(os.getcwd()).resolve() + +def _strip_msvc_runtime_binaries(bin_list): + """ + 从 ``collect_dynamic_libs`` 等额外收集的 binaries 中移除 MSVC 运行库 DLL。 + + chromadb / hnswlib 等 wheel 常自带 ``MSVCP140.dll`` 等；与 PyInstaller 已为 + ``python.exe`` 解析到的 CRT 一并打入单文件包时，易在运行期混用多份 CRT + 导致 ``MSVCP140.dll`` 内 ``0xc0000005``。统一依赖解释器自带的 CRT 集更安全。 + """ + skip = frozenset({ + 'msvcp140.dll', + 'msvcp140_1.dll', + 'msvcp140_2.dll', + 'msvcp140_atomic_wait.dll', + 'msvcp140_codecvt_ids.dll', + 'vcruntime140.dll', + 'vcruntime140_1.dll', + 'concrt140.dll', + }) + out = [] + for item in bin_list: + dest = item[0] + base = Path(dest).name.lower() + if base in skip: + continue + out.append(item) + return out + + # 根目录执行：pyinstaller --clean main.spec # 收集配置文件夹和数据文件 datas = [ @@ -36,6 +65,62 @@ try: except Exception: pass +binaries = _strip_msvc_runtime_binaries(binaries) + + +def _dedupe_msvc_runtime_in_analysis_binaries(toc): + """ + 对 ``Analysis`` 产出的 ``a.binaries`` 去重：同名 MSVC 运行库 DLL 只保留一份。 + + PyInstaller 会从 Python 安装目录、numpy、torch、chromadb 等多个依赖各自打入 + 同名 ``MSVCP140.dll`` 等；单文件解压到 ``_MEI*`` 后若出现多份或加载顺序混乱， + 易在 ``MSVCP140.dll`` 内触发 ``0xc0000005``。优先保留 **源路径位于** + ``sys.base_prefix`` 下的条目（与解释器主 CRT 一致）。 + """ + vc_names = frozenset({ + 'msvcp140.dll', + 'msvcp140_1.dll', + 'msvcp140_2.dll', + 'msvcp140_atomic_wait.dll', + 'msvcp140_codecvt_ids.dll', + 'vcruntime140.dll', + 'vcruntime140_1.dll', + 'concrt140.dll', + }) + try: + base_root = str(Path(sys.base_prefix).resolve()).lower() + except OSError: + base_root = str(Path(sys.base_prefix)).lower() + + rest = [] + vc_groups = {} + for item in toc: + dest = item[0] + bname = Path(dest).name.lower() + if bname not in vc_names: + rest.append(item) + continue + vc_groups.setdefault(bname, []).append(item) + + picked = [] + for bname, items in vc_groups.items(): + if len(items) == 1: + picked.append(items[0]) + continue + + def _score(it): + try: + sp = str(Path(it[1]).resolve()).lower() + except OSError: + sp = str(it[1]).lower() + under = 0 if sp.startswith(base_root) else 1 + return (under, len(sp)) + + items_sorted = sorted(items, key=_score) + picked.append(items_sorted[0]) + + return rest + picked + # 收集所有需要的隐藏导入 hiddenimports = [ # Web 框架 @@ -48,7 +133,7 @@ hiddenimports = [ 'uvicorn.protocols.http.auto', 'uvicorn.protocols.websockets.auto', 'uvicorn.loops.auto', - + # Backend 模块（重要！） 'main', # backend/main.py 'backend.main', @@ -81,7 +166,7 @@ hiddenimports = [ 'backend.utils.question_locale', 'backend.utils.sql_parser', 'backend.utils.validators', - + # 数据库相关 'pymssql', 'sqlalchemy', @@ -91,27 +176,32 @@ hiddenimports = [ 'sqlglot', 'sqlglot.dialects', 'sqlparse', - + # 向量数据库 - 使用 collect_submodules 自动收集所有子模块 + # Embedding 相关 + 'sentencepiece', + 'accelerate', + 'safetensors', + # LLM API 'openai', - + # 数据处理 'numpy', 'jieba', - + # Pydantic 'pydantic', 'pydantic_settings', 'dotenv', - + # 工具库 'tenacity', 'structlog', 'tqdm', 'camel_ai', - + # 标准库中可能被遗漏的 'email.mime.text', 'email.mime.multipart', @@ -137,11 +227,15 @@ a = Analysis( hiddenimports=hiddenimports, hookspath=[], hooksconfig={}, - runtime_hooks=[], + runtime_hooks=[str(_REPO_DIR / 'packaging' / 'pyi_rth_crt_path.py')], excludes=excludes, noarchive=False, optimize=0, ) + +# 去掉 Analysis 自动收集到的重复 MSVC 运行库（比仅过滤手动 binaries 更关键） +a.binaries = _dedupe_msvc_runtime_in_analysis_binaries(list(a.binaries)) + pyz = PYZ(a.pure) exe = EXE(