diff --git a/backend/api/kb_file_router.py b/backend/api/kb_file_router.py index 85315bf..cde52c0 100644 --- a/backend/api/kb_file_router.py +++ b/backend/api/kb_file_router.py @@ -364,6 +364,17 @@ async def upload_file( except ValueError as e: logger.warning(f"❌ 文本文件字数超限: {file.filename}, {e}") raise BadRequestError(str(e)) from e + + existing_file = await KnowledgeBaseFileService.get_active_file_by_name( + conn, kb_id, file.filename + ) + if existing_file and existing_file.status in ("processing", "completed"): + raise BadRequestError(f"文件 '{file.filename}' 已存在于该知识库中") + retry_old_file_path = ( + existing_file.file_path + if existing_file and existing_file.status == "failed" + else None + ) # 生成唯一文件名 timestamp = int(time.time() * 1000) @@ -374,6 +385,7 @@ async def upload_file( oss_service = get_oss_service() file_path = None file_url = None + uploaded_oss_object_name = None logger.info(f"☁️ 开始上传文件，OSS 状态: {'已启用' if oss_service.enabled else '未启用'}") @@ -382,6 +394,7 @@ async def upload_file( file_url = oss_service.upload_file_from_bytes(content, oss_object_name, file.filename) if file_url: file_path = file_url + uploaded_oss_object_name = oss_object_name logger.info(f"✅ 文件已上传到 OSS: {file_url}") # 🔑 图片审核：在创建文件记录前进行审核 @@ -439,12 +452,25 @@ async def upload_file( file_path = str(local_path) logger.info(f"💾 文件已保存到本地: {file_path}") - # 创建文件记录 - logger.info(f"📝 创建文件记录: {file.filename}") - file_record = await KnowledgeBaseFileService.create_file_record( - conn, kb_id, current_user.id, file.filename, file_path, file_size, file_type - ) - logger.info(f"✅ 文件记录已创建: ID={file_record.id}, 状态={file_record.status}") + try: + # 创建文件记录（处理失败的可复用原记录重试） + logger.info(f"📝 创建文件记录: {file.filename}") + file_record = await KnowledgeBaseFileService.create_file_record( + conn, kb_id, current_user.id, file.filename, file_path, file_size, file_type + ) + logger.info(f"✅ 文件记录已创建: ID={file_record.id}, 状态={file_record.status}") + except ValueError: + if uploaded_oss_object_name and oss_service.enabled: + oss_service.delete_file(uploaded_oss_object_name) + raise + + if retry_old_file_path and retry_old_file_path != file_path and retry_old_file_path.startswith( + ("http://", "https://") + ): + old_oss_object = oss_service.extract_object_name_from_url(retry_old_file_path, kb_id) + if old_oss_object: + oss_service.delete_file(old_oss_object) + logger.info(f"🗑️ 已清理失败重试前的 OSS 文件: {old_oss_object}") # 审计日志：上传 await AuditService.write( @@ -479,7 +505,6 @@ async def upload_file( except BadRequestError: raise except ValueError as e: - # 文件名重复等业务错误 logger.warning(f"文件上传验证失败: {e}") raise BadRequestError(str(e)) except Exception as e: diff --git a/backend/core/llm_catalog.py b/backend/core/llm_catalog.py index 8f581c1..fe0a2ac 100644 --- a/backend/core/llm_catalog.py +++ b/backend/core/llm_catalog.py @@ -282,6 +282,8 @@ def build_chat_model( base_url = llm_env.resolved_deepseek_chat_base_url().strip().rstrip("/") else: raise ValueError(f"未知提供方: {provider}") + + print("-----------------------------------走到这里了-----------------------------------",extra_kwargs) return ChatOpenAI( model=api_model, api_key=api_key, diff --git a/backend/core/llm_env.py b/backend/core/llm_env.py index ccd44bf..8634662 100644 --- a/backend/core/llm_env.py +++ b/backend/core/llm_env.py @@ -56,3 +56,18 @@ def dashscope_native_http_api_base() -> str: def resolved_deepseek_chat_base_url() -> str: """DeepSeek OpenAI 兼容 base：仅从 ``DEEPSEEK_API_BASE`` 读取，无内置默认。""" return _getenv_nonempty("DEEPSEEK_API_BASE", "deepseek_api_base").strip().rstrip("/") + + +def tongyi_embedding_api_key() -> str: + """通义 Embedding：优先 ``ZL_DASHSCOPE_API_KEY``，否则回退 ``DASHSCOPE_API_KEY``。""" + return _getenv_nonempty("ZL_DASHSCOPE_API_KEY", "zl_dashscope_api_key") or _getenv_nonempty( + "DASHSCOPE_API_KEY", "dashscope_api_key" + ) + + +def tongyi_embedding_api_base() -> str: + """通义 Embedding OpenAI 兼容 base：优先 ``ZL_DASHSCOPE_API_BASE``，否则 ``DASHSCOPE_API_BASE``。""" + zl = _getenv_nonempty("ZL_DASHSCOPE_API_BASE", "zl_dashscope_api_base").strip().rstrip("/") + if zl: + return zl + return tongyi_openai_compatible_base_url() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index ea0a1b8..78f4fe1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "sse-starlette>=3.0.3", "streamlit>=1.52.0", "tavily-python>=0.7.13", - "unstructured[docx]>=0.18.21", + "unstructured[docx,xlsx]>=0.18.21", "uvicorn>=0.38.0", "redis>=5.0.0", "alibabacloud-dysmsapi20170525>=3.0.0", diff --git a/backend/services/knowledge_base_file_service.py b/backend/services/knowledge_base_file_service.py index 853d53e..53bc5bf 100644 --- a/backend/services/knowledge_base_file_service.py +++ b/backend/services/knowledge_base_file_service.py @@ -17,6 +17,25 @@ logger = get_logger(__name__) class KnowledgeBaseFileService: """知识库文件服务类""" + @staticmethod + async def get_active_file_by_name( + conn: asyncpg.Connection, + knowledge_base_id: int, + file_name: str, + ) -> Optional[KnowledgeBaseFile]: + """按知识库 + 文件名查询未删除的文件记录。""" + row = await conn.fetchrow( + """ + SELECT id, knowledge_base_id, user_id, file_name, file_path, file_size, + file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at + FROM knowledge_base_file + WHERE knowledge_base_id = $1 AND file_name = $2 AND is_deleted = FALSE + """, + knowledge_base_id, + file_name, + ) + return KnowledgeBaseFile(**dict(row)) if row else None + @staticmethod async def create_file_record( conn: asyncpg.Connection, @@ -43,18 +62,35 @@ class KnowledgeBaseFileService: KnowledgeBaseFile: 创建的文件记录 """ try: - # 检查文件名是否已存在 - existing = await conn.fetchrow( - """ - SELECT id FROM knowledge_base_file - WHERE knowledge_base_id = $1 AND file_name = $2 AND is_deleted = FALSE - """, - knowledge_base_id, file_name + existing = await KnowledgeBaseFileService.get_active_file_by_name( + conn, knowledge_base_id, file_name ) - + if existing: + if existing.status == "failed": + row = await conn.fetchrow( + """ + UPDATE knowledge_base_file + SET file_path = $1, file_size = $2, file_type = $3, + status = 'processing', chunk_count = 0, + user_id = $4, updated_at = CURRENT_TIMESTAMP + WHERE id = $5 + RETURNING id, knowledge_base_id, user_id, file_name, file_path, file_size, + file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at + """, + file_path, + file_size, + file_type, + user_id, + existing.id, + ) + logger.info( + f"重试处理失败文件: {file_name}, 知识库 ID: {knowledge_base_id}, file_id={existing.id}" + ) + return KnowledgeBaseFile(**dict(row)) + raise ValueError(f"文件 '{file_name}' 已存在于该知识库中") - + # 插入文件记录 row = await conn.fetchrow( """ diff --git a/backend/services/vector_service.py b/backend/services/vector_service.py index e17486c..06604e2 100644 --- a/backend/services/vector_service.py +++ b/backend/services/vector_service.py @@ -71,17 +71,61 @@ except ImportError: fitz = None from langchain_text_splitters import RecursiveCharacterTextSplitter -from langchain_ollama import OllamaEmbeddings +from langchain_core.embeddings import Embeddings from langchain_openai import OpenAIEmbeddings from langchain_chroma import Chroma import bs4 from logger.logging import get_logger from core.config import settings +from core.llm_env import tongyi_embedding_api_base, tongyi_embedding_api_key from services.kb_text_limits import validate_kb_text_length, validate_chat_file_text_length logger = get_logger(__name__) +# 通义 text-embedding-v4（OpenAI 兼容）单次请求最多 10 条，超出会 400/500 +_TONGYI_EMBEDDING_MAX_BATCH = 10 + + +class TongyiEmbeddings(Embeddings): + """通义千问 Embedding 封装：固定走 ZL/DashScope 网关，并按 API 上限分批请求。""" + + def __init__(self, *, model: str, api_key: str, base_url: str, dimensions: int) -> None: + self._inner = OpenAIEmbeddings( + model=model, + api_key=api_key, + base_url=base_url, + check_embedding_ctx_length=False, + dimensions=dimensions, + ) + + def _batched(self, texts: List[str]) -> List[List[float]]: + if not texts: + return [] + vectors: List[List[float]] = [] + for start in range(0, len(texts), _TONGYI_EMBEDDING_MAX_BATCH): + batch = texts[start : start + _TONGYI_EMBEDDING_MAX_BATCH] + vectors.extend(self._inner.embed_documents(batch)) + return vectors + + def embed_documents(self, texts: List[str]) -> List[List[float]]: + return self._batched(texts) + + def embed_query(self, text: str) -> List[float]: + return self._inner.embed_query(text) + + async def aembed_documents(self, texts: List[str]) -> List[List[float]]: + if not texts: + return [] + vectors: List[List[float]] = [] + for start in range(0, len(texts), _TONGYI_EMBEDDING_MAX_BATCH): + batch = texts[start : start + _TONGYI_EMBEDDING_MAX_BATCH] + vectors.extend(await self._inner.aembed_documents(batch)) + return vectors + + async def aembed_query(self, text: str) -> List[float]: + return await self._inner.aembed_query(text) + @dataclass class ProcessResult: @@ -125,16 +169,28 @@ class VectorService: def __init__(self): """初始化向量服务""" - # 初始化嵌入模型 - # self.embedding = OllamaEmbeddings(model="nomic-embed-text") - # DashScope 兼容网关只接受字符串 input；默认 check_embedding_ctx_length=True - # 会用 tiktoken 转成 token id 列表再请求，导致 400：contents is neither str nor list of str - print(settings.dashscope_api_key, settings.dashscope_api_base) - self.embedding = OpenAIEmbeddings( - model="text-embedding-v4", - api_key=os.getenv("ZL_DASHSCOPE_API_KEY"), # 如果您没有配置环境变量，请在此处用您的API Key进行替换 - base_url=os.getenv("ZL_DASHSCOPE_API_BASE"), - check_embedding_ctx_length=False, + embedding_api_key = tongyi_embedding_api_key() + embedding_api_base = tongyi_embedding_api_base() + if not embedding_api_key: + raise ValueError( + "未配置通义 Embedding API Key，请设置 ZL_DASHSCOPE_API_KEY 或 DASHSCOPE_API_KEY" + ) + if not embedding_api_base: + raise ValueError( + "未配置通义 Embedding API Base，请设置 ZL_DASHSCOPE_API_BASE 或 DASHSCOPE_API_BASE" + ) + # 通义 text-embedding-v4（OpenAI 兼容 /embeddings）；check_embedding_ctx_length=False + # 避免 tiktoken 将输入转成 token id 列表导致 DashScope 返回 400 + self.embedding = TongyiEmbeddings( + model=settings.embedding_model, + api_key=embedding_api_key, + base_url=embedding_api_base, + dimensions=settings.embedding_dimension, + ) + logger.info( + "通义 Embedding 已初始化: model={}, base_url={}", + settings.embedding_model, + embedding_api_base, ) # 文本分割器配置（参考 server：增大 chunk_size 保留更多上下文） diff --git a/backend/uv.lock b/backend/uv.lock index 915f73f..9929a5a 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1124,7 +1124,7 @@ dependencies = [ { name = "sse-starlette" }, { name = "streamlit" }, { name = "tavily-python" }, - { name = "unstructured", extra = ["docx"] }, + { name = "unstructured", extra = ["docx", "xlsx"] }, { name = "uvicorn" }, ] @@ -1188,7 +1188,7 @@ requires-dist = [ { name = "sse-starlette", specifier = ">=3.0.3" }, { name = "streamlit", specifier = ">=1.52.0" }, { name = "tavily-python", specifier = ">=0.7.13" }, - { name = "unstructured", extras = ["docx"], specifier = ">=0.18.21" }, + { name = "unstructured", extras = ["docx", "xlsx"], specifier = ">=0.18.21" }, { name = "uvicorn", specifier = ">=0.38.0" }, ] @@ -1848,6 +1848,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] +[[package]] +name = "msoffcrypto-tool" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "olefile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/34/6250bdddaeaae24098e45449ea362fb3555a65fba30cad0ad5630ea48d1a/msoffcrypto_tool-6.0.0.tar.gz", hash = "sha256:9a5ebc4c0096b42e5d7ebc2350afdc92dc511061e935ca188468094fdd032bbe", size = 40593, upload-time = "2026-01-12T08:59:56.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/85/9e359fa9279e1d6861faaf9b6f037a3226374deb20a054c3937be6992013/msoffcrypto_tool-6.0.0-py3-none-any.whl", hash = "sha256:46c394ed5d9641e802fc79bf3fb0666a53748b23fa8c4aa634ae9d30d46fe397", size = 48791, upload-time = "2026-01-12T08:59:55.394Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -1905,6 +1918,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/cf/1c3795866cefaac6e648d4e98c373cafd97810f6e317c307371007ab4abb/neo4j-6.2.0-py3-none-any.whl", hash = "sha256:b87abdd13a5cc2e3bd51026926c2f20ac38fa3febe98c340520dce19e97388d0", size = 327824, upload-time = "2026-05-04T07:35:39.604Z" }, ] +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + [[package]] name = "nltk" version = "3.9.4" @@ -3353,6 +3375,13 @@ wheels = [ docx = [ { name = "python-docx" }, ] +xlsx = [ + { name = "msoffcrypto-tool" }, + { name = "networkx" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "xlrd" }, +] [[package]] name = "unstructured-client" @@ -3594,6 +3623,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, ] +[[package]] +name = "xlrd" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, +] + [[package]] name = "xxhash" version = "3.7.0"