diff --git a/.gitignore b/.gitignore index ea77a85..fbff7d3 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,5 @@ Thumbs.db # 本地导出的 IDE / 对话历史等(按需) history.txt .cursor/ -backend/.env.docker \ No newline at end of file +backend/.env.docker +backend/uploads/ \ No newline at end of file diff --git a/backend/services/vector_service.py b/backend/services/vector_service.py index 06604e2..f780993 100644 --- a/backend/services/vector_service.py +++ b/backend/services/vector_service.py @@ -370,6 +370,68 @@ class VectorService: except Exception as e: logger.error(f"图片 OCR 处理失败: {e}") raise + + async def _process_image_with_vision_fallback(self, file_path: str) -> List: + """ + 处理知识库图片:优先 OCR;OCR 无文字或失败时,必须使用视觉模型提取内容。 + """ + from langchain_core.documents import Document + from services.vision_service import VisionService + + ocr_docs: List = [] + try: + ocr_docs = await asyncio.to_thread(self._process_image_ocr, file_path) + except Exception as e: + logger.warning(f"图片 OCR 异常,将尝试视觉模型: {e}") + + if ocr_docs: + return ocr_docs + + logger.info("OCR 未识别到文字,使用视觉模型处理图片...") + vision_prompt = ( + "详细描述图片中的内容:场景、人物、物体、图表及所有可见文字(逐字提取)。" + "用通顺中文输出,便于后续检索与问答。" + ) + + try: + with open(file_path, "rb") as f: + image_bytes = f.read() + except OSError as e: + logger.error(f"读取图片文件失败,无法进行视觉理解: {e}") + return [] + + ext = os.path.splitext(file_path)[1].lower() + mime_map = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".bmp": "image/bmp", + ".webp": "image/webp", + ".gif": "image/gif", + } + vision_text = await VisionService.get_image_description_from_bytes( + image_bytes, + prompt=vision_prompt, + mime_hint=mime_map.get(ext, "image/jpeg"), + ) + + if not vision_text or not vision_text.strip(): + logger.warning(f"视觉模型也未提取到内容: {file_path}") + return [] + + logger.info(f"视觉模型成功提取图片内容,共 {len(vision_text)} 字符") + return [ + Document( + page_content=f"【图片内容描述】\n{vision_text.strip()}", + metadata={ + "source": file_path, + "file_type": "image", + "has_ocr": False, + "has_vision": True, + "vision_provider": "qwen-vl", + }, + ) + ] def _extract_images_from_docx(self, docx_path: str) -> List[str]: """ @@ -885,8 +947,8 @@ class VectorService: # 2. 加载文档(特殊处理图片 OCR 和 DOCX,放到线程池执行) if loader == "image_ocr": - logger.info("🔄 在线程池中执行图片 OCR...") - docs = await asyncio.to_thread(self._process_image_ocr, file_path) + logger.info("🔄 处理图片:OCR → 视觉模型回退...") + docs = await self._process_image_with_vision_fallback(file_path) elif loader == "docx_with_images": logger.info("🔄 在线程池中处理 DOCX 文件(提取图片并 OCR)...") docs, _ = await asyncio.to_thread(self._process_docx_with_images, file_path) # 忽略图片路径(知识库暂不使用视觉模型) @@ -913,7 +975,10 @@ class VectorService: logger.info(f"文档加载完成,共 {len(docs)} 个文档片段") if not docs: - error_msg = "未能从文件加载到任何内容" + error_msg = ( + "未能从文件加载到任何内容。" + "图片请确认已配置 DASHSCOPE_API_KEY(视觉模型)和/或阿里云 OCR。" + ) logger.warning(error_msg) return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)