增加文件上传进度,当上传的图片过大时,进行压缩处理
This commit is contained in:
parent
b4b37240d0
commit
4ea62f0831
|
|
@ -1,5 +1,3 @@
|
||||||
ENTERPRISE_BACKEND_VERSION=prod-v1.1.0
|
|
||||||
|
|
||||||
# ==================== 服务器配置 ====================
|
# ==================== 服务器配置 ====================
|
||||||
# API 服务器配置
|
# API 服务器配置
|
||||||
API.HOST=0.0.0.0
|
API.HOST=0.0.0.0
|
||||||
|
|
@ -79,7 +77,6 @@ EMBEDDING_DIMENSION=1536 # Embedding 维度
|
||||||
OCR_ACCESS_KEY_ID=修改此项为阿里云 OCR 访问密钥ID
|
OCR_ACCESS_KEY_ID=修改此项为阿里云 OCR 访问密钥ID
|
||||||
OCR_ACCESS_KEY_SECRET=修改此项为阿里云 OCR 访问密钥Secret
|
OCR_ACCESS_KEY_SECRET=修改此项为阿里云 OCR 访问密钥Secret
|
||||||
OCR_ENDPOINT=修改此项为阿里云 OCR 终端节点
|
OCR_ENDPOINT=修改此项为阿里云 OCR 终端节点
|
||||||
OCR_TIMEOUT_SECONDS=120
|
|
||||||
OCR_USE_LOCAL=false
|
OCR_USE_LOCAL=false
|
||||||
MODERATION_ENABLED=false
|
MODERATION_ENABLED=false
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,14 @@ from services.chat_thread_file_service import ChatThreadFileService
|
||||||
from services.vector_service import get_vector_service
|
from services.vector_service import get_vector_service
|
||||||
from services.oss_service import get_oss_service
|
from services.oss_service import get_oss_service
|
||||||
from services.kb_text_limits import decode_txt_char_count, validate_chat_file_text_length
|
from services.kb_text_limits import decode_txt_char_count, validate_chat_file_text_length
|
||||||
|
from services.file_progress import (
|
||||||
|
FileProgressReporter,
|
||||||
|
make_db_progress_callback,
|
||||||
|
STAGE_PROCESSING,
|
||||||
|
STAGE_DOWNLOADING,
|
||||||
|
STAGE_SUMMARIZING,
|
||||||
|
stage_label,
|
||||||
|
)
|
||||||
from models.chat_thread_file import (
|
from models.chat_thread_file import (
|
||||||
ChatThreadFileUploadResponse,
|
ChatThreadFileUploadResponse,
|
||||||
ChatThreadFileListResponse
|
ChatThreadFileListResponse
|
||||||
|
|
@ -47,10 +55,13 @@ async def process_chat_file_background(
|
||||||
file_type: 文件类型(pdf 或 url)
|
file_type: 文件类型(pdf 或 url)
|
||||||
"""
|
"""
|
||||||
pool = await get_db_pool()
|
pool = await get_db_pool()
|
||||||
|
progress_cb = make_db_progress_callback(pool, file_id, ChatThreadFileService)
|
||||||
|
progress = FileProgressReporter(progress_cb)
|
||||||
async with pool.acquire() as conn:
|
async with pool.acquire() as conn:
|
||||||
local_file_path = None
|
local_file_path = None
|
||||||
try:
|
try:
|
||||||
logger.info(f"开始后台处理聊天文件 ID: {file_id}, thread_id: {thread_id}, 路径: {file_path}")
|
logger.info(f"开始后台处理聊天文件 ID: {file_id}, thread_id: {thread_id}, 路径: {file_path}")
|
||||||
|
await progress.report(30, STAGE_PROCESSING)
|
||||||
|
|
||||||
# file_path 是 OSS URL,需要先下载到本地临时文件
|
# file_path 是 OSS URL,需要先下载到本地临时文件
|
||||||
oss_service = get_oss_service()
|
oss_service = get_oss_service()
|
||||||
|
|
@ -65,6 +76,7 @@ async def process_chat_file_background(
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.info(f"检测到 OSS URL,开始下载文件: {file_path}")
|
logger.info(f"检测到 OSS URL,开始下载文件: {file_path}")
|
||||||
|
await progress.report(32, STAGE_DOWNLOADING)
|
||||||
|
|
||||||
# 从 OSS URL 提取对象名称
|
# 从 OSS URL 提取对象名称
|
||||||
oss_object_name = oss_service.extract_object_name_from_url(file_path, thread_id=thread_id)
|
oss_object_name = oss_service.extract_object_name_from_url(file_path, thread_id=thread_id)
|
||||||
|
|
@ -92,7 +104,8 @@ async def process_chat_file_background(
|
||||||
thread_id,
|
thread_id,
|
||||||
file_type,
|
file_type,
|
||||||
file_id=file_id,
|
file_id=file_id,
|
||||||
source_url=file_path # 🔑 传递原始 OSS URL
|
source_url=file_path, # 🔑 传递原始 OSS URL
|
||||||
|
progress=progress,
|
||||||
)
|
)
|
||||||
|
|
||||||
# 检查处理结果
|
# 检查处理结果
|
||||||
|
|
@ -104,6 +117,7 @@ async def process_chat_file_background(
|
||||||
# 生成文件摘要
|
# 生成文件摘要
|
||||||
summary_text = None
|
summary_text = None
|
||||||
try:
|
try:
|
||||||
|
await progress.report(92, STAGE_SUMMARIZING)
|
||||||
# 判断是否为图片类型
|
# 判断是否为图片类型
|
||||||
image_types = {'png', 'jpg', 'jpeg', 'bmp'}
|
image_types = {'png', 'jpg', 'jpeg', 'bmp'}
|
||||||
is_image = file_type.lower() in image_types
|
is_image = file_type.lower() in image_types
|
||||||
|
|
@ -542,6 +556,9 @@ async def upload_chat_file(
|
||||||
file_size,
|
file_size,
|
||||||
file_type # 使用检测到的文件类型
|
file_type # 使用检测到的文件类型
|
||||||
)
|
)
|
||||||
|
await ChatThreadFileService.update_file_progress(
|
||||||
|
conn, file_record.id, 30, STAGE_PROCESSING
|
||||||
|
)
|
||||||
logger.info(f"✅ 文件记录已创建: ID={file_record.id}, 状态={file_record.status}")
|
logger.info(f"✅ 文件记录已创建: ID={file_record.id}, 状态={file_record.status}")
|
||||||
|
|
||||||
# 添加后台任务处理向量化(传递 OSS URL 和文件类型)
|
# 添加后台任务处理向量化(传递 OSS URL 和文件类型)
|
||||||
|
|
@ -567,6 +584,8 @@ async def upload_chat_file(
|
||||||
file_size=file_record.file_size,
|
file_size=file_record.file_size,
|
||||||
status=file_record.status,
|
status=file_record.status,
|
||||||
chunk_count=file_record.chunk_count,
|
chunk_count=file_record.chunk_count,
|
||||||
|
progress_percent=30,
|
||||||
|
processing_stage=STAGE_PROCESSING,
|
||||||
created_at=file_record.created_at,
|
created_at=file_record.created_at,
|
||||||
file_url=file_url # 返回 OSS URL
|
file_url=file_url # 返回 OSS URL
|
||||||
).dict()
|
).dict()
|
||||||
|
|
@ -644,6 +663,8 @@ async def get_chat_thread_files(
|
||||||
file_size=f.file_size,
|
file_size=f.file_size,
|
||||||
status=f.status,
|
status=f.status,
|
||||||
chunk_count=f.chunk_count,
|
chunk_count=f.chunk_count,
|
||||||
|
progress_percent=f.progress_percent,
|
||||||
|
processing_stage=f.processing_stage,
|
||||||
created_at=f.created_at,
|
created_at=f.created_at,
|
||||||
file_url=f.file_path # file_path 存储的是 OSS URL
|
file_url=f.file_path # file_path 存储的是 OSS URL
|
||||||
).dict()
|
).dict()
|
||||||
|
|
@ -734,6 +755,9 @@ async def get_file_processing_status(
|
||||||
"file_type": file.file_type,
|
"file_type": file.file_type,
|
||||||
"status": file.status,
|
"status": file.status,
|
||||||
"chunk_count": file.chunk_count,
|
"chunk_count": file.chunk_count,
|
||||||
|
"progress_percent": file.progress_percent,
|
||||||
|
"processing_stage": file.processing_stage,
|
||||||
|
"stage_label": stage_label(file.processing_stage),
|
||||||
"created_at": file.created_at.isoformat() if file.created_at else None,
|
"created_at": file.created_at.isoformat() if file.created_at else None,
|
||||||
"updated_at": file.updated_at.isoformat() if file.updated_at else None,
|
"updated_at": file.updated_at.isoformat() if file.updated_at else None,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,14 @@ from services.audit_service import AuditService
|
||||||
from services.vector_service import get_vector_service
|
from services.vector_service import get_vector_service
|
||||||
from services.kb_text_limits import decode_txt_char_count, validate_kb_text_length
|
from services.kb_text_limits import decode_txt_char_count, validate_kb_text_length
|
||||||
from services.oss_service import get_oss_service
|
from services.oss_service import get_oss_service
|
||||||
|
from services.file_progress import (
|
||||||
|
FileProgressReporter,
|
||||||
|
make_db_progress_callback,
|
||||||
|
STAGE_PROCESSING,
|
||||||
|
STAGE_DOWNLOADING,
|
||||||
|
STAGE_SUMMARIZING,
|
||||||
|
stage_label,
|
||||||
|
)
|
||||||
from utils.helpers import BaseResponse
|
from utils.helpers import BaseResponse
|
||||||
from logger.logging import get_logger
|
from logger.logging import get_logger
|
||||||
|
|
||||||
|
|
@ -82,14 +90,18 @@ async def process_file_background(
|
||||||
file_type: 文件类型
|
file_type: 文件类型
|
||||||
"""
|
"""
|
||||||
pool = await get_db_pool()
|
pool = await get_db_pool()
|
||||||
|
progress_cb = make_db_progress_callback(pool, file_id, KnowledgeBaseFileService)
|
||||||
|
progress = FileProgressReporter(progress_cb)
|
||||||
async with pool.acquire() as conn:
|
async with pool.acquire() as conn:
|
||||||
local_file_path = None
|
local_file_path = None
|
||||||
try:
|
try:
|
||||||
logger.info(f"开始后台处理文件 ID: {file_id}, 路径: {file_path}, 类型: {file_type}")
|
logger.info(f"开始后台处理文件 ID: {file_id}, 路径: {file_path}, 类型: {file_type}")
|
||||||
|
await progress.report(30, STAGE_PROCESSING)
|
||||||
|
|
||||||
oss_service = get_oss_service()
|
oss_service = get_oss_service()
|
||||||
if oss_service.enabled and file_path.startswith(('http://', 'https://')):
|
if oss_service.enabled and file_path.startswith(('http://', 'https://')):
|
||||||
logger.info(f"检测到 OSS URL,开始下载文件: {file_path}")
|
logger.info(f"检测到 OSS URL,开始下载文件: {file_path}")
|
||||||
|
await progress.report(32, STAGE_DOWNLOADING)
|
||||||
oss_object_name = oss_service.extract_object_name_from_url(file_path, knowledge_base_id)
|
oss_object_name = oss_service.extract_object_name_from_url(file_path, knowledge_base_id)
|
||||||
if not oss_object_name:
|
if not oss_object_name:
|
||||||
logger.error(f"无法从 OSS URL 提取对象名称: {file_path}")
|
logger.error(f"无法从 OSS URL 提取对象名称: {file_path}")
|
||||||
|
|
@ -114,7 +126,8 @@ async def process_file_background(
|
||||||
knowledge_base_id,
|
knowledge_base_id,
|
||||||
file_type,
|
file_type,
|
||||||
file_id=file_id,
|
file_id=file_id,
|
||||||
source_url=file_path # 🔑 传递原始 OSS URL
|
source_url=file_path, # 🔑 传递原始 OSS URL
|
||||||
|
progress=progress,
|
||||||
)
|
)
|
||||||
|
|
||||||
# 检查处理结果
|
# 检查处理结果
|
||||||
|
|
@ -126,6 +139,7 @@ async def process_file_background(
|
||||||
# 生成文件摘要
|
# 生成文件摘要
|
||||||
summary_text = None
|
summary_text = None
|
||||||
try:
|
try:
|
||||||
|
await progress.report(92, STAGE_SUMMARIZING)
|
||||||
from services.summary_service import SummaryService
|
from services.summary_service import SummaryService
|
||||||
from langchain_core.documents import Document
|
from langchain_core.documents import Document
|
||||||
|
|
||||||
|
|
@ -480,6 +494,9 @@ async def upload_file(
|
||||||
|
|
||||||
# 添加后台任务
|
# 添加后台任务
|
||||||
logger.info(f"🚀 添加后台向量化任务: file_id={file_record.id}, type={file_type}")
|
logger.info(f"🚀 添加后台向量化任务: file_id={file_record.id}, type={file_type}")
|
||||||
|
await KnowledgeBaseFileService.update_file_progress(
|
||||||
|
conn, file_record.id, 30, STAGE_PROCESSING
|
||||||
|
)
|
||||||
background_tasks.add_task(process_file_background, file_record.id, file_path, kb_id, file_type)
|
background_tasks.add_task(process_file_background, file_record.id, file_path, kb_id, file_type)
|
||||||
|
|
||||||
return BaseResponse(
|
return BaseResponse(
|
||||||
|
|
@ -491,6 +508,8 @@ async def upload_file(
|
||||||
file_size=file_record.file_size,
|
file_size=file_record.file_size,
|
||||||
status=file_record.status,
|
status=file_record.status,
|
||||||
chunk_count=file_record.chunk_count,
|
chunk_count=file_record.chunk_count,
|
||||||
|
progress_percent=30,
|
||||||
|
processing_stage=STAGE_PROCESSING,
|
||||||
created_at=file_record.created_at,
|
created_at=file_record.created_at,
|
||||||
file_url=file_url or file_path
|
file_url=file_url or file_path
|
||||||
).dict()
|
).dict()
|
||||||
|
|
@ -575,6 +594,8 @@ async def get_knowledge_base_files(
|
||||||
"file_type": r["file_type"],
|
"file_type": r["file_type"],
|
||||||
"status": r["status"],
|
"status": r["status"],
|
||||||
"chunk_count": r["chunk_count"],
|
"chunk_count": r["chunk_count"],
|
||||||
|
"progress_percent": r.get("progress_percent", 0),
|
||||||
|
"processing_stage": r.get("processing_stage"),
|
||||||
"created_at": r["created_at"].isoformat() if r.get("created_at") else None,
|
"created_at": r["created_at"].isoformat() if r.get("created_at") else None,
|
||||||
"file_url": r["file_path"],
|
"file_url": r["file_path"],
|
||||||
"uploader_name": r.get("uploader_name"),
|
"uploader_name": r.get("uploader_name"),
|
||||||
|
|
@ -613,6 +634,8 @@ async def get_file_detail(
|
||||||
file_size=file.file_size,
|
file_size=file.file_size,
|
||||||
status=file.status,
|
status=file.status,
|
||||||
chunk_count=file.chunk_count,
|
chunk_count=file.chunk_count,
|
||||||
|
progress_percent=file.progress_percent,
|
||||||
|
processing_stage=file.processing_stage,
|
||||||
created_at=file.created_at,
|
created_at=file.created_at,
|
||||||
file_url=file.file_path
|
file_url=file.file_path
|
||||||
).dict()
|
).dict()
|
||||||
|
|
@ -652,6 +675,9 @@ async def get_file_processing_status(
|
||||||
"file_type": file.file_type,
|
"file_type": file.file_type,
|
||||||
"status": file.status,
|
"status": file.status,
|
||||||
"chunk_count": file.chunk_count,
|
"chunk_count": file.chunk_count,
|
||||||
|
"progress_percent": file.progress_percent,
|
||||||
|
"processing_stage": file.processing_stage,
|
||||||
|
"stage_label": stage_label(file.processing_stage),
|
||||||
"created_at": file.created_at.isoformat() if file.created_at else None,
|
"created_at": file.created_at.isoformat() if file.created_at else None,
|
||||||
"updated_at": file.updated_at.isoformat() if file.updated_at else None,
|
"updated_at": file.updated_at.isoformat() if file.updated_at else None,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,11 +11,8 @@ from urllib.parse import quote_plus
|
||||||
from pydantic import AliasChoices, Field, model_validator
|
from pydantic import AliasChoices, Field, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
# backend/ 目录
|
# backend/ 目录(与 uvicorn CWD 无关,始终读取该目录下的 .env)
|
||||||
# 编译为二进制后 __file__ 路径不可靠,优先使用环境变量 APP_DIR,
|
_BACKEND_DIR = Path(__file__).resolve().parent.parent
|
||||||
# 其次使用当前工作目录(Docker WORKDIR /app 固定为 /app)。
|
|
||||||
import os as _os
|
|
||||||
_BACKEND_DIR = Path(_os.environ.get("APP_DIR", "")).resolve() if _os.environ.get("APP_DIR") else Path(_os.getcwd())
|
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
|
|
@ -105,7 +102,6 @@ class Settings(BaseSettings):
|
||||||
ocr_access_key_id: Optional[str] = None
|
ocr_access_key_id: Optional[str] = None
|
||||||
ocr_access_key_secret: Optional[str] = None
|
ocr_access_key_secret: Optional[str] = None
|
||||||
ocr_endpoint: str = "ocr-api.cn-hangzhou.aliyuncs.com" # OCR 服务端点
|
ocr_endpoint: str = "ocr-api.cn-hangzhou.aliyuncs.com" # OCR 服务端点
|
||||||
ocr_timeout_seconds: float = 120.0 # OCR 请求超时(含上传图片 body)
|
|
||||||
|
|
||||||
# 微信小程序配置
|
# 微信小程序配置
|
||||||
wechat_app_id: Optional[str] = None
|
wechat_app_id: Optional[str] = None
|
||||||
|
|
|
||||||
|
|
@ -248,6 +248,7 @@ def build_chat_model(
|
||||||
p = normalize_provider(provider)
|
p = normalize_provider(provider)
|
||||||
if p == "tongyi":
|
if p == "tongyi":
|
||||||
api_key = (os.getenv("DASHSCOPE_API_KEY") or "").strip()
|
api_key = (os.getenv("DASHSCOPE_API_KEY") or "").strip()
|
||||||
|
print("-----------------------------------api_key: ", api_key)
|
||||||
if not api_key:
|
if not api_key:
|
||||||
raise ValueError("缺少 DASHSCOPE_API_KEY")
|
raise ValueError("缺少 DASHSCOPE_API_KEY")
|
||||||
base_url = llm_env.tongyi_openai_compatible_base_url().strip().rstrip("/")
|
base_url = llm_env.tongyi_openai_compatible_base_url().strip().rstrip("/")
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
-- 文件/任务处理进度(方案 B:真实百分比 + 阶段)
|
||||||
|
-- 在 PostgreSQL 上执行本脚本后再部署新版后端与前端。
|
||||||
|
|
||||||
|
-- 知识库文件
|
||||||
|
ALTER TABLE public.knowledge_base_file
|
||||||
|
ADD COLUMN IF NOT EXISTS progress_percent smallint NOT NULL DEFAULT 0,
|
||||||
|
ADD COLUMN IF NOT EXISTS processing_stage varchar(50) NULL;
|
||||||
|
|
||||||
|
ALTER TABLE public.knowledge_base_file
|
||||||
|
DROP CONSTRAINT IF EXISTS ck_kb_file_progress_percent;
|
||||||
|
ALTER TABLE public.knowledge_base_file
|
||||||
|
ADD CONSTRAINT ck_kb_file_progress_percent
|
||||||
|
CHECK (progress_percent >= 0 AND progress_percent <= 100);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN public.knowledge_base_file.progress_percent IS '处理进度 0-100';
|
||||||
|
COMMENT ON COLUMN public.knowledge_base_file.processing_stage IS 'processing 阶段: uploading/parsing/ocr/embedding/summarizing/completed 等';
|
||||||
|
|
||||||
|
-- 聊天文件
|
||||||
|
ALTER TABLE public.chat_thread_file
|
||||||
|
ADD COLUMN IF NOT EXISTS progress_percent smallint NOT NULL DEFAULT 0,
|
||||||
|
ADD COLUMN IF NOT EXISTS processing_stage varchar(50) NULL;
|
||||||
|
|
||||||
|
ALTER TABLE public.chat_thread_file
|
||||||
|
DROP CONSTRAINT IF EXISTS ck_chat_thread_file_progress_percent;
|
||||||
|
ALTER TABLE public.chat_thread_file
|
||||||
|
ADD CONSTRAINT ck_chat_thread_file_progress_percent
|
||||||
|
CHECK (progress_percent >= 0 AND progress_percent <= 100);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN public.chat_thread_file.progress_percent IS '处理进度 0-100';
|
||||||
|
COMMENT ON COLUMN public.chat_thread_file.processing_stage IS 'processing 阶段';
|
||||||
|
|
||||||
|
-- 知识图谱
|
||||||
|
ALTER TABLE public.graphs
|
||||||
|
ADD COLUMN IF NOT EXISTS progress_percent smallint NOT NULL DEFAULT 0,
|
||||||
|
ADD COLUMN IF NOT EXISTS processing_stage varchar(50) NULL;
|
||||||
|
|
||||||
|
ALTER TABLE public.graphs
|
||||||
|
DROP CONSTRAINT IF EXISTS ck_graphs_progress_percent;
|
||||||
|
ALTER TABLE public.graphs
|
||||||
|
ADD CONSTRAINT ck_graphs_progress_percent
|
||||||
|
CHECK (progress_percent >= 0 AND progress_percent <= 100);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN public.graphs.progress_percent IS '构建进度 0-100';
|
||||||
|
COMMENT ON COLUMN public.graphs.processing_stage IS '构建阶段: extracting/indexing/completed 等';
|
||||||
|
|
||||||
|
-- 知识加工任务(可选:与文件上传同属长任务)
|
||||||
|
ALTER TABLE public.knowledge_processing_task
|
||||||
|
ADD COLUMN IF NOT EXISTS progress_percent smallint NOT NULL DEFAULT 0,
|
||||||
|
ADD COLUMN IF NOT EXISTS processing_stage varchar(50) NULL;
|
||||||
|
|
||||||
|
ALTER TABLE public.knowledge_processing_task
|
||||||
|
DROP CONSTRAINT IF EXISTS ck_kb_processing_task_progress_percent;
|
||||||
|
ALTER TABLE public.knowledge_processing_task
|
||||||
|
ADD CONSTRAINT ck_kb_processing_task_progress_percent
|
||||||
|
CHECK (progress_percent >= 0 AND progress_percent <= 100);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN public.knowledge_processing_task.progress_percent IS '任务进度 0-100';
|
||||||
|
COMMENT ON COLUMN public.knowledge_processing_task.processing_stage IS '任务阶段';
|
||||||
|
|
@ -17,6 +17,8 @@ class ChatThreadFile(BaseModel):
|
||||||
file_type: str = Field(default="pdf", max_length=50)
|
file_type: str = Field(default="pdf", max_length=50)
|
||||||
status: str = Field(default="processing", max_length=20)
|
status: str = Field(default="processing", max_length=20)
|
||||||
chunk_count: int = 0
|
chunk_count: int = 0
|
||||||
|
progress_percent: int = 0
|
||||||
|
processing_stage: Optional[str] = Field(default=None, max_length=50)
|
||||||
created_at: Optional[datetime] = None
|
created_at: Optional[datetime] = None
|
||||||
updated_at: Optional[datetime] = None
|
updated_at: Optional[datetime] = None
|
||||||
is_deleted: bool = False
|
is_deleted: bool = False
|
||||||
|
|
@ -48,6 +50,8 @@ class ChatThreadFileUploadResponse(BaseModel):
|
||||||
file_size: int
|
file_size: int
|
||||||
status: str
|
status: str
|
||||||
chunk_count: int
|
chunk_count: int
|
||||||
|
progress_percent: int = 0
|
||||||
|
processing_stage: Optional[str] = None
|
||||||
created_at: datetime
|
created_at: datetime
|
||||||
file_url: Optional[str] = Field(None, description="文件访问 URL(OSS 或本地路径)")
|
file_url: Optional[str] = Field(None, description="文件访问 URL(OSS 或本地路径)")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,8 @@ class KnowledgeBaseFile(BaseModel):
|
||||||
file_type: str = Field(default="pdf", max_length=50)
|
file_type: str = Field(default="pdf", max_length=50)
|
||||||
status: str = Field(default="processing", max_length=20)
|
status: str = Field(default="processing", max_length=20)
|
||||||
chunk_count: int = 0
|
chunk_count: int = 0
|
||||||
|
progress_percent: int = 0
|
||||||
|
processing_stage: Optional[str] = Field(default=None, max_length=50)
|
||||||
created_at: Optional[datetime] = None
|
created_at: Optional[datetime] = None
|
||||||
updated_at: Optional[datetime] = None
|
updated_at: Optional[datetime] = None
|
||||||
is_deleted: bool = False
|
is_deleted: bool = False
|
||||||
|
|
@ -48,6 +50,8 @@ class FileUploadResponse(BaseModel):
|
||||||
file_size: int
|
file_size: int
|
||||||
status: str
|
status: str
|
||||||
chunk_count: int
|
chunk_count: int
|
||||||
|
progress_percent: int = 0
|
||||||
|
processing_stage: Optional[str] = None
|
||||||
created_at: datetime
|
created_at: datetime
|
||||||
file_url: Optional[str] = Field(None, description="文件访问 URL(OSS 或本地路径)")
|
file_url: Optional[str] = Field(None, description="文件访问 URL(OSS 或本地路径)")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,12 @@ from logger.logging import get_logger
|
||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
_FILE_COLUMNS = """
|
||||||
|
id, thread_id, user_id, file_name, file_path, file_size,
|
||||||
|
file_type, status, chunk_count, progress_percent, processing_stage,
|
||||||
|
created_at, updated_at, is_deleted, deleted_at
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
class ChatThreadFileService:
|
class ChatThreadFileService:
|
||||||
"""聊天对话文件服务类"""
|
"""聊天对话文件服务类"""
|
||||||
|
|
@ -57,12 +63,11 @@ class ChatThreadFileService:
|
||||||
|
|
||||||
# 插入文件记录
|
# 插入文件记录
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
f"""
|
||||||
INSERT INTO chat_thread_file
|
INSERT INTO chat_thread_file
|
||||||
(thread_id, user_id, file_name, file_path, file_size, file_type, status)
|
(thread_id, user_id, file_name, file_path, file_size, file_type, status)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, 'processing')
|
VALUES ($1, $2, $3, $4, $5, $6, 'processing')
|
||||||
RETURNING id, thread_id, user_id, file_name, file_path, file_size,
|
RETURNING {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
""",
|
""",
|
||||||
thread_id, user_id, file_name, file_path, file_size, file_type
|
thread_id, user_id, file_name, file_path, file_size, file_type
|
||||||
)
|
)
|
||||||
|
|
@ -96,14 +101,37 @@ class ChatThreadFileService:
|
||||||
bool: 是否更新成功
|
bool: 是否更新成功
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = await conn.execute(
|
if status == "completed":
|
||||||
"""
|
result = await conn.execute(
|
||||||
UPDATE chat_thread_file
|
"""
|
||||||
SET status = $1, chunk_count = $2
|
UPDATE chat_thread_file
|
||||||
WHERE id = $3
|
SET status = $1, chunk_count = $2,
|
||||||
""",
|
progress_percent = 100, processing_stage = 'completed',
|
||||||
status, chunk_count, file_id
|
updated_at = CURRENT_TIMESTAMP
|
||||||
)
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
status, chunk_count, file_id
|
||||||
|
)
|
||||||
|
elif status == "failed":
|
||||||
|
result = await conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE chat_thread_file
|
||||||
|
SET status = $1, chunk_count = $2,
|
||||||
|
progress_percent = 0, processing_stage = 'failed',
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
status, chunk_count, file_id
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
result = await conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE chat_thread_file
|
||||||
|
SET status = $1, chunk_count = $2, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
status, chunk_count, file_id
|
||||||
|
)
|
||||||
|
|
||||||
return result == "UPDATE 1"
|
return result == "UPDATE 1"
|
||||||
|
|
||||||
|
|
@ -111,6 +139,30 @@ class ChatThreadFileService:
|
||||||
logger.error(f"更新文件状态失败: {e}")
|
logger.error(f"更新文件状态失败: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def update_file_progress(
|
||||||
|
conn: asyncpg.Connection,
|
||||||
|
file_id: int,
|
||||||
|
progress_percent: int,
|
||||||
|
processing_stage: str,
|
||||||
|
) -> bool:
|
||||||
|
"""更新文件处理进度(0-100)与阶段。"""
|
||||||
|
try:
|
||||||
|
result = await conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE chat_thread_file
|
||||||
|
SET progress_percent = $1, processing_stage = $2, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
max(0, min(100, int(progress_percent))),
|
||||||
|
processing_stage,
|
||||||
|
file_id,
|
||||||
|
)
|
||||||
|
return result == "UPDATE 1"
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"更新文件进度失败: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def save_chunks(
|
async def save_chunks(
|
||||||
conn: asyncpg.Connection,
|
conn: asyncpg.Connection,
|
||||||
|
|
@ -174,9 +226,8 @@ class ChatThreadFileService:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
f"""
|
||||||
SELECT id, thread_id, user_id, file_name, file_path, file_size,
|
SELECT {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
FROM chat_thread_file
|
FROM chat_thread_file
|
||||||
WHERE id = $1 AND user_id = $2 AND is_deleted = FALSE
|
WHERE id = $1 AND user_id = $2 AND is_deleted = FALSE
|
||||||
""",
|
""",
|
||||||
|
|
@ -278,9 +329,8 @@ class ChatThreadFileService:
|
||||||
|
|
||||||
# 获取列表
|
# 获取列表
|
||||||
rows = await conn.fetch(
|
rows = await conn.fetch(
|
||||||
"""
|
f"""
|
||||||
SELECT id, thread_id, user_id, file_name, file_path, file_size,
|
SELECT {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
FROM chat_thread_file
|
FROM chat_thread_file
|
||||||
WHERE thread_id = $1 AND user_id = $2 AND is_deleted = FALSE
|
WHERE thread_id = $1 AND user_id = $2 AND is_deleted = FALSE
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
|
|
@ -313,9 +363,8 @@ class ChatThreadFileService:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
rows = await conn.fetch(
|
rows = await conn.fetch(
|
||||||
"""
|
f"""
|
||||||
SELECT id, thread_id, user_id, file_name, file_path, file_size,
|
SELECT {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
FROM chat_thread_file
|
FROM chat_thread_file
|
||||||
WHERE thread_id = $1 AND is_deleted = FALSE
|
WHERE thread_id = $1 AND is_deleted = FALSE
|
||||||
""",
|
""",
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
"""
|
||||||
|
文件/长任务处理进度:阶段常量与上报助手。
|
||||||
|
|
||||||
|
进度约定:
|
||||||
|
- 0–30:HTTP 上传(主要由前端 onUploadProgress 展示;入库后后端从 30 起)
|
||||||
|
- 30–90:解析 / OCR / 分块 / 向量化
|
||||||
|
- 90–100:摘要与收尾
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Awaitable, Callable, Optional, Type
|
||||||
|
|
||||||
|
import asyncpg
|
||||||
|
|
||||||
|
# ---------- 阶段 ----------
|
||||||
|
STAGE_UPLOADING = "uploading"
|
||||||
|
STAGE_PROCESSING = "processing"
|
||||||
|
STAGE_DOWNLOADING = "downloading"
|
||||||
|
STAGE_PARSING = "parsing"
|
||||||
|
STAGE_OCR = "ocr"
|
||||||
|
STAGE_SPLITTING = "splitting"
|
||||||
|
STAGE_EMBEDDING = "embedding"
|
||||||
|
STAGE_SUMMARIZING = "summarizing"
|
||||||
|
STAGE_EXTRACTING = "extracting"
|
||||||
|
STAGE_INDEXING = "indexing"
|
||||||
|
STAGE_COMPLETED = "completed"
|
||||||
|
STAGE_FAILED = "failed"
|
||||||
|
|
||||||
|
ProgressCallback = Callable[[int, str], Awaitable[None]]
|
||||||
|
|
||||||
|
STAGE_LABELS: dict[str, str] = {
|
||||||
|
STAGE_UPLOADING: "上传中",
|
||||||
|
STAGE_PROCESSING: "处理中",
|
||||||
|
STAGE_DOWNLOADING: "下载文件",
|
||||||
|
STAGE_PARSING: "解析文档",
|
||||||
|
STAGE_OCR: "文字识别",
|
||||||
|
STAGE_SPLITTING: "文本分块",
|
||||||
|
STAGE_EMBEDDING: "向量化",
|
||||||
|
STAGE_SUMMARIZING: "生成摘要",
|
||||||
|
STAGE_EXTRACTING: "抽取实体关系",
|
||||||
|
STAGE_INDEXING: "建立索引",
|
||||||
|
STAGE_COMPLETED: "已完成",
|
||||||
|
STAGE_FAILED: "失败",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _noop_progress(_percent: int, _stage: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class FileProgressReporter:
|
||||||
|
"""节流上报,避免同一进度重复写库。"""
|
||||||
|
|
||||||
|
def __init__(self, callback: Optional[ProgressCallback] = None):
|
||||||
|
self._callback = callback or _noop_progress
|
||||||
|
self._last_percent = -1
|
||||||
|
|
||||||
|
async def report(self, percent: int, stage: str) -> None:
|
||||||
|
percent = max(0, min(100, int(percent)))
|
||||||
|
if percent <= self._last_percent:
|
||||||
|
return
|
||||||
|
self._last_percent = percent
|
||||||
|
await self._callback(percent, stage)
|
||||||
|
|
||||||
|
|
||||||
|
def make_db_progress_callback(
|
||||||
|
pool,
|
||||||
|
file_id: int,
|
||||||
|
service_cls: Type,
|
||||||
|
) -> ProgressCallback:
|
||||||
|
"""为知识库/聊天文件创建写库进度回调。"""
|
||||||
|
|
||||||
|
async def _callback(percent: int, stage: str) -> None:
|
||||||
|
async with pool.acquire() as conn:
|
||||||
|
await service_cls.update_file_progress(conn, file_id, percent, stage)
|
||||||
|
|
||||||
|
return _callback
|
||||||
|
|
||||||
|
|
||||||
|
def stage_label(stage: Optional[str]) -> str:
|
||||||
|
if not stage:
|
||||||
|
return "处理中"
|
||||||
|
return STAGE_LABELS.get(stage, stage)
|
||||||
|
|
@ -13,6 +13,12 @@ from logger.logging import get_logger
|
||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
_FILE_COLUMNS = """
|
||||||
|
id, knowledge_base_id, user_id, file_name, file_path, file_size,
|
||||||
|
file_type, status, chunk_count, progress_percent, processing_stage,
|
||||||
|
created_at, updated_at, is_deleted, deleted_at
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
class KnowledgeBaseFileService:
|
class KnowledgeBaseFileService:
|
||||||
"""知识库文件服务类"""
|
"""知识库文件服务类"""
|
||||||
|
|
@ -25,9 +31,8 @@ class KnowledgeBaseFileService:
|
||||||
) -> Optional[KnowledgeBaseFile]:
|
) -> Optional[KnowledgeBaseFile]:
|
||||||
"""按知识库 + 文件名查询未删除的文件记录。"""
|
"""按知识库 + 文件名查询未删除的文件记录。"""
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
f"""
|
||||||
SELECT id, knowledge_base_id, user_id, file_name, file_path, file_size,
|
SELECT {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
FROM knowledge_base_file
|
FROM knowledge_base_file
|
||||||
WHERE knowledge_base_id = $1 AND file_name = $2 AND is_deleted = FALSE
|
WHERE knowledge_base_id = $1 AND file_name = $2 AND is_deleted = FALSE
|
||||||
""",
|
""",
|
||||||
|
|
@ -69,14 +74,14 @@ class KnowledgeBaseFileService:
|
||||||
if existing:
|
if existing:
|
||||||
if existing.status == "failed":
|
if existing.status == "failed":
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
f"""
|
||||||
UPDATE knowledge_base_file
|
UPDATE knowledge_base_file
|
||||||
SET file_path = $1, file_size = $2, file_type = $3,
|
SET file_path = $1, file_size = $2, file_type = $3,
|
||||||
status = 'processing', chunk_count = 0,
|
status = 'processing', chunk_count = 0,
|
||||||
|
progress_percent = 0, processing_stage = 'processing',
|
||||||
user_id = $4, updated_at = CURRENT_TIMESTAMP
|
user_id = $4, updated_at = CURRENT_TIMESTAMP
|
||||||
WHERE id = $5
|
WHERE id = $5
|
||||||
RETURNING id, knowledge_base_id, user_id, file_name, file_path, file_size,
|
RETURNING {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
""",
|
""",
|
||||||
file_path,
|
file_path,
|
||||||
file_size,
|
file_size,
|
||||||
|
|
@ -93,12 +98,11 @@ class KnowledgeBaseFileService:
|
||||||
|
|
||||||
# 插入文件记录
|
# 插入文件记录
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
f"""
|
||||||
INSERT INTO knowledge_base_file
|
INSERT INTO knowledge_base_file
|
||||||
(knowledge_base_id, user_id, file_name, file_path, file_size, file_type, status)
|
(knowledge_base_id, user_id, file_name, file_path, file_size, file_type, status)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, 'processing')
|
VALUES ($1, $2, $3, $4, $5, $6, 'processing')
|
||||||
RETURNING id, knowledge_base_id, user_id, file_name, file_path, file_size,
|
RETURNING {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
""",
|
""",
|
||||||
knowledge_base_id, user_id, file_name, file_path, file_size, file_type
|
knowledge_base_id, user_id, file_name, file_path, file_size, file_type
|
||||||
)
|
)
|
||||||
|
|
@ -132,14 +136,37 @@ class KnowledgeBaseFileService:
|
||||||
bool: 是否更新成功
|
bool: 是否更新成功
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = await conn.execute(
|
if status == "completed":
|
||||||
"""
|
result = await conn.execute(
|
||||||
UPDATE knowledge_base_file
|
"""
|
||||||
SET status = $1, chunk_count = $2
|
UPDATE knowledge_base_file
|
||||||
WHERE id = $3
|
SET status = $1, chunk_count = $2,
|
||||||
""",
|
progress_percent = 100, processing_stage = 'completed',
|
||||||
status, chunk_count, file_id
|
updated_at = CURRENT_TIMESTAMP
|
||||||
)
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
status, chunk_count, file_id
|
||||||
|
)
|
||||||
|
elif status == "failed":
|
||||||
|
result = await conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE knowledge_base_file
|
||||||
|
SET status = $1, chunk_count = $2,
|
||||||
|
progress_percent = 0, processing_stage = 'failed',
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
status, chunk_count, file_id
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
result = await conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE knowledge_base_file
|
||||||
|
SET status = $1, chunk_count = $2, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
status, chunk_count, file_id
|
||||||
|
)
|
||||||
|
|
||||||
return result == "UPDATE 1"
|
return result == "UPDATE 1"
|
||||||
|
|
||||||
|
|
@ -147,6 +174,30 @@ class KnowledgeBaseFileService:
|
||||||
logger.error(f"更新文件状态失败: {e}")
|
logger.error(f"更新文件状态失败: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def update_file_progress(
|
||||||
|
conn: asyncpg.Connection,
|
||||||
|
file_id: int,
|
||||||
|
progress_percent: int,
|
||||||
|
processing_stage: str,
|
||||||
|
) -> bool:
|
||||||
|
"""更新文件处理进度(0-100)与阶段。"""
|
||||||
|
try:
|
||||||
|
result = await conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE knowledge_base_file
|
||||||
|
SET progress_percent = $1, processing_stage = $2, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = $3
|
||||||
|
""",
|
||||||
|
max(0, min(100, int(progress_percent))),
|
||||||
|
processing_stage,
|
||||||
|
file_id,
|
||||||
|
)
|
||||||
|
return result == "UPDATE 1"
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"更新文件进度失败: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def save_chunks(
|
async def save_chunks(
|
||||||
conn: asyncpg.Connection,
|
conn: asyncpg.Connection,
|
||||||
|
|
@ -204,9 +255,8 @@ class KnowledgeBaseFileService:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
f"""
|
||||||
SELECT id, knowledge_base_id, user_id, file_name, file_path, file_size,
|
SELECT {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
FROM knowledge_base_file
|
FROM knowledge_base_file
|
||||||
WHERE id = $1 AND is_deleted = FALSE
|
WHERE id = $1 AND is_deleted = FALSE
|
||||||
""",
|
""",
|
||||||
|
|
@ -252,6 +302,7 @@ class KnowledgeBaseFileService:
|
||||||
"""
|
"""
|
||||||
SELECT f.id, f.knowledge_base_id, f.user_id, f.file_name, f.file_path,
|
SELECT f.id, f.knowledge_base_id, f.user_id, f.file_name, f.file_path,
|
||||||
f.file_size, f.file_type, f.status, f.chunk_count,
|
f.file_size, f.file_type, f.status, f.chunk_count,
|
||||||
|
f.progress_percent, f.processing_stage,
|
||||||
f.created_at, f.updated_at, f.is_deleted, f.deleted_at,
|
f.created_at, f.updated_at, f.is_deleted, f.deleted_at,
|
||||||
COALESCE(NULLIF(TRIM(u.display_name),''), u.username) AS uploader_name
|
COALESCE(NULLIF(TRIM(u.display_name),''), u.username) AS uploader_name
|
||||||
FROM knowledge_base_file f
|
FROM knowledge_base_file f
|
||||||
|
|
@ -316,9 +367,8 @@ class KnowledgeBaseFileService:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
rows = await conn.fetch(
|
rows = await conn.fetch(
|
||||||
"""
|
f"""
|
||||||
SELECT id, knowledge_base_id, user_id, file_name, file_path, file_size,
|
SELECT {_FILE_COLUMNS}
|
||||||
file_type, status, chunk_count, created_at, updated_at, is_deleted, deleted_at
|
|
||||||
FROM knowledge_base_file
|
FROM knowledge_base_file
|
||||||
WHERE knowledge_base_id = $1
|
WHERE knowledge_base_id = $1
|
||||||
""",
|
""",
|
||||||
|
|
|
||||||
|
|
@ -80,12 +80,37 @@ from logger.logging import get_logger
|
||||||
from core.config import settings
|
from core.config import settings
|
||||||
from core.llm_env import tongyi_embedding_api_base, tongyi_embedding_api_key
|
from core.llm_env import tongyi_embedding_api_base, tongyi_embedding_api_key
|
||||||
from services.kb_text_limits import validate_kb_text_length, validate_chat_file_text_length
|
from services.kb_text_limits import validate_kb_text_length, validate_chat_file_text_length
|
||||||
|
from services.file_progress import (
|
||||||
|
FileProgressReporter,
|
||||||
|
STAGE_PARSING,
|
||||||
|
STAGE_OCR,
|
||||||
|
STAGE_SPLITTING,
|
||||||
|
STAGE_EMBEDDING,
|
||||||
|
)
|
||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
# 阿里云 OCR 图片边长限制(见 illegalImageSize 错误说明)
|
||||||
|
ALIYUN_OCR_MAX_SIDE = 8192
|
||||||
|
ALIYUN_OCR_MIN_SIDE = 5
|
||||||
|
|
||||||
|
_VISION_EXTRACT_PROMPT = (
|
||||||
|
"详细描述图片中的内容:场景、人物、物体、图表及所有可见文字(逐字提取)。"
|
||||||
|
"用通顺中文输出,便于后续检索与问答。"
|
||||||
|
)
|
||||||
|
|
||||||
# 通义 text-embedding-v4(OpenAI 兼容)单次请求最多 10 条,超出会 400/500
|
# 通义 text-embedding-v4(OpenAI 兼容)单次请求最多 10 条,超出会 400/500
|
||||||
_TONGYI_EMBEDDING_MAX_BATCH = 10
|
_TONGYI_EMBEDDING_MAX_BATCH = 10
|
||||||
|
|
||||||
|
# 阿里云 OCR 图片边长限制:https://help.aliyun.com/document_detail/442266.html
|
||||||
|
ALIYUN_OCR_MAX_SIDE = 8192
|
||||||
|
ALIYUN_OCR_MIN_SIDE = 5
|
||||||
|
|
||||||
|
_VISION_IMAGE_PROMPT = (
|
||||||
|
"详细描述图片中的内容:场景、人物、物体、图表及所有可见文字(逐字提取)。"
|
||||||
|
"用通顺中文输出,便于后续检索与问答。"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TongyiEmbeddings(Embeddings):
|
class TongyiEmbeddings(Embeddings):
|
||||||
"""通义千问 Embedding 封装:固定走 ZL/DashScope 网关,并按 API 上限分批请求。"""
|
"""通义千问 Embedding 封装:固定走 ZL/DashScope 网关,并按 API 上限分批请求。"""
|
||||||
|
|
@ -205,21 +230,15 @@ class VectorService:
|
||||||
|
|
||||||
# 初始化阿里云 OCR(图片、扫描 PDF、DOCX 内嵌图均依赖云端识别)
|
# 初始化阿里云 OCR(图片、扫描 PDF、DOCX 内嵌图均依赖云端识别)
|
||||||
self.ocr_engine = None
|
self.ocr_engine = None
|
||||||
self._ocr_timeout_ms = int(settings.ocr_timeout_seconds * 1000)
|
|
||||||
if ALIYUN_OCR_AVAILABLE and settings.ocr_access_key_id and settings.ocr_access_key_secret:
|
if ALIYUN_OCR_AVAILABLE and settings.ocr_access_key_id and settings.ocr_access_key_secret:
|
||||||
try:
|
try:
|
||||||
config = open_api_models.Config(
|
config = open_api_models.Config(
|
||||||
access_key_id=settings.ocr_access_key_id,
|
access_key_id=settings.ocr_access_key_id,
|
||||||
access_key_secret=settings.ocr_access_key_secret,
|
access_key_secret=settings.ocr_access_key_secret,
|
||||||
endpoint=settings.ocr_endpoint,
|
endpoint=settings.ocr_endpoint
|
||||||
connect_timeout=self._ocr_timeout_ms,
|
|
||||||
read_timeout=self._ocr_timeout_ms,
|
|
||||||
)
|
)
|
||||||
self.ocr_engine = OcrClient(config)
|
self.ocr_engine = OcrClient(config)
|
||||||
logger.info(
|
logger.info("✅ 阿里云 OCR 已启用,将使用云端 OCR 服务识别图片文字")
|
||||||
"✅ 阿里云 OCR 已启用,将使用云端 OCR 服务识别图片文字 "
|
|
||||||
f"(endpoint={settings.ocr_endpoint}, timeout={settings.ocr_timeout_seconds}s)"
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"⚠️ 阿里云 OCR 初始化失败: {e}")
|
logger.warning(f"⚠️ 阿里云 OCR 初始化失败: {e}")
|
||||||
elif not ALIYUN_OCR_AVAILABLE:
|
elif not ALIYUN_OCR_AVAILABLE:
|
||||||
|
|
@ -230,6 +249,89 @@ class VectorService:
|
||||||
if not self.ocr_engine:
|
if not self.ocr_engine:
|
||||||
logger.warning("⚠️ OCR 服务不可用,图片与扫描件内容将无法通过 OCR 提取。请配置阿里云 OCR")
|
logger.warning("⚠️ OCR 服务不可用,图片与扫描件内容将无法通过 OCR 提取。请配置阿里云 OCR")
|
||||||
|
|
||||||
|
def _resize_image_bytes_for_ocr(self, image_bytes: bytes) -> bytes:
|
||||||
|
"""将图片缩放到阿里云 OCR 允许的像素范围内。"""
|
||||||
|
if not PILLOW_AVAILABLE or not image_bytes:
|
||||||
|
return image_bytes
|
||||||
|
try:
|
||||||
|
with Image.open(io.BytesIO(image_bytes)) as img:
|
||||||
|
w, h = img.size
|
||||||
|
if (
|
||||||
|
ALIYUN_OCR_MIN_SIDE <= w <= ALIYUN_OCR_MAX_SIDE
|
||||||
|
and ALIYUN_OCR_MIN_SIDE <= h <= ALIYUN_OCR_MAX_SIDE
|
||||||
|
):
|
||||||
|
return image_bytes
|
||||||
|
|
||||||
|
scale = min(ALIYUN_OCR_MAX_SIDE / w, ALIYUN_OCR_MAX_SIDE / h, 1.0)
|
||||||
|
if w < ALIYUN_OCR_MIN_SIDE or h < ALIYUN_OCR_MIN_SIDE:
|
||||||
|
scale = max(scale, ALIYUN_OCR_MIN_SIDE / min(w, h))
|
||||||
|
|
||||||
|
new_w = max(ALIYUN_OCR_MIN_SIDE, min(ALIYUN_OCR_MAX_SIDE, int(w * scale)))
|
||||||
|
new_h = max(ALIYUN_OCR_MIN_SIDE, min(ALIYUN_OCR_MAX_SIDE, int(h * scale)))
|
||||||
|
if (new_w, new_h) == (w, h):
|
||||||
|
return image_bytes
|
||||||
|
|
||||||
|
rgb = img.convert("RGB")
|
||||||
|
resized = rgb.resize((new_w, new_h), Image.Resampling.LANCZOS)
|
||||||
|
buf = io.BytesIO()
|
||||||
|
fmt = "PNG" if image_bytes[:8] == b"\x89PNG\r\n\x1a\n" else "JPEG"
|
||||||
|
save_kwargs = {"format": fmt}
|
||||||
|
if fmt == "JPEG":
|
||||||
|
save_kwargs["quality"] = 90
|
||||||
|
resized.save(buf, **save_kwargs)
|
||||||
|
logger.info(f"📐 [OCR] 图片已缩放: {w}x{h} -> {new_w}x{new_h}")
|
||||||
|
return buf.getvalue()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"图片缩放失败,将使用原图尝试 OCR: {e}")
|
||||||
|
return image_bytes
|
||||||
|
|
||||||
|
def _ensure_image_file_for_ocr(self, image_path: str) -> None:
|
||||||
|
"""就地规范化图片文件,满足阿里云 OCR 像素限制。"""
|
||||||
|
try:
|
||||||
|
with open(image_path, "rb") as f:
|
||||||
|
raw = f.read()
|
||||||
|
normalized = self._resize_image_bytes_for_ocr(raw)
|
||||||
|
if normalized != raw:
|
||||||
|
with open(image_path, "wb") as f:
|
||||||
|
f.write(normalized)
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f"规范化 OCR 图片文件失败: {image_path}, {e}")
|
||||||
|
|
||||||
|
async def _describe_image_file_with_vision(self, image_path: str) -> str:
|
||||||
|
"""使用视觉模型理解单张图片(本地文件)。"""
|
||||||
|
from services.vision_service import VisionService
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(image_path, "rb") as f:
|
||||||
|
image_bytes = f.read()
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f"读取图片失败,无法使用视觉模型: {image_path}, {e}")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
ext = os.path.splitext(image_path)[1].lower()
|
||||||
|
mime_map = {
|
||||||
|
".png": "image/png",
|
||||||
|
".jpg": "image/jpeg",
|
||||||
|
".jpeg": "image/jpeg",
|
||||||
|
".bmp": "image/bmp",
|
||||||
|
".webp": "image/webp",
|
||||||
|
".gif": "image/gif",
|
||||||
|
}
|
||||||
|
vision_text = await VisionService.get_image_description_from_bytes(
|
||||||
|
image_bytes,
|
||||||
|
prompt=_VISION_EXTRACT_PROMPT,
|
||||||
|
mime_hint=mime_map.get(ext, "image/jpeg"),
|
||||||
|
)
|
||||||
|
return (vision_text or "").strip()
|
||||||
|
|
||||||
|
def _cleanup_temp_image_paths(self, image_paths: List[str]) -> None:
|
||||||
|
for img_path in image_paths:
|
||||||
|
try:
|
||||||
|
if os.path.exists(img_path):
|
||||||
|
os.remove(img_path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
def _ocr_image(self, image_path: str) -> str:
|
def _ocr_image(self, image_path: str) -> str:
|
||||||
"""
|
"""
|
||||||
使用阿里云 OCR 识别单张图片中的文字。
|
使用阿里云 OCR 识别单张图片中的文字。
|
||||||
|
|
@ -266,6 +368,8 @@ class VectorService:
|
||||||
with open(image_path, 'rb') as f:
|
with open(image_path, 'rb') as f:
|
||||||
image_bytes = f.read()
|
image_bytes = f.read()
|
||||||
|
|
||||||
|
image_bytes = self._resize_image_bytes_for_ocr(image_bytes)
|
||||||
|
|
||||||
image_size_kb = len(image_bytes) / 1024
|
image_size_kb = len(image_bytes) / 1024
|
||||||
logger.info(f"📊 [阿里云OCR] 图片大小: {image_size_kb:.2f}KB")
|
logger.info(f"📊 [阿里云OCR] 图片大小: {image_size_kb:.2f}KB")
|
||||||
|
|
||||||
|
|
@ -276,13 +380,8 @@ class VectorService:
|
||||||
# 构建请求
|
# 构建请求
|
||||||
request = ocr_models.RecognizeGeneralRequest(body=body_stream)
|
request = ocr_models.RecognizeGeneralRequest(body=body_stream)
|
||||||
|
|
||||||
# 运行时选项(默认超时过短会导致大图片 write timeout)
|
# 运行时选项
|
||||||
runtime = util_models.RuntimeOptions(
|
runtime = util_models.RuntimeOptions()
|
||||||
connect_timeout=self._ocr_timeout_ms,
|
|
||||||
read_timeout=self._ocr_timeout_ms,
|
|
||||||
autoretry=True,
|
|
||||||
max_attempts=2,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.debug(f"☁️ [阿里云OCR] 调用 API: recognize_general_with_options")
|
logger.debug(f"☁️ [阿里云OCR] 调用 API: recognize_general_with_options")
|
||||||
# 调用阿里云 OCR API(使用 with_options 版本)
|
# 调用阿里云 OCR API(使用 with_options 版本)
|
||||||
|
|
@ -387,7 +486,6 @@ class VectorService:
|
||||||
处理知识库图片:优先 OCR;OCR 无文字或失败时,必须使用视觉模型提取内容。
|
处理知识库图片:优先 OCR;OCR 无文字或失败时,必须使用视觉模型提取内容。
|
||||||
"""
|
"""
|
||||||
from langchain_core.documents import Document
|
from langchain_core.documents import Document
|
||||||
from services.vision_service import VisionService
|
|
||||||
|
|
||||||
ocr_docs: List = []
|
ocr_docs: List = []
|
||||||
try:
|
try:
|
||||||
|
|
@ -399,41 +497,17 @@ class VectorService:
|
||||||
return ocr_docs
|
return ocr_docs
|
||||||
|
|
||||||
logger.info("OCR 未识别到文字,使用视觉模型处理图片...")
|
logger.info("OCR 未识别到文字,使用视觉模型处理图片...")
|
||||||
vision_prompt = (
|
|
||||||
"详细描述图片中的内容:场景、人物、物体、图表及所有可见文字(逐字提取)。"
|
|
||||||
"用通顺中文输出,便于后续检索与问答。"
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
vision_text = await self._describe_image_file_with_vision(file_path)
|
||||||
with open(file_path, "rb") as f:
|
|
||||||
image_bytes = f.read()
|
|
||||||
except OSError as e:
|
|
||||||
logger.error(f"读取图片文件失败,无法进行视觉理解: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
ext = os.path.splitext(file_path)[1].lower()
|
if not vision_text:
|
||||||
mime_map = {
|
|
||||||
".png": "image/png",
|
|
||||||
".jpg": "image/jpeg",
|
|
||||||
".jpeg": "image/jpeg",
|
|
||||||
".bmp": "image/bmp",
|
|
||||||
".webp": "image/webp",
|
|
||||||
".gif": "image/gif",
|
|
||||||
}
|
|
||||||
vision_text = await VisionService.get_image_description_from_bytes(
|
|
||||||
image_bytes,
|
|
||||||
prompt=vision_prompt,
|
|
||||||
mime_hint=mime_map.get(ext, "image/jpeg"),
|
|
||||||
)
|
|
||||||
|
|
||||||
if not vision_text or not vision_text.strip():
|
|
||||||
logger.warning(f"视觉模型也未提取到内容: {file_path}")
|
logger.warning(f"视觉模型也未提取到内容: {file_path}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
logger.info(f"视觉模型成功提取图片内容,共 {len(vision_text)} 字符")
|
logger.info(f"视觉模型成功提取图片内容,共 {len(vision_text)} 字符")
|
||||||
return [
|
return [
|
||||||
Document(
|
Document(
|
||||||
page_content=f"【图片内容描述】\n{vision_text.strip()}",
|
page_content=f"【图片内容描述】\n{vision_text}",
|
||||||
metadata={
|
metadata={
|
||||||
"source": file_path,
|
"source": file_path,
|
||||||
"file_type": "image",
|
"file_type": "image",
|
||||||
|
|
@ -444,6 +518,83 @@ class VectorService:
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
async def _process_pdf_with_ocr_and_vision_fallback(self, file_path: str) -> List:
|
||||||
|
"""
|
||||||
|
处理扫描版 PDF:逐页 OCR;单页 OCR 失败或无文字时,回退视觉模型。
|
||||||
|
"""
|
||||||
|
from langchain_core.documents import Document
|
||||||
|
|
||||||
|
image_paths = await asyncio.to_thread(self._extract_images_from_pdf, file_path)
|
||||||
|
if not image_paths:
|
||||||
|
logger.warning("未能从 PDF 提取任何页面")
|
||||||
|
return []
|
||||||
|
|
||||||
|
page_texts: List[str] = []
|
||||||
|
ocr_page_count = 0
|
||||||
|
vision_page_count = 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
for img_path in image_paths:
|
||||||
|
await asyncio.to_thread(self._ensure_image_file_for_ocr, img_path)
|
||||||
|
|
||||||
|
ocr_results: List[Tuple[int, str]] = []
|
||||||
|
if self.ocr_engine:
|
||||||
|
logger.info(f"开始并发 OCR 识别 {len(image_paths)} 页 PDF(并发数: 4)")
|
||||||
|
ocr_results = await asyncio.to_thread(
|
||||||
|
self._ocr_images_concurrent, image_paths, 4
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning("OCR 不可用,扫描版 PDF 将直接使用视觉模型逐页识别")
|
||||||
|
ocr_results = [(idx, "") for idx in range(len(image_paths))]
|
||||||
|
|
||||||
|
for idx, ocr_text in ocr_results:
|
||||||
|
page_no = idx + 1
|
||||||
|
if ocr_text and ocr_text.strip():
|
||||||
|
page_texts.append(f"[第 {page_no} 页]\n{ocr_text.strip()}")
|
||||||
|
ocr_page_count += 1
|
||||||
|
logger.info(f"第 {page_no} 页 OCR 识别到 {len(ocr_text)} 字符")
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.info(f"第 {page_no} 页 OCR 无结果,尝试视觉模型...")
|
||||||
|
try:
|
||||||
|
vision_text = await self._describe_image_file_with_vision(image_paths[idx])
|
||||||
|
if vision_text:
|
||||||
|
page_texts.append(f"[第 {page_no} 页 - 视觉理解]\n{vision_text}")
|
||||||
|
vision_page_count += 1
|
||||||
|
logger.info(f"第 {page_no} 页视觉模型提取 {len(vision_text)} 字符")
|
||||||
|
else:
|
||||||
|
logger.warning(f"第 {page_no} 页视觉模型也未提取到内容")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"第 {page_no} 页视觉模型失败: {e}")
|
||||||
|
|
||||||
|
if not page_texts:
|
||||||
|
logger.warning("PDF OCR/视觉均未识别到任何文字内容")
|
||||||
|
return []
|
||||||
|
|
||||||
|
full_content = "\n\n".join(page_texts)
|
||||||
|
logger.info(
|
||||||
|
f"PDF 处理完成:共 {len(page_texts)} 页有效内容 "
|
||||||
|
f"(OCR {ocr_page_count} 页, 视觉 {vision_page_count} 页),"
|
||||||
|
f"总计 {len(full_content)} 字符"
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
Document(
|
||||||
|
page_content=full_content,
|
||||||
|
metadata={
|
||||||
|
"source": file_path,
|
||||||
|
"file_type": "pdf",
|
||||||
|
"is_image_pdf": True,
|
||||||
|
"page_count": len(page_texts),
|
||||||
|
"has_ocr": ocr_page_count > 0,
|
||||||
|
"has_vision": vision_page_count > 0,
|
||||||
|
"ocr_provider": "aliyun" if ocr_page_count > 0 else "",
|
||||||
|
"vision_provider": "qwen-vl" if vision_page_count > 0 else "",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
finally:
|
||||||
|
self._cleanup_temp_image_paths(image_paths)
|
||||||
|
|
||||||
def _extract_images_from_docx(self, docx_path: str) -> List[str]:
|
def _extract_images_from_docx(self, docx_path: str) -> List[str]:
|
||||||
"""
|
"""
|
||||||
从 DOCX 文件中提取所有图片并转换为标准格式(PNG/JPG)
|
从 DOCX 文件中提取所有图片并转换为标准格式(PNG/JPG)
|
||||||
|
|
@ -620,6 +771,8 @@ class VectorService:
|
||||||
extracted_image_paths = image_paths.copy() # 保存副本供后续使用
|
extracted_image_paths = image_paths.copy() # 保存副本供后续使用
|
||||||
|
|
||||||
if image_paths:
|
if image_paths:
|
||||||
|
for img_path in image_paths:
|
||||||
|
self._ensure_image_file_for_ocr(img_path)
|
||||||
# 使用多线程并发处理(最多并发 4 张图片)
|
# 使用多线程并发处理(最多并发 4 张图片)
|
||||||
logger.info(f"开始并发 OCR 识别 {len(image_paths)} 张图片(并发数: 4)")
|
logger.info(f"开始并发 OCR 识别 {len(image_paths)} 张图片(并发数: 4)")
|
||||||
ocr_results = self._ocr_images_concurrent(image_paths, max_workers=4)
|
ocr_results = self._ocr_images_concurrent(image_paths, max_workers=4)
|
||||||
|
|
@ -687,9 +840,11 @@ class VectorService:
|
||||||
logger.debug(f" 🔄 [PDF页面提取] 处理第 {page_num + 1}/{total_pages} 页")
|
logger.debug(f" 🔄 [PDF页面提取] 处理第 {page_num + 1}/{total_pages} 页")
|
||||||
page = pdf_document[page_num]
|
page = pdf_document[page_num]
|
||||||
|
|
||||||
# 将页面转换为图片(提高分辨率以提升 OCR 效果)
|
# 按页尺寸计算缩放,避免超过阿里云 OCR 8192px 上限
|
||||||
# zoom=2 表示 2 倍分辨率(DPI 约 144)
|
max_dim = max(page.rect.width, page.rect.height, 1.0)
|
||||||
mat = fitz.Matrix(2, 2)
|
zoom = min(2.0, ALIYUN_OCR_MAX_SIDE / max_dim)
|
||||||
|
zoom = max(zoom, 0.1)
|
||||||
|
mat = fitz.Matrix(zoom, zoom)
|
||||||
pix = page.get_pixmap(matrix=mat)
|
pix = page.get_pixmap(matrix=mat)
|
||||||
|
|
||||||
# 保存为临时图片文件
|
# 保存为临时图片文件
|
||||||
|
|
@ -697,6 +852,7 @@ class VectorService:
|
||||||
pix.save(tmp_file.name)
|
pix.save(tmp_file.name)
|
||||||
tmp_file.close()
|
tmp_file.close()
|
||||||
|
|
||||||
|
self._ensure_image_file_for_ocr(tmp_file.name)
|
||||||
file_size_kb = os.path.getsize(tmp_file.name) / 1024
|
file_size_kb = os.path.getsize(tmp_file.name) / 1024
|
||||||
image_paths.append(tmp_file.name)
|
image_paths.append(tmp_file.name)
|
||||||
logger.info(f"✅ [PDF页面提取] 第 {page_num + 1} 页已转换: {os.path.basename(tmp_file.name)} ({file_size_kb:.2f}KB)")
|
logger.info(f"✅ [PDF页面提取] 第 {page_num + 1} 页已转换: {os.path.basename(tmp_file.name)} ({file_size_kb:.2f}KB)")
|
||||||
|
|
@ -761,6 +917,9 @@ class VectorService:
|
||||||
logger.warning("未能从 PDF 提取任何页面")
|
logger.warning("未能从 PDF 提取任何页面")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
for img_path in image_paths:
|
||||||
|
self._ensure_image_file_for_ocr(img_path)
|
||||||
|
|
||||||
# 2. 使用多线程并发 OCR 识别
|
# 2. 使用多线程并发 OCR 识别
|
||||||
logger.info(f"开始并发 OCR 识别 {len(image_paths)} 页 PDF(并发数: 4)")
|
logger.info(f"开始并发 OCR 识别 {len(image_paths)} 页 PDF(并发数: 4)")
|
||||||
ocr_results = self._ocr_images_concurrent(image_paths, max_workers=4)
|
ocr_results = self._ocr_images_concurrent(image_paths, max_workers=4)
|
||||||
|
|
@ -775,12 +934,7 @@ class VectorService:
|
||||||
logger.warning(f"第 {idx + 1} 页 OCR 未识别到文字")
|
logger.warning(f"第 {idx + 1} 页 OCR 未识别到文字")
|
||||||
|
|
||||||
# 4. 清理临时图片文件
|
# 4. 清理临时图片文件
|
||||||
for img_path in image_paths:
|
self._cleanup_temp_image_paths(image_paths)
|
||||||
try:
|
|
||||||
if os.path.exists(img_path):
|
|
||||||
os.remove(img_path)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if not page_texts:
|
if not page_texts:
|
||||||
logger.warning("PDF OCR 未识别到任何文字内容")
|
logger.warning("PDF OCR 未识别到任何文字内容")
|
||||||
|
|
@ -927,7 +1081,8 @@ class VectorService:
|
||||||
knowledge_base_id: int,
|
knowledge_base_id: int,
|
||||||
file_type: str = "pdf",
|
file_type: str = "pdf",
|
||||||
file_id: Optional[int] = None,
|
file_id: Optional[int] = None,
|
||||||
source_url: Optional[str] = None
|
source_url: Optional[str] = None,
|
||||||
|
progress: Optional[FileProgressReporter] = None,
|
||||||
) -> ProcessResult:
|
) -> ProcessResult:
|
||||||
"""
|
"""
|
||||||
处理文档文件:加载、分割、向量化(支持多种文档格式,包括图片 OCR)
|
处理文档文件:加载、分割、向量化(支持多种文档格式,包括图片 OCR)
|
||||||
|
|
@ -947,6 +1102,8 @@ class VectorService:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
logger.info(f"开始处理文件: {file_path}, 类型: {file_type}")
|
logger.info(f"开始处理文件: {file_path}, 类型: {file_type}")
|
||||||
|
if progress:
|
||||||
|
await progress.report(35, STAGE_PARSING)
|
||||||
|
|
||||||
# 1. 获取合适的加载器
|
# 1. 获取合适的加载器
|
||||||
loader = self._get_loader_for_file(file_path, file_type)
|
loader = self._get_loader_for_file(file_path, file_type)
|
||||||
|
|
@ -956,9 +1113,13 @@ class VectorService:
|
||||||
logger.warning(error_msg)
|
logger.warning(error_msg)
|
||||||
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
||||||
|
|
||||||
|
ocr_stage = loader in ("image_ocr", "docx_with_images") or file_type.lower() == "pdf"
|
||||||
|
|
||||||
# 2. 加载文档(特殊处理图片 OCR 和 DOCX,放到线程池执行)
|
# 2. 加载文档(特殊处理图片 OCR 和 DOCX,放到线程池执行)
|
||||||
if loader == "image_ocr":
|
if loader == "image_ocr":
|
||||||
logger.info("🔄 处理图片:OCR → 视觉模型回退...")
|
logger.info("🔄 处理图片:OCR → 视觉模型回退...")
|
||||||
|
if progress:
|
||||||
|
await progress.report(38, STAGE_OCR)
|
||||||
docs = await self._process_image_with_vision_fallback(file_path)
|
docs = await self._process_image_with_vision_fallback(file_path)
|
||||||
elif loader == "docx_with_images":
|
elif loader == "docx_with_images":
|
||||||
logger.info("🔄 在线程池中处理 DOCX 文件(提取图片并 OCR)...")
|
logger.info("🔄 在线程池中处理 DOCX 文件(提取图片并 OCR)...")
|
||||||
|
|
@ -970,20 +1131,18 @@ class VectorService:
|
||||||
|
|
||||||
# 特殊处理:检测 PDF 是否为图片型(扫描版)
|
# 特殊处理:检测 PDF 是否为图片型(扫描版)
|
||||||
if file_type.lower() == "pdf" and self._is_image_pdf(docs):
|
if file_type.lower() == "pdf" and self._is_image_pdf(docs):
|
||||||
logger.info("检测到图片型 PDF(扫描版),切换到 OCR 模式")
|
logger.info("检测到图片型 PDF(扫描版),OCR → 视觉模型回退")
|
||||||
if self.ocr_engine:
|
if progress:
|
||||||
logger.info("🔄 在线程池中执行 PDF OCR...")
|
await progress.report(38, STAGE_OCR)
|
||||||
docs = await asyncio.to_thread(self._process_pdf_with_ocr, file_path)
|
docs = await self._process_pdf_with_ocr_and_vision_fallback(file_path)
|
||||||
if not docs:
|
if not docs:
|
||||||
error_msg = "图片型 PDF OCR 识别失败"
|
error_msg = "图片型 PDF OCR/视觉识别均失败"
|
||||||
logger.warning(error_msg)
|
|
||||||
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
|
||||||
else:
|
|
||||||
error_msg = "检测到图片型 PDF(扫描版),但 OCR 服务不可用"
|
|
||||||
logger.warning(error_msg)
|
logger.warning(error_msg)
|
||||||
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
||||||
|
|
||||||
logger.info(f"文档加载完成,共 {len(docs)} 个文档片段")
|
logger.info(f"文档加载完成,共 {len(docs)} 个文档片段")
|
||||||
|
if progress:
|
||||||
|
await progress.report(50, STAGE_OCR if ocr_stage else STAGE_PARSING)
|
||||||
|
|
||||||
if not docs:
|
if not docs:
|
||||||
error_msg = (
|
error_msg = (
|
||||||
|
|
@ -1004,6 +1163,8 @@ class VectorService:
|
||||||
# 2. 分割文本
|
# 2. 分割文本
|
||||||
all_splits = self.text_splitter.split_documents(docs)
|
all_splits = self.text_splitter.split_documents(docs)
|
||||||
logger.info(f"文本分割完成,共 {len(all_splits)} 个块")
|
logger.info(f"文本分割完成,共 {len(all_splits)} 个块")
|
||||||
|
if progress:
|
||||||
|
await progress.report(55, STAGE_SPLITTING)
|
||||||
|
|
||||||
# 检查是否有内容
|
# 检查是否有内容
|
||||||
if not all_splits:
|
if not all_splits:
|
||||||
|
|
@ -1021,6 +1182,8 @@ class VectorService:
|
||||||
# 3. 向量化并存储
|
# 3. 向量化并存储
|
||||||
collection_name = f"kb_{knowledge_base_id}"
|
collection_name = f"kb_{knowledge_base_id}"
|
||||||
vector_store = self.get_vector_store(collection_name)
|
vector_store = self.get_vector_store(collection_name)
|
||||||
|
if progress:
|
||||||
|
await progress.report(60, STAGE_EMBEDDING)
|
||||||
|
|
||||||
# 🔑 关键:在向量化前,将 file_id、chunk_index 和 source_url 添加到 metadata
|
# 🔑 关键:在向量化前,将 file_id、chunk_index 和 source_url 添加到 metadata
|
||||||
if file_id is not None or source_url is not None:
|
if file_id is not None or source_url is not None:
|
||||||
|
|
@ -1041,6 +1204,8 @@ class VectorService:
|
||||||
# 添加文档到向量库
|
# 添加文档到向量库
|
||||||
vector_ids = vector_store.add_documents(documents=all_splits)
|
vector_ids = vector_store.add_documents(documents=all_splits)
|
||||||
logger.info(f"向量化完成,共 {len(vector_ids)} 个向量")
|
logger.info(f"向量化完成,共 {len(vector_ids)} 个向量")
|
||||||
|
if progress:
|
||||||
|
await progress.report(88, STAGE_EMBEDDING)
|
||||||
|
|
||||||
# 4. 准备返回数据
|
# 4. 准备返回数据
|
||||||
chunks = []
|
chunks = []
|
||||||
|
|
@ -1166,7 +1331,8 @@ class VectorService:
|
||||||
thread_id: str,
|
thread_id: str,
|
||||||
file_type: str = "pdf",
|
file_type: str = "pdf",
|
||||||
file_id: Optional[int] = None,
|
file_id: Optional[int] = None,
|
||||||
source_url: Optional[str] = None
|
source_url: Optional[str] = None,
|
||||||
|
progress: Optional[FileProgressReporter] = None,
|
||||||
) -> ProcessResult:
|
) -> ProcessResult:
|
||||||
"""
|
"""
|
||||||
处理聊天对话文件:加载、分割、向量化(支持多种格式,包括 URL 和图片 OCR)
|
处理聊天对话文件:加载、分割、向量化(支持多种格式,包括 URL 和图片 OCR)
|
||||||
|
|
@ -1187,9 +1353,12 @@ class VectorService:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
logger.info(f"开始处理聊天文件: {file_path}, thread_id: {thread_id}, 类型: {file_type}")
|
logger.info(f"开始处理聊天文件: {file_path}, thread_id: {thread_id}, 类型: {file_type}")
|
||||||
|
if progress:
|
||||||
|
await progress.report(35, STAGE_PARSING)
|
||||||
|
|
||||||
docs = []
|
docs = []
|
||||||
extracted_image_paths = [] # 用于保存 DOCX 中提取的图片路径
|
extracted_image_paths = [] # 用于保存 DOCX 中提取的图片路径
|
||||||
|
ocr_stage = file_type.lower() in ("png", "jpg", "jpeg", "bmp", "pdf")
|
||||||
|
|
||||||
# 特殊处理 URL(放到线程池执行)
|
# 特殊处理 URL(放到线程池执行)
|
||||||
if file_type == "url":
|
if file_type == "url":
|
||||||
|
|
@ -1213,8 +1382,10 @@ class VectorService:
|
||||||
|
|
||||||
# 特殊处理图片 OCR 和 DOCX(放到线程池执行,避免阻塞事件循环)
|
# 特殊处理图片 OCR 和 DOCX(放到线程池执行,避免阻塞事件循环)
|
||||||
if loader == "image_ocr":
|
if loader == "image_ocr":
|
||||||
logger.info("🔄 在线程池中执行图片 OCR...")
|
logger.info("🔄 处理图片:OCR → 视觉模型回退...")
|
||||||
docs = await asyncio.to_thread(self._process_image_ocr, file_path)
|
if progress:
|
||||||
|
await progress.report(38, STAGE_OCR)
|
||||||
|
docs = await self._process_image_with_vision_fallback(file_path)
|
||||||
elif loader == "docx_with_images":
|
elif loader == "docx_with_images":
|
||||||
logger.info("🔄 在线程池中处理 DOCX 文件(提取图片并 OCR)...")
|
logger.info("🔄 在线程池中处理 DOCX 文件(提取图片并 OCR)...")
|
||||||
docs, extracted_image_paths = await asyncio.to_thread(self._process_docx_with_images, file_path)
|
docs, extracted_image_paths = await asyncio.to_thread(self._process_docx_with_images, file_path)
|
||||||
|
|
@ -1225,21 +1396,20 @@ class VectorService:
|
||||||
|
|
||||||
# 特殊处理:检测 PDF 是否为图片型(扫描版)
|
# 特殊处理:检测 PDF 是否为图片型(扫描版)
|
||||||
if file_type.lower() == "pdf" and self._is_image_pdf(docs):
|
if file_type.lower() == "pdf" and self._is_image_pdf(docs):
|
||||||
logger.info("检测到图片型 PDF(扫描版),切换到 OCR 模式")
|
logger.info("检测到图片型 PDF(扫描版),OCR → 视觉模型回退")
|
||||||
if self.ocr_engine:
|
if progress:
|
||||||
logger.info("🔄 在线程池中执行 PDF OCR...")
|
await progress.report(38, STAGE_OCR)
|
||||||
docs = await asyncio.to_thread(self._process_pdf_with_ocr, file_path)
|
docs = await self._process_pdf_with_ocr_and_vision_fallback(file_path)
|
||||||
if not docs:
|
if not docs:
|
||||||
error_msg = "图片型 PDF OCR 识别失败"
|
error_msg = "图片型 PDF OCR/视觉识别均失败"
|
||||||
logger.warning(error_msg)
|
|
||||||
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
|
||||||
else:
|
|
||||||
error_msg = "检测到图片型 PDF(扫描版),但 OCR 服务不可用"
|
|
||||||
logger.warning(error_msg)
|
logger.warning(error_msg)
|
||||||
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
return ProcessResult(success=False, chunks=[], chunk_count=0, error_message=error_msg)
|
||||||
|
|
||||||
logger.info(f"文档加载完成,共 {len(docs)} 个文档片段")
|
logger.info(f"文档加载完成,共 {len(docs)} 个文档片段")
|
||||||
|
|
||||||
|
if progress and docs:
|
||||||
|
await progress.report(50, STAGE_OCR if ocr_stage else STAGE_PARSING)
|
||||||
|
|
||||||
if not docs:
|
if not docs:
|
||||||
error_msg = "未能加载到任何内容"
|
error_msg = "未能加载到任何内容"
|
||||||
logger.warning(error_msg)
|
logger.warning(error_msg)
|
||||||
|
|
@ -1256,6 +1426,8 @@ class VectorService:
|
||||||
# 分割文本
|
# 分割文本
|
||||||
all_splits = self.text_splitter.split_documents(docs)
|
all_splits = self.text_splitter.split_documents(docs)
|
||||||
logger.info(f"文本分割完成,共 {len(all_splits)} 个块")
|
logger.info(f"文本分割完成,共 {len(all_splits)} 个块")
|
||||||
|
if progress:
|
||||||
|
await progress.report(55, STAGE_SPLITTING)
|
||||||
|
|
||||||
# 检查是否有内容
|
# 检查是否有内容
|
||||||
if not all_splits:
|
if not all_splits:
|
||||||
|
|
@ -1273,6 +1445,8 @@ class VectorService:
|
||||||
# 向量化并存储(使用 thread_id 作为集合名)
|
# 向量化并存储(使用 thread_id 作为集合名)
|
||||||
collection_name = f"thread_{thread_id}"
|
collection_name = f"thread_{thread_id}"
|
||||||
vector_store = self.get_vector_store(collection_name)
|
vector_store = self.get_vector_store(collection_name)
|
||||||
|
if progress:
|
||||||
|
await progress.report(60, STAGE_EMBEDDING)
|
||||||
|
|
||||||
# 🔑 关键:在向量化前,将 file_id、chunk_index 和 source_url 添加到 metadata
|
# 🔑 关键:在向量化前,将 file_id、chunk_index 和 source_url 添加到 metadata
|
||||||
if file_id is not None or source_url is not None:
|
if file_id is not None or source_url is not None:
|
||||||
|
|
@ -1292,6 +1466,8 @@ class VectorService:
|
||||||
# 添加文档到向量库
|
# 添加文档到向量库
|
||||||
vector_ids = vector_store.add_documents(documents=all_splits)
|
vector_ids = vector_store.add_documents(documents=all_splits)
|
||||||
logger.info(f"向量化完成,共 {len(vector_ids)} 个向量")
|
logger.info(f"向量化完成,共 {len(vector_ids)} 个向量")
|
||||||
|
if progress:
|
||||||
|
await progress.report(88, STAGE_EMBEDDING)
|
||||||
|
|
||||||
# 准备返回数据
|
# 准备返回数据
|
||||||
chunks = []
|
chunks = []
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
/** 方案 B:上传 0–30%,后端处理 30–100% */
|
||||||
|
|
||||||
|
export const UPLOAD_PHASE_MAX = 30
|
||||||
|
|
||||||
|
const STAGE_LABELS = {
|
||||||
|
uploading: '上传中',
|
||||||
|
processing: '处理中',
|
||||||
|
downloading: '下载文件',
|
||||||
|
parsing: '解析文档',
|
||||||
|
ocr: '文字识别',
|
||||||
|
splitting: '文本分块',
|
||||||
|
embedding: '向量化',
|
||||||
|
summarizing: '生成摘要',
|
||||||
|
extracting: '抽取实体关系',
|
||||||
|
indexing: '建立索引',
|
||||||
|
completed: '已完成',
|
||||||
|
failed: '失败',
|
||||||
|
}
|
||||||
|
|
||||||
|
export function calcUploadPercent(loaded, total) {
|
||||||
|
if (!total) return 0
|
||||||
|
return Math.min(UPLOAD_PHASE_MAX, Math.round((loaded * UPLOAD_PHASE_MAX) / total))
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mergeProgressPercent(uploadPercent, backendPercent) {
|
||||||
|
const backend = backendPercent ?? 0
|
||||||
|
const upload = uploadPercent ?? 0
|
||||||
|
if (backend > UPLOAD_PHASE_MAX) return backend
|
||||||
|
return Math.max(upload, backend)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stageDisplayLabel(stage) {
|
||||||
|
if (!stage) return '处理中'
|
||||||
|
return STAGE_LABELS[stage] || stage
|
||||||
|
}
|
||||||
|
|
||||||
|
export function progressMessage(stage, percent) {
|
||||||
|
const label = stageDisplayLabel(stage)
|
||||||
|
return percent > 0 ? `${label}... ${percent}%` : `${label}...`
|
||||||
|
}
|
||||||
|
|
@ -350,6 +350,9 @@
|
||||||
<div class="text-muted" style="font-size: 0.75rem;">
|
<div class="text-muted" style="font-size: 0.75rem;">
|
||||||
{{ (file.file_type || 'pdf').toUpperCase() }} {{ formatFileSize(file.file_size) }}
|
{{ (file.file_type || 'pdf').toUpperCase() }} {{ formatFileSize(file.file_size) }}
|
||||||
</div>
|
</div>
|
||||||
|
<div v-if="file.status === 'processing'" class="text-warning" style="font-size: 0.7rem;">
|
||||||
|
{{ stageDisplayLabel(file.processing_stage) }} {{ file.progress_percent || 0 }}%
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<!-- 文件状态图标 -->
|
<!-- 文件状态图标 -->
|
||||||
<div class="file-status-icon">
|
<div class="file-status-icon">
|
||||||
|
|
@ -386,6 +389,9 @@
|
||||||
<div class="text-muted" style="font-size: 0.75rem;">
|
<div class="text-muted" style="font-size: 0.75rem;">
|
||||||
{{ (file.file_type || 'pdf').toUpperCase() }} {{ formatFileSize(file.file_size) }}
|
{{ (file.file_type || 'pdf').toUpperCase() }} {{ formatFileSize(file.file_size) }}
|
||||||
</div>
|
</div>
|
||||||
|
<div v-if="file.status === 'processing'" class="text-warning" style="font-size: 0.7rem;">
|
||||||
|
{{ stageDisplayLabel(file.processing_stage) }} {{ file.progress_percent || 0 }}%
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<!-- 文件状态图标 -->
|
<!-- 文件状态图标 -->
|
||||||
<div class="file-status-icon">
|
<div class="file-status-icon">
|
||||||
|
|
@ -878,6 +884,7 @@ import { marked } from 'marked'
|
||||||
import axios from 'axios'
|
import axios from 'axios'
|
||||||
import { getGreeting } from '../utils/greeting'
|
import { getGreeting } from '../utils/greeting'
|
||||||
import { apiUrl } from '../utils/apiUrl'
|
import { apiUrl } from '../utils/apiUrl'
|
||||||
|
import { stageDisplayLabel } from '../utils/fileUploadProgress'
|
||||||
import AppSidebarShell from '../components/AppSidebarShell.vue'
|
import AppSidebarShell from '../components/AppSidebarShell.vue'
|
||||||
import AppGradientHeader from '../components/AppGradientHeader.vue'
|
import AppGradientHeader from '../components/AppGradientHeader.vue'
|
||||||
|
|
||||||
|
|
@ -2163,6 +2170,8 @@ async function uploadChatFile(file) {
|
||||||
file_size: fileData.file_size,
|
file_size: fileData.file_size,
|
||||||
file_type: fileData.file_type || getFileTypeFromName(fileData.file_name),
|
file_type: fileData.file_type || getFileTypeFromName(fileData.file_name),
|
||||||
status: fileData.status,
|
status: fileData.status,
|
||||||
|
progress_percent: fileData.progress_percent ?? 30,
|
||||||
|
processing_stage: fileData.processing_stage,
|
||||||
created_at: fileData.created_at
|
created_at: fileData.created_at
|
||||||
}]
|
}]
|
||||||
})
|
})
|
||||||
|
|
@ -2244,7 +2253,7 @@ async function checkFileProcessingStatus(fileId) {
|
||||||
if (pollCount >= MAX_POLL_COUNT) {
|
if (pollCount >= MAX_POLL_COUNT) {
|
||||||
console.warn(`文件 ${fileId} 处理超时(已轮询 ${MAX_POLL_COUNT} 次)`)
|
console.warn(`文件 ${fileId} 处理超时(已轮询 ${MAX_POLL_COUNT} 次)`)
|
||||||
// 更新文件状态为超时(在内存中标记)
|
// 更新文件状态为超时(在内存中标记)
|
||||||
updateFileStatusInMessages(fileId, 'timeout')
|
updateFileInMessages(fileId, { status: 'timeout' })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2261,11 +2270,17 @@ async function checkFileProcessingStatus(fileId) {
|
||||||
|
|
||||||
if (response.data.code === 200) {
|
if (response.data.code === 200) {
|
||||||
const fileStatus = response.data.data.status
|
const fileStatus = response.data.data.status
|
||||||
|
const backendPercent = response.data.data.progress_percent
|
||||||
|
const processingStage = response.data.data.processing_stage
|
||||||
|
|
||||||
console.log(`📊 文件 ${fileId} 状态: ${fileStatus}, 轮询次数: ${pollCount + 1}`)
|
console.log(`📊 文件 ${fileId} 状态: ${fileStatus}, 进度: ${backendPercent}%, 轮询次数: ${pollCount + 1}`)
|
||||||
|
|
||||||
// 更新消息中的文件状态
|
// 更新消息中的文件状态
|
||||||
updateFileStatusInMessages(fileId, fileStatus)
|
updateFileInMessages(fileId, {
|
||||||
|
status: fileStatus,
|
||||||
|
progress_percent: backendPercent,
|
||||||
|
processing_stage: processingStage,
|
||||||
|
})
|
||||||
|
|
||||||
if (fileStatus === 'completed') {
|
if (fileStatus === 'completed') {
|
||||||
// ✅ 处理完成
|
// ✅ 处理完成
|
||||||
|
|
@ -2284,7 +2299,7 @@ async function checkFileProcessingStatus(fileId) {
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`查询文件 ${fileId} 状态失败:`, error)
|
console.error(`查询文件 ${fileId} 状态失败:`, error)
|
||||||
// 网络错误或接口错误,停止轮询
|
// 网络错误或接口错误,停止轮询
|
||||||
updateFileStatusInMessages(fileId, 'error')
|
updateFileInMessages(fileId, { status: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2293,25 +2308,27 @@ async function checkFileProcessingStatus(fileId) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// 更新消息中的文件状态(用于 UI 反馈)
|
// 更新消息中的文件状态(用于 UI 反馈)
|
||||||
function updateFileStatusInMessages(fileId, status) {
|
function updateFileInMessages(fileId, patch) {
|
||||||
// 遍历所有消息,找到包含该文件的消息并更新状态
|
|
||||||
chatStore.messages.forEach(message => {
|
chatStore.messages.forEach(message => {
|
||||||
if (message.files && Array.isArray(message.files)) {
|
if (message.files && Array.isArray(message.files)) {
|
||||||
message.files.forEach(file => {
|
message.files.forEach(file => {
|
||||||
if (file.file_id === fileId) {
|
if (file.file_id === fileId) {
|
||||||
file.status = status
|
Object.assign(file, patch)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// 也更新 chatFiles 列表中的状态
|
|
||||||
const fileInList = chatFiles.value.find(f => f.id === fileId)
|
const fileInList = chatFiles.value.find(f => f.id === fileId)
|
||||||
if (fileInList) {
|
if (fileInList) {
|
||||||
fileInList.status = status
|
Object.assign(fileInList, patch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function updateFileStatusInMessages(fileId, status) {
|
||||||
|
updateFileInMessages(fileId, { status })
|
||||||
|
}
|
||||||
|
|
||||||
// 处理删除文件
|
// 处理删除文件
|
||||||
function handleDeleteFile(fileId) {
|
function handleDeleteFile(fileId) {
|
||||||
deleteFileId.value = fileId
|
deleteFileId.value = fileId
|
||||||
|
|
|
||||||
|
|
@ -321,7 +321,7 @@
|
||||||
>
|
>
|
||||||
<span v-if="file.status === 'processing'">
|
<span v-if="file.status === 'processing'">
|
||||||
<span class="spinner-border spinner-border-sm me-1"></span>
|
<span class="spinner-border spinner-border-sm me-1"></span>
|
||||||
处理中
|
{{ file.progress_percent > 0 ? `${stageDisplayLabel(file.processing_stage)} ${file.progress_percent}%` : '处理中' }}
|
||||||
</span>
|
</span>
|
||||||
<span v-else-if="file.status === 'completed'">已完成</span>
|
<span v-else-if="file.status === 'completed'">已完成</span>
|
||||||
<span v-else-if="file.status === 'failed'">失败</span>
|
<span v-else-if="file.status === 'failed'">失败</span>
|
||||||
|
|
@ -799,6 +799,13 @@ import { useKnowledgeBaseStore } from '../stores/knowledgeBase'
|
||||||
import axios from 'axios'
|
import axios from 'axios'
|
||||||
import AppSidebarShell from '../components/AppSidebarShell.vue'
|
import AppSidebarShell from '../components/AppSidebarShell.vue'
|
||||||
import AppGradientHeader from '../components/AppGradientHeader.vue'
|
import AppGradientHeader from '../components/AppGradientHeader.vue'
|
||||||
|
import {
|
||||||
|
calcUploadPercent,
|
||||||
|
mergeProgressPercent,
|
||||||
|
progressMessage,
|
||||||
|
stageDisplayLabel,
|
||||||
|
UPLOAD_PHASE_MAX,
|
||||||
|
} from '../utils/fileUploadProgress'
|
||||||
|
|
||||||
const authStore = useAuthStore()
|
const authStore = useAuthStore()
|
||||||
const kbStore = useKnowledgeBaseStore()
|
const kbStore = useKnowledgeBaseStore()
|
||||||
|
|
@ -1150,8 +1157,8 @@ async function handleUrlSubmit() {
|
||||||
)
|
)
|
||||||
|
|
||||||
if (response.data.code === 200) {
|
if (response.data.code === 200) {
|
||||||
uploadProgress.value.message = 'URL 添加成功,正在处理中...'
|
uploadProgress.value.message = progressMessage('processing', UPLOAD_PHASE_MAX)
|
||||||
uploadProgress.value.percent = 100
|
uploadProgress.value.percent = UPLOAD_PHASE_MAX
|
||||||
|
|
||||||
// 清空输入
|
// 清空输入
|
||||||
urlInput.value = ''
|
urlInput.value = ''
|
||||||
|
|
@ -1161,12 +1168,6 @@ async function handleUrlSubmit() {
|
||||||
|
|
||||||
// 开始轮询检查处理状态
|
// 开始轮询检查处理状态
|
||||||
startStatusCheck()
|
startStatusCheck()
|
||||||
|
|
||||||
// 3秒后隐藏进度条
|
|
||||||
setTimeout(() => {
|
|
||||||
uploadProgress.value.show = false
|
|
||||||
uploadProgress.value.percent = 0
|
|
||||||
}, 3000)
|
|
||||||
} else {
|
} else {
|
||||||
throw new Error(response.data.msg || '添加失败')
|
throw new Error(response.data.msg || '添加失败')
|
||||||
}
|
}
|
||||||
|
|
@ -1206,29 +1207,23 @@ async function uploadFile(file) {
|
||||||
},
|
},
|
||||||
onUploadProgress: (progressEvent) => {
|
onUploadProgress: (progressEvent) => {
|
||||||
if (progressEvent.total) {
|
if (progressEvent.total) {
|
||||||
const percent = Math.round((progressEvent.loaded * 100) / progressEvent.total)
|
const percent = calcUploadPercent(progressEvent.loaded, progressEvent.total)
|
||||||
uploadProgress.value.percent = percent
|
uploadProgress.value.percent = percent
|
||||||
uploadProgress.value.message = `正在上传文件... ${percent}%`
|
uploadProgress.value.message = progressMessage('uploading', percent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
if (response.data.code === 200) {
|
if (response.data.code === 200) {
|
||||||
uploadProgress.value.message = '文件上传成功,正在处理中...'
|
uploadProgress.value.message = progressMessage('processing', UPLOAD_PHASE_MAX)
|
||||||
uploadProgress.value.percent = 100
|
uploadProgress.value.percent = UPLOAD_PHASE_MAX
|
||||||
|
|
||||||
// 重新加载文件列表
|
// 重新加载文件列表
|
||||||
await loadFiles(fileCurrentPage.value)
|
await loadFiles(fileCurrentPage.value)
|
||||||
|
|
||||||
// 开始轮询检查处理状态
|
// 开始轮询检查处理状态
|
||||||
startStatusCheck()
|
startStatusCheck()
|
||||||
|
|
||||||
// 3秒后隐藏进度条
|
|
||||||
setTimeout(() => {
|
|
||||||
uploadProgress.value.show = false
|
|
||||||
uploadProgress.value.percent = 0
|
|
||||||
}, 3000)
|
|
||||||
} else {
|
} else {
|
||||||
throw new Error(response.data.msg || '上传失败')
|
throw new Error(response.data.msg || '上传失败')
|
||||||
}
|
}
|
||||||
|
|
@ -1313,6 +1308,10 @@ function startStatusCheck() {
|
||||||
// 检查是否有处理中的文件
|
// 检查是否有处理中的文件
|
||||||
const processingFiles = files.value.filter(f => f.status === 'processing')
|
const processingFiles = files.value.filter(f => f.status === 'processing')
|
||||||
if (processingFiles.length === 0) {
|
if (processingFiles.length === 0) {
|
||||||
|
if (uploadProgress.value.show) {
|
||||||
|
uploadProgress.value.show = false
|
||||||
|
uploadProgress.value.percent = 0
|
||||||
|
}
|
||||||
clearInterval(statusCheckInterval)
|
clearInterval(statusCheckInterval)
|
||||||
statusCheckInterval = null
|
statusCheckInterval = null
|
||||||
return
|
return
|
||||||
|
|
@ -1340,12 +1339,22 @@ async function checkProcessingFiles(processingFiles) {
|
||||||
|
|
||||||
if (response.data.code === 200) {
|
if (response.data.code === 200) {
|
||||||
const fileStatus = response.data.data.status
|
const fileStatus = response.data.data.status
|
||||||
|
const backendPercent = response.data.data.progress_percent
|
||||||
|
const processingStage = response.data.data.processing_stage
|
||||||
|
|
||||||
// 更新文件状态
|
// 更新文件状态
|
||||||
const fileInList = files.value.find(f => f.id === file.id)
|
const fileInList = files.value.find(f => f.id === file.id)
|
||||||
if (fileInList) {
|
if (fileInList) {
|
||||||
fileInList.status = fileStatus
|
fileInList.status = fileStatus
|
||||||
fileInList.chunk_count = response.data.data.chunk_count || fileInList.chunk_count
|
fileInList.chunk_count = response.data.data.chunk_count || fileInList.chunk_count
|
||||||
|
fileInList.progress_percent = backendPercent ?? fileInList.progress_percent
|
||||||
|
fileInList.processing_stage = processingStage ?? fileInList.processing_stage
|
||||||
|
}
|
||||||
|
|
||||||
|
if (uploadProgress.value.show && file.status === 'processing') {
|
||||||
|
const merged = mergeProgressPercent(uploadProgress.value.percent, backendPercent)
|
||||||
|
uploadProgress.value.percent = merged
|
||||||
|
uploadProgress.value.message = progressMessage(processingStage, merged)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fileStatus === 'completed') {
|
if (fileStatus === 'completed') {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue