Merge branch 'master' of http://192.168.5.5:10010/r/rag-gateway

zhaoqingang

2024-11-22 7a14bea3aa54e889524620efea074fa583e7de41

Merge branch 'master' of http://192.168.5.5:10010/r/rag-gateway

 app/api/chat.py

@@ -261,10 +261,19 @@
                                    f"&file_type={file_type}")
                        excel_url = build_file_url(excel_name, 'excel')
                        image_url = build_file_url(image_name, 'image')
                        try:
                            SessionService(db).update_session(chat_id, message={"content": output, "role": "assistant"})
                        except Exception as e:
                            logger.error(f"Unexpected error when update_session: {e}")
                        if excel_url or data.get("e", ""):
                            try:
                                SessionService(db).update_session(chat_id,
                                                                  message={
                                                                      "content": output,
                                                                      "excel_url": excel_url,
                                                                      "image_url": image_url,
                                                                      "sql": data.get("sql", ""),
                                                                      "code": data.get("code", ""),
                                                                      "e": data.get("e", ""),
                                                                      "role": "assistant"})
                            except Exception as e:
                                logger.error(f"Unexpected error when update_session: {e}")
                        # 发送结果给客户端
                        data["type"] = "message"
                        data["message"] = output

 app/api/excel_talk.py

File was deleted

 app/api/files.py

@@ -1,5 +1,5 @@
import io
from typing import Optional
from typing import Optional, List

import requests
from fastapi import Depends, APIRouter, HTTPException, UploadFile, File, Query, Form
@@ -23,48 +23,69 @@


@router.post("/upload/{agent_id}", response_model=Response)
async def upload_file(agent_id: str,
                      file: UploadFile = File(...),
                      chat_id: str = Query(None, description="The ID of the chat"),
                      db: Session = Depends(get_db),
                      current_user: UserModel = Depends(get_current_user)
                      ):
async def upload_files(
    agent_id: str,
    file: List[UploadFile] = File(...),  # 修改这里，接受文件列表
    chat_id: str = Query(None, description="The ID of the chat"),
    db: Session = Depends(get_db),
    current_user: UserModel = Depends(get_current_user)
):
    agent = db.query(AgentModel).filter(AgentModel.id == agent_id).first()
    if not agent:
        return Response(code=404, msg="Agent not found")
    # 读取上传的文件内容
    try:
        file_content = await file.read()
    except Exception as e:
        return Response(code=400, msg=str(e))

    if agent.agent_type == AgentType.RAGFLOW:
        token = get_ragflow_token(db, current_user.id)
        ragflow_service = RagflowService(base_url=settings.fwr_base_url)
        # 查询会话是否存在，不存在先创建会话
        history = await ragflow_service.get_session_history(token, chat_id)
        if len(history) == 0:
            message = {"role": "user", "message": file.filename}
            await ragflow_service.set_session(token, agent_id, message, chat_id, True)
    # 检查 agent 类型，确定是否允许上传多个文件
    if agent.agent_type in [AgentType.RAGFLOW, AgentType.BISHENG]:
        if len(file) > 1:
            return Response(code=400, msg="这个智能体只支持传单个文件")

        ragflow_service = RagflowService(base_url=settings.fwr_base_url)
        token = get_ragflow_token(db, current_user.id)
        doc_ids = await ragflow_service.upload_and_parse(token, chat_id, file.filename, file_content)
        return Response(code=200, msg="", data={"doc_ids": doc_ids, "file_name": file.filename})

    elif agent.agent_type == AgentType.BISHENG:
        bisheng_service = BishengService(base_url=settings.sgb_base_url)
    if agent.agent_type == AgentType.RAGFLOW or agent.agent_type == AgentType.BISHENG:
        file = file[0]
        # 读取上传的文件内容
        try:
            token = get_bisheng_token(db, current_user.id)
            result = await bisheng_service.upload(token, file.filename, file_content)
            file_content = await file.read()
        except Exception as e:
            raise HTTPException(status_code=500, detail=str(e))
        result["file_name"] = file.filename
        return Response(code=200, msg="", data=result)
            return Response(code=400, msg=str(e))

        if agent.agent_type == AgentType.RAGFLOW:
            token = get_ragflow_token(db, current_user.id)
            ragflow_service = RagflowService(base_url=settings.fwr_base_url)
            # 查询会话是否存在，不存在先创建会话
            history = await ragflow_service.get_session_history(token, chat_id)
            if len(history) == 0:
                message = {"role": "user", "message": file.filename}
                await ragflow_service.set_session(token, agent_id, message, chat_id, True)

            doc_ids = await ragflow_service.upload_and_parse(token, chat_id, file.filename, file_content)
            # 对于多文件，可能需要收集所有doc_ids
            return Response(code=200, msg="", data={"doc_ids": doc_ids, "file_name": file.filename})

        elif agent.agent_type == AgentType.BISHENG:
            bisheng_service = BishengService(base_url=settings.sgb_base_url)
            try:
                token = get_bisheng_token(db, current_user.id)
                result = await bisheng_service.upload(token, file.filename, file_content)
            except Exception as e:
                raise HTTPException(status_code=500, detail=str(e))
            result["file_name"] = file.filename
            return Response(code=200, msg="", data=result)

    elif agent.agent_type == AgentType.BASIC:
        if agent_id == "basic_excel_talk":
            # 处理单个文件的情况
            file_list = file
            if len(file) == 1 and agent.agent_type != AgentType.BASIC:
                file_list = [file[0]]  # 如果只有一个文件，确保它是一个列表
            service = BasicService(base_url=settings.basic_base_url)
            result = await service.excel_talk_upload(chat_id, file.filename, file_content)
            # 遍历file_list，存到files 列表中
            files = []
            for item in file_list:
                file_content = await item.read()
                files.append(('files', (item.filename, file_content, 'application/octet-stream')))

            result = await service.excel_talk_upload(chat_id, files)
            if not result:
                return Response(code=400, msg="上传文件出错")

            return Response(code=200, msg="", data=result)

@@ -72,10 +93,7 @@
            service = BasicService(base_url=settings.basic_paper_url)
            result = await service.paper_file_upload(chat_id, file.filename, file_content)

            return Response(code=200, msg="", data=result)

    else:
        return Response(code=200, msg="Unsupported agent type")
        return Response(code=200, msg="", data=result)


@router.get("/download/", response_model=Response)

 app/service/basic.py

@@ -1,6 +1,7 @@
import json

import httpx
import requests

from Log import logger

@@ -44,12 +45,9 @@
        url = f"{self.base_url}/exceltalk/download/excel"
        return await self.download_from_url(url, params={'excel_name': file_id})

    async def excel_talk_upload(self, chat_id: str, filename: str, file_content: bytes):
    async def excel_talk_upload(self, chat_id: str, files):
        url = f"{self.base_url}/exceltalk/upload/files"
        params = {'chat_id': chat_id, 'is_col': '0'}

        # 创建 FormData 对象
        files = [('files', (filename, file_content, 'application/octet-stream'))]

        async with httpx.AsyncClient() as client:
            response = await client.post(
@@ -64,28 +62,25 @@
        params = {'chat_id': chat_id}
        data = {"query": question}
        headers = {'Content-Type': 'application/json'}
        buffer = bytearray()
        async with httpx.AsyncClient(timeout=300.0) as client:
            async with client.stream("POST", url, params=params, json=data, headers=headers) as response:
                if response.status_code == 200:
        with requests.post(url, headers=headers, json=data, params=params, timeout=60,
                           stream=True) as response:
            for line in response.iter_lines():
                if line:
                    decoded_line = line.decode("utf-8")
                    try:
                        async for chunk in response.aiter_bytes():
                            json_data = process_buffer(chunk, buffer)
                            if json_data:
                                yield json_data
                                buffer.clear()
                        if decoded_line.startswith("data:"):
                            decoded_line = decoded_line[5:]
                        answer = json.loads(decoded_line)
                        yield answer
                    except GeneratorExit as e:
                        print(e)
                        yield {"message": "内部错误", "type": "close"}
                    finally:
                        # 在所有数据接收完毕后记录日志
                        logger.info("All messages received and processed - over")
                        # 在所有数据接收完毕后返回close
                        yield {"message": "", "type": "close"}

                else:
                    yield f"Error: {response.status_code}"



    async def questions_talk(self, question, chat_id: str):
        logger.error("---------------questions_talk--------------------------")
@@ -103,27 +98,4 @@

    async def questions_talk_word_download(self, file_id: str):
        url = f"{self.base_url}/questions/download/word"
        return await self.download_from_url(url, params={'word_name': file_id})


def process_buffer(data, buffer):
    def try_parse_json(data1):
        try:
            return True, json.loads(data1)
        except json.JSONDecodeError:
            return False, None

    if data.startswith(b'data:'):
        # 删除 'data:' 头
        data = data[5:].strip()
    else:
        pass

    # 直接拼接到缓冲区尝试解析JSON
    buffer.extend(data.strip())
    success, parsed_data = try_parse_json(buffer)
    if success:
        return parsed_data
    else:
        # 解析失败，继续拼接
        return None
        return await self.download_from_url(url, params={'word_name': file_id})