From e6c5e89f09637b8d9ebca6895d781663f12646d6 Mon Sep 17 00:00:00 2001
From: zhaoqingang <zhaoqg0118@163.com>
Date: 星期二, 17 十二月 2024 16:43:55 +0800
Subject: [PATCH] dify 文档智能
---
app/task/fetch_agent.py | 195 +++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 190 insertions(+), 5 deletions(-)
diff --git a/app/task/fetch_agent.py b/app/task/fetch_agent.py
index 36b3c2e..eec88ad 100644
--- a/app/task/fetch_agent.py
+++ b/app/task/fetch_agent.py
@@ -1,19 +1,26 @@
+from pickle import PROTO
from typing import Dict, List, Tuple
-from sqlalchemy import create_engine, Column, String, Integer
+from sqlalchemy import create_engine, Column, String, Integer, Text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from app.config.config import settings
+from app.config.const import RAGFLOW, BISHENG, DIFY
+from app.models.dialog_model import DialogModel
+from app.models.user_model import UserAppModel
from app.models.agent_model import AgentModel
from app.models.base_model import SessionLocal, Base
+from app.service.v2.app_register import AppRegisterDao
# 鍒涘缓鏁版嵁搴撳紩鎿庡拰浼氳瘽宸ュ巶
engine_bisheng = create_engine(settings.sgb_db_url)
engine_ragflow = create_engine(settings.fwr_db_url)
+engine_dify = create_engine(settings.dify_database_url)
SessionBisheng = sessionmaker(autocommit=False, autoflush=False, bind=engine_bisheng)
SessionRagflow = sessionmaker(autocommit=False, autoflush=False, bind=engine_ragflow)
+SessionDify = sessionmaker(autocommit=False, autoflush=False, bind=engine_dify)
class Flow(Base):
@@ -21,6 +28,8 @@
id = Column(String(255), primary_key=True)
name = Column(String(255), nullable=False)
status = Column(Integer, nullable=False)
+ description = Column(String(255), nullable=False)
+ user_id = Column(Integer, nullable=False)
class Dialog(Base):
@@ -28,6 +37,17 @@
id = Column(String(255), primary_key=True)
name = Column(String(255), nullable=False)
status = Column(String(1), nullable=False)
+ description = Column(String(255), nullable=False)
+ tenant_id = Column(String(36), nullable=False)
+
+
+class DfApps(Base):
+ __tablename__ = 'apps'
+ id = Column(String(36), primary_key=True)
+ name = Column(String(255), nullable=False)
+ status = Column(String(16), nullable=False)
+ description = Column(Text, nullable=False)
+ tenant_id = Column(String(36), nullable=False)
# 瑙ｆ瀽鍚嶅瓧
@@ -111,9 +131,9 @@
result = db.query(AgentModel).delete()
db.commit() # 鎻愪氦浜嬪姟
initial_agents = [
- ('80ee430a-e396-48c4-a12c-7c7cdf5eda51', 1, '鎶ュ憡鐢熸垚', 'BISHENG', 'report'),
+ # ('80ee430a-e396-48c4-a12c-7c7cdf5eda51', 1, '鎶ュ憡鐢熸垚', 'DIFY', 'report'),
('basic_excel_merge', 2, '鎶ヨ〃鍚堝苟', 'BASIC', 'excelMerge'),
- ('bfd090d589d811efb3630242ac190006', 4, '鏂囨。鏅鸿兘', 'BISHENG', 'report'),
+ ('bfd090d589d811efb3630242ac190006', 4, '鏂囨。鏅鸿兘', 'DIFY', 'documentIa'),
('da3451da89d911efb9490242ac190006', 3, '鐭ヨ瘑闂瓟', 'RAGFLOW', 'knowledgeQA'),
('e96eb7a589db11ef87d20242ac190006', 5, '鏅鸿兘闂瓟', 'RAGFLOW', 'chat'),
('basic_excel_talk', 6, '鏅鸿兘鏁版嵁', 'BASIC', 'excelTalk'),
@@ -138,12 +158,177 @@
def sync_agents():
try:
- bisheng_data = get_data_from_bisheng(BISHENG_NAMES_TO_SYNC)
+ # bisheng_data = get_data_from_bisheng(BISHENG_NAMES_TO_SYNC)
ragflow_data = get_data_from_ragflow(RAGFLOW_NAMES_TO_SYNC)
- update_ids_in_local(bisheng_data)
+ # update_ids_in_local(bisheng_data)
update_ids_in_local(ragflow_data)
print("Agents synchronized successfully")
except Exception as e:
print(f"Failed to sync agents: {str(e)}")
+
+
+def update_ids_in_local(data: List[Tuple]):
+ db = SessionLocal()
+ try:
+ for row in data:
+ name = row[1]
+ new_id = row[0]
+ existing_agent = db.query(AgentModel).filter_by(name=name).first()
+ if existing_agent:
+ existing_agent.id = new_id
+ db.add(existing_agent)
+ db.commit()
+ except IntegrityError:
+ db.rollback()
+ raise
+ finally:
+ db.close()
+
+def get_rag_user_id(db, tenant_id, app_type):
+
+ user = db.query(UserAppModel).filter(UserAppModel.app_type==app_type, UserAppModel.app_id==tenant_id).first()
+ if user:
+ return user.user_id
+ return tenant_id
+
+
+def get_data_from_bisheng_v2(names: List[str]) -> List[Dict]:
+ db = SessionBisheng()
+ try:
+ if names:
+ query = db.query(Flow.id, Flow.name, Flow.description, Flow.status, Flow.user_id) \
+ .filter(Flow.name.in_(names), Flow.status==2)
+ else:
+ query = db.query(Flow.id, Flow.name, Flow.description, Flow.status, Flow.user_id).filter(Flow.status==2)
+
+ results = query.all()
+ # print(f"Executing query: {query}")
+ # 鏍煎紡鍖杋d涓篣UID
+ formatted_results = [{"id":format_uuid(row[0]), "name": row[1], "description": row[2], "status": str(row[3]-1), "user_id": str(row[4])} for row in results]
+ return formatted_results
+ finally:
+ db.close()
+
+def get_data_from_ragflow_v2(names: List[str]) -> List[Dict]:
+ db = SessionRagflow()
+ try:
+ if names:
+ query = db.query(Dialog.id, Dialog.name, Dialog.description, Dialog.status, Dialog.tenant_id) \
+ .filter( Dialog.name.in_(names))
+ else:
+ query = db.query(Dialog.id, Dialog.name, Dialog.description, Dialog.status, Dialog.tenant_id)
+
+ results = query.all()
+ formatted_results = [
+ {"id": format_uuid(row[0]), "name": row[1], "description": row[2], "status": str(row[3]),
+ "user_id": str(row[4])} for row in results]
+ return formatted_results
+ finally:
+ db.close()
+
+
+def get_data_from_dify_v2(names: List[str]) -> List[Dict]:
+ db = SessionDify()
+ try:
+ if names:
+ query = db.query(DfApps.id, DfApps.name, DfApps.description, DfApps.status, DfApps.tenant_id) \
+ .filter( DfApps.name.in_(names))
+ else:
+ query = db.query(DfApps.id, DfApps.name, DfApps.description, DfApps.status, DfApps.tenant_id)
+
+ results = query.all()
+ formatted_results = [
+ {"id": str(row[0]), "name": row[1], "description": row[2], "status": "1",
+ "user_id": str(row[4])} for row in results]
+ return formatted_results
+ finally:
+ db.close()
+
+
+
+def update_ids_in_local_v2(data: List[Dict], dialog_type:str):
+ db = SessionLocal()
+ agent_id_list = []
+ type_dict = {"1": RAGFLOW,"2": BISHENG,"4": DIFY}
+ try:
+ for row in data:
+ agent_id_list.append(row["id"])
+ existing_agent = db.query(DialogModel).filter_by(id=row["id"]).first()
+ if existing_agent:
+ existing_agent.name = row["name"]
+ existing_agent.description = row["description"]
+ existing_agent.tenant_id = get_rag_user_id(db, row["user_id"], type_dict[dialog_type])
+ else:
+ existing = DialogModel(id=row["id"], name=row["name"], description=row["description"], tenant_id=get_rag_user_id(db, row["user_id"], type_dict[dialog_type]), dialog_type=dialog_type)
+ db.add(existing)
+ db.commit()
+ for dialog in db.query(DialogModel).filter_by(dialog_type=dialog_type).all():
+ if dialog.id not in agent_id_list:
+ db.query(DialogModel).filter_by(id=dialog.id).delete()
+ db.commit()
+ except IntegrityError:
+ db.rollback()
+ raise
+ finally:
+ db.close()
+
+
+
+def get_data_from_ragflow_knowledge():
+ db = SessionRagflow()
+ try:
+
+ results = db.query(Dialog.id, Dialog.name, Dialog.description, Dialog.status, Dialog.tenant_id).all()
+ formatted_results = [
+ {"id": format_uuid(row[0]), "name": row[1], "description": row[2], "status": str(row[3]),
+ "user_id": str(row[4])} for row in results]
+ return formatted_results
+ finally:
+ db.close()
+
+def sync_agents_v2():
+ db = SessionLocal()
+
+ try:
+ app_register = AppRegisterDao(db).get_apps()
+ for app in app_register:
+ if app["id"] == RAGFLOW:
+ ragflow_data = get_data_from_ragflow_v2([])
+ update_ids_in_local_v2(ragflow_data, "1")
+ elif app["id"] == BISHENG:
+ bisheng_data = get_data_from_bisheng_v2([])
+ update_ids_in_local_v2(bisheng_data, "2")
+ elif app["id"] == DIFY:
+ dify_data = get_data_from_dify_v2([])
+ update_ids_in_local_v2(dify_data, "4")
+ print("Agents synchronized successfully")
+ except Exception as e:
+ print(f"Failed to sync agents: {str(e)}")
+
+
+
+def sync_knowledge():
+ db = SessionLocal()
+
+ try:
+ app_register = AppRegisterDao(db).get_apps()
+ for app in app_register:
+ if app["id"] == RAGFLOW:
+ ragflow_data = get_data_from_ragflow_knowledge([])
+ update_ids_in_local_v2(ragflow_data, "1")
+ # elif app["id"] == BISHENG:
+ # bisheng_data = get_data_from_bisheng_v2([])
+ # update_ids_in_local_v2(bisheng_data, "2")
+ # elif app["id"] == DIFY:
+ # dify_data = get_data_from_dify_v2([])
+ # update_ids_in_local_v2(dify_data, "4")
+ print("Agents synchronized successfully")
+ except Exception as e:
+ print(f"Failed to sync agents: {str(e)}")
+
+
+if __name__ == "__main__":
+ a = get_data_from_dify_v2([])
+ print(a)
--
Gitblit v1.8.0