From dabd20efe434704c96044dbd6c084e8266d1ef3f Mon Sep 17 00:00:00 2001
From: xqz <837938965@qq.com>
Date: 星期二, 08 七月 2025 18:34:40 +0800
Subject: [PATCH] Merge remote-tracking branch 'origin/develop' into develop
---
qwen_ollama.py | 224 ++++++++++++++++++++++++++++++++-----------------------
1 files changed, 131 insertions(+), 93 deletions(-)
diff --git a/qwen_ollama.py b/qwen_ollama.py
index 92a197a..8fa71e7 100644
--- a/qwen_ollama.py
+++ b/qwen_ollama.py
@@ -10,7 +10,7 @@
import logging
from PIL import Image
from logging.handlers import RotatingFileHandler
-
+from transformers import AutoModelForVision2Seq, AutoProcessor
class detect_tasks():
def __init__(self):
@@ -19,6 +19,20 @@
# 鍒濆鍖朚ilvus闆嗗悎
self.collection = None
+
+ self.model = AutoModelForVision2Seq.from_pretrained(
+ "./Qwen2.5-VL-3B-Instruct-GPTQ-Int4",
+ device_map="auto"
+ )
+
+ self.processor = AutoProcessor.from_pretrained(
+ "./Qwen2.5-VL-3B-Instruct-GPTQ-Int4",
+ trust_remote_code=True,
+ use_fast=True # 寮哄埗鍚敤蹇�熷鐞嗗櫒
+ )
+
+
+
def remove_duplicate_lines(self, text):
@@ -67,43 +81,50 @@
self.logger.addHandler(handler)
def image_desc(self, image_path, ollama_url, ollama_mode="qwen2.5vl:3b"):
- try:
- image_des = None
- # 鍥剧墖棰勫鐞�
- img = Image.open(image_path)
- buffered = io.BytesIO()
- img.save(buffered, format="JPEG", quality=85)
- img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
- # 璋冪敤API
- response = requests.post(
- f"{ollama_url}/api/generate",
- json={
- "model": ollama_mode,
- "prompt": "璇锋寜浠ヤ笅瑕佹眰鎻忚堪鍥剧墖锛歕n1. 鍒楀嚭涓昏鐗╀綋\n2.涓嶈繘琛屾帹鐞嗗拰think\n杩斿洖灏忎簬2000瀛楃殑鏁存鎻忚堪,鎻忚堪涓殑鐗╀綋淇℃伅涓嶅姞鏁板瓧搴忓彿",
- "images": [img_str],
- "options": {
- "num_gpu_layers": 35
- },
- "stream": False # 闈炴祦寮�
- },
- headers={"Content-Type": "application/json"}
- )
- result = response.json()
- if result and result["response"]:
- image_des = (result["response"]).replace('\n', '')
- if len(image_des) > 4 and image_des.startswith("杩欏紶鍥剧墖"):
- image_des = image_des[4:]
+ image = Image.open(image_path).convert("RGB") # 鏇挎崲涓烘偍鐨勫浘鐗囪矾寰�
+ image = image.resize((600, 600), Image.Resampling.LANCZOS) # 楂樿川閲忕缉鏀�
- image_des = self.remove_duplicate_lines(image_des)
- image_des = self.remove_duplicate_lines_d(image_des)
- image_des = self.remove_duplicate_lines_n(image_des)
- # self.logger.info(image_des)
- else:
- self.logger.info(f"{self._thread_name}绾跨▼锛氭墽琛屽浘鐗囨弿杩版椂鍑洪敊:{image_path, result, response}")
- return image_des
- except Exception as e:
- self.logger.info(f"{self._thread_name}绾跨▼锛氭墽琛屽浘鐗囨弿杩版椂鍑洪敊:{image_path, e}")
- return None
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ },
+ {"type": "text", "text": "璇疯缁嗘弿杩板浘鐗囦腑鐨勭洰鏍囦俊鎭強鐗瑰緛銆傝繑鍥炴牸寮忎负鏁存鏂囧瓧鎻忚堪"},
+ ],
+ }
+ ]
+
+ # Preparation for inference
+ text = self.processor.apply_chat_template(
+ messages, add_generation_prompt=True
+ )
+
+ inputs = self.processor(
+ text=[text],
+ images=[image],
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to("cuda")
+
+ with torch.no_grad():
+ outputs = self.model.generate(**inputs,
+ max_new_tokens=300,
+ do_sample=True,
+ temperature=0.7,
+ renormalize_logits=True
+ )
+
+ generated_ids = outputs[:, len(inputs.input_ids[0]):]
+ image_text = self.processor.batch_decode(
+ generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True
+ )
+
+ image_des = (image_text[0]).strip()
+
+ return image_des
def get_rule(self, ragurl):
try:
@@ -193,6 +214,8 @@
)
# 浠巎son鎻愬彇data瀛楁鍐呭
ret = response.json()["data"]
+ #result = response.json()
+ #ret = result.get("data") or result.get("message", {}).get("content", "")
# 绉婚櫎<think>鏍囩鍜屽唴瀹�
ret = re.sub(r'<think>.*?</think>', '', ret, flags=re.DOTALL)
# 瀛楃涓叉竻鐞�,绉婚櫎绌烘牸,鍒惰〃绗�,鎹㈣绗�,鏄熷彿
@@ -202,12 +225,14 @@
#澶勭悊寤鸿
def image_rule_chat_suggestion(self, rule_text, ollama_url, ollama_mode="qwen2.5vl:3b"):
self.logger.info("----------------------------------------------------------------")
+ # 璇锋眰鍐呭
content = (
f"璇锋牴鎹繚瑙勫唴瀹筟{rule_text}]\n杩涜杩斿洖澶勭悊杩濊寤鸿锛屼笉杩涜鎺ㄧ悊鍜宼hink銆傝繑鍥炵簿鍑嗕俊鎭�")
response = requests.post(
+ # ollama鍦板潃
url=f"{ollama_url}/chat",
json={
-
+ # 鎸囧畾妯″瀷
"llm_name": "qwen3:8b",
"messages": [
{"role": "user", "content": content}
@@ -215,118 +240,131 @@
"stream": False # 鍏抽棴娴佸紡杈撳嚭
}
)
+ # 浠巎son鎻愬彇data瀛楁鍐呭
ret = response.json()["data"]
+ #result = response.json()
+ #ret = result.get("data") or result.get("message", {}).get("content", "")
+ # 绉婚櫎<think>鏍囩鍜屽唴瀹�
ret = re.sub(r'<think>.*?</think>', '', ret, flags=re.DOTALL)
+ # 瀛楃涓叉竻鐞�,绉婚櫎绌烘牸,鍒惰〃绗�,鎹㈣绗�,鏄熷彿
ret = ret.replace(" ", "").replace("\t", "").replace("\n", "").replace("**","")
print(ret)
return ret
- # print(response.json())
- # ret = response.json()["detail"]
-
+ # RAG鏈嶅姟鍙戦�佽姹�,鑾峰彇鐭ヨ瘑搴撳唴瀹�
def get_filedata(self, searchtext, ragurl):
search_data = {
+ # 鐭ヨ瘑搴撻泦鍚�
"collection_name": "smart_knowledge",
+ # 鏌ヨ鏂囨湰
"query_text": searchtext,
+ # 鎼滅储妯″紡
"search_mode": "hybrid",
+ # 鏈�澶氳繑鍥炵粨鏋�
"limit": 100,
+ # 璋冨瘑鍚戦噺鎼滅储鏉冮噸
"weight_dense": 0.7,
+ # 绋�鐤忓悜閲忔悳绱㈡潈閲�
"weight_sparse": 0.3,
+ # 绌哄瓧绗︿覆
"filter_expr": "",
+ # 鍙繑鍥� text 瀛楁
"output_fields": ["text"]
}
+ # 鍚� ragurl + "/search" 绔偣鍙戦�丳OST璇锋眰
response = requests.post(ragurl + "/search", json=search_data)
+ # 浠庡搷搴斾腑鑾峰彇'results'瀛楁
results = response.json().get('results')
+ # 鍒濆鍖� text
text = ""
+ # 閬嶅巻鎵�鏈夌粨鏋滆鍒�(rule)锛屽皢姣忔潯瑙勫垯鐨�'entity'涓殑'text'瀛楁鍙栧嚭.
for rule in results:
text = text + rule['entity'].get('text') + ";\n"
return text
-
+ # 鍚慠AG绯荤粺鎻掑叆json鏍煎紡鏁版嵁
async def insert_json_data(self, ragurl, data):
try:
+ # data 瑕佹彃鍏ョ殑鏁版嵁
data = {'collection_name': "smartrag", "data": data, "description": ""}
+ # 鏈嶅姟鐨勫熀鍦板潃
requests.post(ragurl + "/insert_json_data", json=data, timeout=(0.3, 0.3))
# self.logger.info(f"璋冪敤褰曞儚鏈嶅姟:{ragurl, data}")
except Exception as e:
# self.logger.info(f"{self._thread_name}绾跨▼锛氳皟鐢ㄥ綍鍍忔椂鍑洪敊:鍦板潃锛歿ragurl}锛歿e}")
return
- def tark_do(self, image_id, ollamaurl, ragurl, ollamamode, ragmode):
+ def tark_do(self, res, ollamaurl, ragurl, ollamamode, ragmode):
try:
# 1. 浠庨泦鍚圓鑾峰彇鍚戦噺鍜屽厓鏁版嵁
is_waning = 0
- res_a = self.collection.query(
- expr=f"id == 458942504686042840",
- output_fields=["id", "zh_desc_class", "text_vector", "bounding_box", "video_point_name", "task_id",
- "task_name", "event_level_id", "event_level_name",
- "video_point_id", "detect_num", "is_waning", "waning_value", "rule_id", "detect_id",
- "detect_time", "image_path", "image_desc_path", "video_path","risk_description","suggestion","knowledge_id"],
- consistency_level="Strong"
- )
+ is_desc = 2
- if not res_a:
- self.logger.info(f"{self._thread_name}绾跨▼锛氭湭鎵惧埌ID涓� {image_id} 鐨勮褰�")
- return 0
-
-
- image_des = self.image_desc(f"/opt/smart/img/1748763385.245874.jpg", ollamaurl, ollamamode)
+ # 鐢熸垚鍥剧墖鎻忚堪
+ image_des = self.image_desc(res['image_desc_path'], ollamaurl, ollamamode)
risk_description = ""
suggestion = ""
+ # 鍥剧墖鎻忚堪鐢熸垚鎴愬姛
if image_des:
- is_waning = self.image_rule_chat(image_des, res_a[0]['waning_value'], ollamaurl, ollamamode)
+ is_desc = 2
+ # 璋冪敤瑙勫垯鍖归厤鏂规硶,鍒ゆ柇鏄惁棰勮
+ is_waning = self.image_rule_chat(image_des, res['waning_value'], ollamaurl, ollamamode)
+ # 濡傛灉棰勮,鍒欑敓鎴愰殣鎮ｆ弿杩板拰澶勭悊寤鸿
+ if is_waning == 1:
+ # 鑾峰彇瑙勭珷鍒跺害鏁版嵁
+ filedata = self.get_filedata(res['waning_value'], ragurl)
+ # 鐢熸垚闅愭偅鎻忚堪
+ risk_description = self.image_rule_chat_with_detail(filedata,res['waning_value'], ragurl, ragmode)
+ # 鐢熸垚澶勭悊寤鸿
+ suggestion = self.image_rule_chat_suggestion(res['waning_value'], ragurl, ragmode)
- if is_waning == 0:
-
- filedata = self.get_filedata(res_a[0]['waning_value'], ragurl)
- risk_description = self.image_rule_chat_with_detail(filedata,res_a[0]['waning_value'], ragurl, ragmode)
- suggestion = self.image_rule_chat_suggestion(res_a[0]['waning_value'], ragurl, ragmode)
+ else:
+ is_desc = 3
# 鏁版嵁缁�
data = {
- "id": image_id,
- "event_level_id": res_a[0]['event_level_id'], # event_level_id
- "event_level_name": res_a[0]['event_level_name'], # event_level_id
- "rule_id": res_a[0]["rule_id"],
- "video_point_id": res_a[0]['video_point_id'], # video_point_id
- "video_point_name": res_a[0]['video_point_name'],
+ "id": res['id'],
+ "event_level_id": res['event_level_id'], # event_level_id
+ "event_level_name": res['event_level_name'], # event_level_id
+ "rule_id": res["rule_id"],
+ "video_point_id": res['video_point_id'], # video_point_id
+ "video_point_name": res['video_point_name'],
"is_waning": is_waning,
+ "is_desc": is_desc,
"zh_desc_class": image_des, # text_vector
- "bounding_box": res_a[0]['bounding_box'], # bounding_box
- "task_id": res_a[0]['task_id'], # task_id
- "task_name": res_a[0]['task_name'], # task_id
- "detect_id": res_a[0]['detect_id'], # detect_id
- "detect_time": res_a[0]['detect_time'], # detect_time
- "detect_num": res_a[0]['detect_num'],
- "waning_value": res_a[0]['waning_value'],
- "image_path": res_a[0]['image_path'], # image_path
- "image_desc_path": res_a[0]['image_desc_path'], # image_desc_path
- "video_path": res_a[0]['video_path'],
- "text_vector": res_a[0]['text_vector'],
+ "bounding_box": res['bounding_box'], # bounding_box
+ "task_id": res['task_id'], # task_id
+ "task_name": res['task_name'], # task_id
+ "detect_id": res['detect_id'], # detect_id
+ "detect_time": res['detect_time'], # detect_time
+ "detect_num": res['detect_num'],
+ "waning_value": res['waning_value'],
+ "image_path": res['image_path'], # image_path
+ "image_desc_path": res['image_desc_path'], # image_desc_path
+ "video_path": res['video_path'],
+ "text_vector": res['text_vector'],
"risk_description": risk_description,
"suggestion": suggestion,
- "knowledge_id": res_a[0]['knowledge_id']
+ "knowledge_id": res['knowledge_id']
}
-
-
# 淇濆瓨鍒癿ilvus
image_id = self.collection.upsert(data).primary_keys
+ logging.info(image_id)
data = {
"id": str(image_id[0]),
- "video_point_id": res_a[0]['video_point_id'],
- "video_path": res_a[0]["video_point_name"],
+ "video_point_id": res['video_point_id'],
+ "video_path": res["video_point_name"],
"zh_desc_class": image_des,
- "detect_time": res_a[0]['detect_time'],
- "image_path": f"{res_a[0]['image_path']}",
- "task_name": res_a[0]["task_name"],
- "event_level_name": res_a[0]["event_level_name"],
- "rtsp_address": f"{res_a[0]['video_path']}"
+ "detect_time": res['detect_time'],
+ "image_path": f"{res['image_path']}",
+ "task_name": res["task_name"],
+ "event_level_name": res["event_level_name"],
+ "rtsp_address": f"{res['video_path']}"
}
# 璋冪敤rag
asyncio.run(self.insert_json_data(ragurl, data))
return image_id
except Exception as e:
- self.logger.info(f"{self._thread_name}绾跨▼锛氭墽琛屾ā鍨嬭В鏋愭椂鍑洪敊:浠诲姟锛歿image_id} :{e}")
- return 0
-
+ self.logger.info(f"{self._thread_name}绾跨▼锛氭墽琛屾ā鍨嬭В鏋愭椂鍑洪敊:浠诲姟锛歿res['id']} :{e}")
+ return 0
\ No newline at end of file
--
Gitblit v1.8.0