From 16b34c939d6790fe6ebe61f8f0e85fe80ee52224 Mon Sep 17 00:00:00 2001
From: sunty <1172534965@qq.com>
Date: 星期三, 29 五月 2024 14:44:16 +0800
Subject: [PATCH] 修改准备数据流程，适应较大数据量的操作
---
db/elastic.go | 55 +++++++++++++++++++++++++++++++++----------------------
1 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/db/elastic.go b/db/elastic.go
index eeb7253..a215726 100644
--- a/db/elastic.go
+++ b/db/elastic.go
@@ -8,6 +8,7 @@
"ruleModelEngine/config"
"ruleModelEngine/util"
"strconv"
+ "strings"
)
func decodeDocumentInfos(docInfo []map[string]interface{}) ([]CaptureInfo, error) {
@@ -16,8 +17,6 @@
var captureInfo = CaptureInfo{}
captureInfo.DocumentNumber = info["key"].(string)
buckets := info["top_hits"].(map[string]interface{})["hits"].(map[string]interface{})["hits"].([]interface{})
- orgId := buckets[0].(map[string]interface{})["_source"].(map[string]interface{})["orgId"].(string)
- captureInfo.OrgId = orgId
for _, sourceInfo := range buckets {
rSourceInfo := sourceInfo.(map[string]interface{})
source := rSourceInfo["_source"].(map[string]interface{})
@@ -98,7 +97,8 @@
"top_hits": {
"_source": [
"documentNumber",
- "id"
+ "id",
+ "alarmRules.ruleId"
],
"size": 1,
"sort": [
@@ -114,8 +114,8 @@
}
}
}`
- //fmt.Println(queryDSL)
//fmt.Println(esURL)
+ //fmt.Println(queryDSL)
docNumberMap := make(map[string]string)
buf, err := esutil.EsReq("POST", esURL, []byte(queryDSL))
if err != nil {
@@ -125,14 +125,28 @@
if err != nil {
return nil, err
}
- fmt.Println("source", source)
+ //fmt.Println("source", source)
for _, info := range source {
buckets := info["top_hits"].(map[string]interface{})["hits"].(map[string]interface{})["hits"].([]interface{})
for _, sourceInfo := range buckets {
rSourceInfo := sourceInfo.(map[string]interface{})
- source := rSourceInfo["_source"].(map[string]interface{})
- documentNumber := source["documentNumber"].(string)
- id := source["id"].(string)
+ rSource := rSourceInfo["_source"].(map[string]interface{})
+ //濡傛灉璇ュぉ鏈�鍚庝竴鏉″凡缁忛璀﹁繃杩涘嚭寮傚父锛屽皢杩囨护鎺変笉鍐嶉璀�
+ alarmFlag := false
+ alarmRules := rSource["alarmRules"].([]interface{})
+ for _, alarmRule := range alarmRules {
+ ruleId := alarmRule.(map[string]interface{})["ruleId"].(string)
+ //fmt.Println("ruleId", ruleId,rSource["documentNumber"].(string),rSource["id"].(string))
+ if ruleId == "4" {
+ alarmFlag = true
+ break
+ }
+ }
+ if alarmFlag == true {
+ continue
+ }
+ documentNumber := rSource["documentNumber"].(string)
+ id := rSource["id"].(string)
docNumberMap[documentNumber] = id
}
}
@@ -252,8 +266,8 @@
logger.Info("鏈湁棰勮 id: ", id)
return true, nil
}
- fmt.Println("棰勮 id: ", id)
- fmt.Println("棰勮 AlarmRule: ", alarmRules)
+ logger.Info("棰勮 id: ", id)
+ logger.Info("棰勮 AlarmRule: ", alarmRules)
esURL := "http://" + config.Elastic.Host + ":" + config.Elastic.Port + "/" + config.Elastic.Index + "/_update_by_query"
newAlarmRulesJson, err := json.Marshal(alarmRules)
if err != nil {
@@ -300,9 +314,10 @@
return false, nil
}
-func Query1MDataByCommunityId(communityId string) ([]CaptureInfo, error) {
+func Query1MDataByCommunityId(communityId string, documentNumber []string, days int) ([]CaptureInfo, error) {
//fmt.Println(config.Elastic.DocumentSize)
//fmt.Println(config.Elastic.TopHitsSize)
+ documentNumberStr := strings.Replace(strings.Trim(fmt.Sprint(documentNumber), "[]"), " ", "\",\"", -1)
esURL := "http://" + config.Elastic.Host + ":" + config.Elastic.Port + "/" + config.Elastic.Index + "/_search"
queryDSL := `
{
@@ -312,7 +327,7 @@
{
"range": {
"picDate": {
- "gte": "now-30d/d",
+ "gte": "now-` + strconv.Itoa(days) + `d/d",
"lt": "now/d"
}
}
@@ -321,16 +336,13 @@
"term":{
"communityId":"` + communityId + `"
}
+ },
+ {
+ "terms":{
+ "documentNumber":["` + documentNumberStr + `"]
+ }
}
- ],
- "must_not": [
- {
- "term": {
- "documentNumber": ""
- }
- }
- ],
- "should": []
+ ]
}
},
"size": 0,
@@ -346,7 +358,6 @@
"_source": [
"documentNumber",
"picDate",
- "orgId",
"cameraLocation.building",
"cameraLocation.unit",
"cameraLocation.floor",
--
Gitblit v1.8.0