From 16b34c939d6790fe6ebe61f8f0e85fe80ee52224 Mon Sep 17 00:00:00 2001
From: sunty <1172534965@qq.com>
Date: 星期三, 29 五月 2024 14:44:16 +0800
Subject: [PATCH] 修改准备数据流程，适应较大数据量的操作
---
db/elastic.go | 57 +++++++++++++++++++++++++++++++--------------------------
1 files changed, 31 insertions(+), 26 deletions(-)
diff --git a/db/elastic.go b/db/elastic.go
index 5e638c9..a215726 100644
--- a/db/elastic.go
+++ b/db/elastic.go
@@ -4,9 +4,11 @@
"basic.com/pubsub/esutil.git"
"basic.com/valib/logger.git"
"encoding/json"
+ "fmt"
"ruleModelEngine/config"
"ruleModelEngine/util"
"strconv"
+ "strings"
)
func decodeDocumentInfos(docInfo []map[string]interface{}) ([]CaptureInfo, error) {
@@ -15,8 +17,6 @@
var captureInfo = CaptureInfo{}
captureInfo.DocumentNumber = info["key"].(string)
buckets := info["top_hits"].(map[string]interface{})["hits"].(map[string]interface{})["hits"].([]interface{})
- orgId := buckets[0].(map[string]interface{})["_source"].(map[string]interface{})["orgId"].(string)
- captureInfo.OrgId = orgId
for _, sourceInfo := range buckets {
rSourceInfo := sourceInfo.(map[string]interface{})
source := rSourceInfo["_source"].(map[string]interface{})
@@ -82,13 +82,6 @@
}
}
}
- ],
- "must_not": [
- {
- "term": {
- "alarmRules.ruleId": 4
- }
- }
]
}
},
@@ -104,7 +97,8 @@
"top_hits": {
"_source": [
"documentNumber",
- "id"
+ "id",
+ "alarmRules.ruleId"
],
"size": 1,
"sort": [
@@ -120,8 +114,8 @@
}
}
}`
- //fmt.Println(queryDSL)
//fmt.Println(esURL)
+ //fmt.Println(queryDSL)
docNumberMap := make(map[string]string)
buf, err := esutil.EsReq("POST", esURL, []byte(queryDSL))
if err != nil {
@@ -136,9 +130,23 @@
buckets := info["top_hits"].(map[string]interface{})["hits"].(map[string]interface{})["hits"].([]interface{})
for _, sourceInfo := range buckets {
rSourceInfo := sourceInfo.(map[string]interface{})
- source := rSourceInfo["_source"].(map[string]interface{})
- documentNumber := source["documentNumber"].(string)
- id := source["id"].(string)
+ rSource := rSourceInfo["_source"].(map[string]interface{})
+ //濡傛灉璇ュぉ鏈�鍚庝竴鏉″凡缁忛璀﹁繃杩涘嚭寮傚父锛屽皢杩囨护鎺変笉鍐嶉璀�
+ alarmFlag := false
+ alarmRules := rSource["alarmRules"].([]interface{})
+ for _, alarmRule := range alarmRules {
+ ruleId := alarmRule.(map[string]interface{})["ruleId"].(string)
+ //fmt.Println("ruleId", ruleId,rSource["documentNumber"].(string),rSource["id"].(string))
+ if ruleId == "4" {
+ alarmFlag = true
+ break
+ }
+ }
+ if alarmFlag == true {
+ continue
+ }
+ documentNumber := rSource["documentNumber"].(string)
+ id := rSource["id"].(string)
docNumberMap[documentNumber] = id
}
}
@@ -306,9 +314,10 @@
return false, nil
}
-func Query1MDataByCommunityId(communityId string) ([]CaptureInfo, error) {
+func Query1MDataByCommunityId(communityId string, documentNumber []string, days int) ([]CaptureInfo, error) {
//fmt.Println(config.Elastic.DocumentSize)
//fmt.Println(config.Elastic.TopHitsSize)
+ documentNumberStr := strings.Replace(strings.Trim(fmt.Sprint(documentNumber), "[]"), " ", "\",\"", -1)
esURL := "http://" + config.Elastic.Host + ":" + config.Elastic.Port + "/" + config.Elastic.Index + "/_search"
queryDSL := `
{
@@ -318,7 +327,7 @@
{
"range": {
"picDate": {
- "gte": "now-30d/d",
+ "gte": "now-` + strconv.Itoa(days) + `d/d",
"lt": "now/d"
}
}
@@ -327,16 +336,13 @@
"term":{
"communityId":"` + communityId + `"
}
+ },
+ {
+ "terms":{
+ "documentNumber":["` + documentNumberStr + `"]
+ }
}
- ],
- "must_not": [
- {
- "term": {
- "documentNumber": ""
- }
- }
- ],
- "should": []
+ ]
}
},
"size": 0,
@@ -352,7 +358,6 @@
"_source": [
"documentNumber",
"picDate",
- "orgId",
"cameraLocation.building",
"cameraLocation.unit",
"cameraLocation.floor",
--
Gitblit v1.8.0