修改准备数据流程，适应较大数据量的操作

sunty

2024-05-29 16b34c939d6790fe6ebe61f8f0e85fe80ee52224

 db/elastic.go

@@ -4,9 +4,11 @@
   "basic.com/pubsub/esutil.git"
   "basic.com/valib/logger.git"
   "encoding/json"
   "fmt"
   "ruleModelEngine/config"
   "ruleModelEngine/util"
   "strconv"
   "strings"
)

func decodeDocumentInfos(docInfo []map[string]interface{}) ([]CaptureInfo, error) {
@@ -15,8 +17,6 @@
      var captureInfo = CaptureInfo{}
      captureInfo.DocumentNumber = info["key"].(string)
      buckets := info["top_hits"].(map[string]interface{})["hits"].(map[string]interface{})["hits"].([]interface{})
      orgId := buckets[0].(map[string]interface{})["_source"].(map[string]interface{})["orgId"].(string)
      captureInfo.OrgId = orgId
      for _, sourceInfo := range buckets {
         rSourceInfo := sourceInfo.(map[string]interface{})
         source := rSourceInfo["_source"].(map[string]interface{})
@@ -82,13 +82,6 @@
                        }
                    }
                }
            ],
            "must_not": [
                {
                    "term": {
                        "alarmRules.ruleId": 4
                    }
                }
            ]
        }
    },
@@ -104,7 +97,8 @@
                    "top_hits": {
                        "_source": [
                            "documentNumber",
                            "id"
                            "id",
                     "alarmRules.ruleId"
                        ],
                        "size": 1,
                        "sort": [
@@ -120,8 +114,8 @@
        }
    }
}`
   //fmt.Println(queryDSL)
   //fmt.Println(esURL)
   //fmt.Println(queryDSL)
   docNumberMap := make(map[string]string)
   buf, err := esutil.EsReq("POST", esURL, []byte(queryDSL))
   if err != nil {
@@ -136,9 +130,23 @@
      buckets := info["top_hits"].(map[string]interface{})["hits"].(map[string]interface{})["hits"].([]interface{})
      for _, sourceInfo := range buckets {
         rSourceInfo := sourceInfo.(map[string]interface{})
         source := rSourceInfo["_source"].(map[string]interface{})
         documentNumber := source["documentNumber"].(string)
         id := source["id"].(string)
         rSource := rSourceInfo["_source"].(map[string]interface{})
         //如果该天最后一条已经预警过进出异常，将过滤掉不再预警
         alarmFlag := false
         alarmRules := rSource["alarmRules"].([]interface{})
         for _, alarmRule := range alarmRules {
            ruleId := alarmRule.(map[string]interface{})["ruleId"].(string)
            //fmt.Println("ruleId", ruleId,rSource["documentNumber"].(string),rSource["id"].(string))
            if ruleId == "4" {
               alarmFlag = true
               break
            }
         }
         if alarmFlag == true {
            continue
         }
         documentNumber := rSource["documentNumber"].(string)
         id := rSource["id"].(string)
         docNumberMap[documentNumber] = id
      }
   }
@@ -306,9 +314,10 @@
   return false, nil
}

func Query1MDataByCommunityId(communityId string) ([]CaptureInfo, error) {
func Query1MDataByCommunityId(communityId string, documentNumber []string, days int) ([]CaptureInfo, error) {
   //fmt.Println(config.Elastic.DocumentSize)
   //fmt.Println(config.Elastic.TopHitsSize)
   documentNumberStr := strings.Replace(strings.Trim(fmt.Sprint(documentNumber), "[]"), " ", "\",\"", -1)
   esURL := "http://" + config.Elastic.Host + ":" + config.Elastic.Port + "/" + config.Elastic.Index + "/_search"
   queryDSL := `
   {
@@ -318,7 +327,7 @@
                   {
                       "range": {
                           "picDate": {
                               "gte": "now-30d/d",
                               "gte": "now-` + strconv.Itoa(days) + `d/d",
                               "lt": "now/d"
                           }
                       }
@@ -327,16 +336,13 @@
                       "term":{
                           "communityId":"` + communityId + `"
                       }
                   },
                   {
                       "terms":{
                           "documentNumber":["` + documentNumberStr + `"]
                       }
                   }
               ],
               "must_not": [
                   {
                       "term": {
                           "documentNumber": ""
                       }
                   }
               ],
               "should": []
               ]
           }
       },
       "size": 0,
@@ -352,7 +358,6 @@
                           "_source": [
                               "documentNumber",
                               "picDate",
                        "orgId",
                               "cameraLocation.building",
                               "cameraLocation.unit",
                               "cameraLocation.floor",