fix source aggs

sunty

2020-09-08 171d94e0f254b485ed5d09cef9a208b0f5672048

 EsApi.go

@@ -1,15 +1,15 @@
package esutil

import (
   "basic.com/pubsub/protomsg.git"
   "encoding/json"
   "errors"
   "fmt"
   "sort"
   "strconv"
   "strings"
   "sync"
   "time"

   "basic.com/pubsub/protomsg.git"
)

var logPrint = func(i ...interface{}) {
@@ -159,11 +159,30 @@

}

//根据摄像机列表和时间查询人员浏览轨迹
func GetPersonDataByCameraIdAndTime(cameraId []string, startTime string, endTime string, serverIp string, ServerPort string, indexName string) ([]map[string]interface{}, error) {
   esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
   personUrl := "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
   personBody := `{
/**************************************customer analysis util start**************************************/
/*******************sort []map util*******************/
type MapsSort struct {
   Key     string
   MapList []map[string]interface{}
}

func (m *MapsSort) Len() int {
   return len(m.MapList)
}

func (m *MapsSort) Less(i, j int) bool {
   return m.MapList[i][m.Key].(string) > m.MapList[j][m.Key].(string)
}

func (m *MapsSort) Swap(i, j int) {
   m.MapList[i], m.MapList[j] = m.MapList[j], m.MapList[i]
}

/*******************sort []map util*******************/
//根据时间范围聚合所有区域人信息，返回固定条数
func GetFaceDataByTimeAndTotal(startTime string, endTime string, total int, thresholdTime int, thresholdStayTime int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
   var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
   var requestBody = `{
    "query": {
        "bool": {
            "filter": [
@@ -171,62 +190,193 @@
                    "range": {
                        "picDate": {
                            "gte": "` + startTime + `",
                            "lte": "` + endTime + `"
                     "lte": "` + endTime + `"
                        }
                    }
                },
                {
                    "term": {
                        "targetInfo.targetType.raw": "Yolo"
                    }
                },
                {
                    "terms": {
                        "cameraId": [
                           "` + esCameraId + `"
                        ]
                    "term":{
                        "targetInfo.targetType.raw": "FaceDetect"
                    }
                }
            ]
        }
    },
    "size": 10,
    "_source": {
        "includes": [
            "cameraId",
            "cameraName",
            "cameraAddr",
            "targetInfo.targetScore",
            "picDate",
            "updateTime",
            "picMaxUrl",
            "targetInfo.belongsTargetId",
            "targetInfo.targetLocation"
        ]
    "size": 0,
    "aggs": {
        "buckets_aggs": {
            "composite": {
                "sources": [
                    {
                        "faceId": {
                            "terms": {
                                "field": "baseInfo.targetId"
                            }
                        }
                    },
                    {
                        "areaId": {
                            "terms": {
                                "field": "targetInfo.areaId"
                            }
                        }
                    }
                ],
                "size": 10000000
            },
            "aggs": {
                "top_attention_hits": {
                    "top_hits": {
                        "size": 1000000,
                        "sort": [
                            {
                                "picDate": {
                                    "order": "asc"
                                }
                            }
                        ],
                        "_source": {
                            "includes": [
                                "baseInfo.targetId",
                                "targetInfo.picSmUrl",
                        "targetInfo.areaId",
                                "picDate"
                            ]
                        }
                    }
                }
            }
        }
    }
}`
   //fmt.Println(personUrl)
   //fmt.Println(personBody)
   buf, err := EsReq("POST", personUrl, []byte(personBody))
   buf, err := EsReq("POST", requestUrl, []byte(requestBody))
   if err != nil {
      return nil, err
   }

   sources, err := Sourcelist(buf)
   source, err := FaceSourceAggregations(buf, thresholdTime, thresholdStayTime)
   if err != nil {
      return nil, err
   }
   resData,err := PerSonAnalysis(sources)
   //println(sources)
   return resData, nil

   if len(source) == 0{
      return source,nil
   }
   faceSource := make([]map[string]interface{}, 0)
   for index, info := range source {
      if int(info["stayTime"].(float64)) > thresholdStayTime {
         faceSource = append(faceSource, source[index])
      }
   }
   mapsSort := MapsSort{}
   mapsSort.Key = "endTime"
   mapsSort.MapList = faceSource
   sort.Sort(&mapsSort)
   if len(faceSource) > total {
      return mapsSort.MapList[:total], nil
   }
   return mapsSort.MapList, nil
}

//根据时间范围，摄像机列表，分组聚合人脸列表
func GetFaceDataBucketsByCameraIdAndTime(cameraId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
   esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
   var buckersUrl = "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
   var buckersBody = `{
func GetFaceDataByTimeAndId(startTime string, endTime string, id string, thresholdTime int, thresholdStayTime int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
   var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
   var requestBody = `{
    "query": {
        "bool": {
            "filter": [
                {
                    "range": {
                        "picDate": {
                            "gte": "` + startTime + `",
                      "lte": "` + endTime + `"
                        }
                    }
                },
                {
                    "term":{
                        "targetInfo.targetType.raw": "FaceDetect"
                    }
                },
            {
                    "term":{
                        "baseInfo.targetId": "` + id + `"
                    }
                }
            ]
        }
    },
    "size": 0,
    "aggs": {
        "buckets_aggs": {
            "composite": {
                "sources": [
                    {
                        "faceId": {
                            "terms": {
                                "field": "baseInfo.targetId"
                            }
                        }
                    },
                    {
                        "areaId": {
                            "terms": {
                                "field": "targetInfo.areaId"
                            }
                        }
                    }
                ],
                "size": 10000000
            },
            "aggs": {
                "top_attention_hits": {
                    "top_hits": {
                        "size": 1000000,
                        "sort": [
                            {
                                "picDate": {
                                    "order": "asc"
                                }
                            }
                        ],
                        "_source": {
                            "includes": [
                                "baseInfo.targetId",
                                "targetInfo.picSmUrl",
                        "targetInfo.areaId",
                                "picDate"
                            ]
                        }
                    }
                }
            }
        }
    }
}`
   buf, err := EsReq("POST", requestUrl, []byte(requestBody))
   if err != nil {
      return nil, err
   }
   source, err := FaceSourceAggregations(buf, thresholdTime, thresholdStayTime)
   if err != nil {
      return nil, err
   }
   if len(source) == 0{
      return source,nil
   }
   faceSource := make([]map[string]interface{}, 0)
   for index, info := range source {
      if int(info["stayTime"].(float64)) > thresholdStayTime {
         faceSource = append(faceSource, source[index])
      }
   }
   mapsSort := MapsSort{}
   mapsSort.Key = "startTime"
   mapsSort.MapList = faceSource
   sort.Sort(&mapsSort)
   return mapsSort.MapList, nil
}

func GetFaceIdDeduplication(startTime string, endTime string, serverIp string, serverPort string, indexName string) (ids []map[string]interface{}, err error) {
   var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
   var requestBody = `{
    "query": {
        "bool": {
            "filter": [
@@ -242,12 +392,235 @@
                    "term": {
                        "targetInfo.targetType.raw": "FaceDetect"
                    }
                }
            ]
        }
    },
    "size": 0,
    "aggs": {
        "buckets_aggs": {
            "composite": {
                "sources": [
                    {
                        "faceId": {
                            "terms": {
                                "field": "baseInfo.targetId"
                            }
                        }
                    }
                ],
                "size": 10000000
            },
               "aggs": {
                   "top_attention_hits": {
                       "top_hits": {
                           "size": 1,
                           "sort": [
                               {
                                   "picDate": {
                                       "order": "desc"
                                   }
                               }
                           ],
                           "_source": {
                               "includes": [
                                   "picDate"
                               ]
                           }
                       }
                      }
                  }
           }
        }
    }
}`
   //fmt.Println(requestUrl)
   //fmt.Println(requestBody)
   buf, err := EsReq("POST", requestUrl, []byte(requestBody))
   if err != nil {
      return nil, err
   }
   ids, err1 := SourceDeduplication(buf)
   if err1 != nil {
      return nil, err1
   }
   if len(ids) > 1 {
      mapsSort := MapsSort{}
      mapsSort.Key = "lastTime"
      mapsSort.MapList = ids
      sort.Sort(&mapsSort)
      return mapsSort.MapList, nil
   }
   return ids, nil
}

//统计各个区域人数
func StatisticsEveryAreaPersonsNumber(startTime string, endTime string, serverIp string, serverPort string, indexName string) ([]map[string]interface{}, error) {
   var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
   var requestBody = `{
    "query": {
        "bool": {
            "filter": [
                {
                    "range": {
                        "picDate": {
                            "gte": "` + startTime + `",
                            "lte": "` + endTime + `"
                        }
                    }
                },
                {
                    "terms": {
                        "cameraId": ["` + esCameraId + `"]
                    "term": {
                        "targetInfo.targetType.raw": "Yolo"
                    }
                }
            ]
        }
    },
    "size": 0,
    "aggs": {
        "buckets_aggs": {
            "composite": {
                "sources": [
                    {
                        "areaId": {
                            "terms": {
                                "field": "targetInfo.areaId"
                            }
                        }
                    }
                ],
                "size": 10000000
            }
        }
    }
}`
   buf, err := EsReq("POST", requestUrl, []byte(requestBody))
   if err != nil {
      return nil, err
   }
   result, err := SourceStatistics(buf)
   if err != nil {
      return nil, err
   }
   return result, nil
}

/**************************************customer analysis util end**************************************/
//根据摄像机列表和时间查询人员浏览轨迹
func GetPersonDataByCameraIdAndTime(cameraId []string, startTime string, endTime string, serverIp string, serverPort string, indexName string) (map[string]interface{}, error) {

   var filterArr []string
   if cameraId != nil && len(cameraId) > 0 {
      esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
      filterArr = append(filterArr, `{
                     "terms": {
                           "cameraId": ["`+esCameraId+`"]
                  }
            }`)
   }
   filterArr = append(filterArr, `{
                    "range": {
                        "picDate": {
                            "gte": "`+startTime+`",
                            "lte": "`+endTime+`"
                        }
                    }
                }`)
   filterArr = append(filterArr, `                {
                    "term": {
                        "targetInfo.targetType.raw": "Yolo"
                    }
                }`)
   queryStr := strings.Join(filterArr, ",")

   personUrl := "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
   personBody := `{
    "query": {
        "bool": {
            "filter": [
            ` + queryStr + `
            ]
        }
    },
    "size": 2147483647,
    "_source": {
        "includes": [
            "cameraId",
            "cameraName",
            "cameraAddr",
            "targetInfo.targetScore",
            "picDate",
            "updateTime",
            "picMaxUrl",
            "targetInfo.belongsTargetId",
            "targetInfo.targetLocation",
         "picWH"
        ]
    }
}`
   //fmt.Println(personUrl)
   //fmt.Println(personBody)
   source := make(map[string]interface{})
   queryStartTime := time.Now()
   buf, err := EsReq("POST", personUrl, []byte(personBody))
   if err != nil {
      return nil, err
   }
   queryUseTime := time.Now().Sub(queryStartTime).Seconds() * 1000
   sources, err := Sourcelist(buf)
   if err != nil {
      return nil, err
   }
   resData, err := PerSonAnalysis(sources)
   source["result"] = resData
   source["total"] = len(resData)
   source["queryUseTime"] = queryUseTime
   //println(sources)
   return source, nil

}

//根据时间范围，摄像机列表，分组聚合人脸列表,返回分组数据
func GetFaceDataBucketsByCameraIdAndTimeReturnByGrouped(cameraId []string, personId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
   var filterArr []string
   if cameraId != nil && len(cameraId) > 0 {
      esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
      filterArr = append(filterArr, `{
                     "terms": {
                           "cameraId": ["`+esCameraId+`"]
                  }
            }`)
   }
   if personId != nil && len(personId) > 0 {
      esPersonId := strings.Replace(strings.Trim(fmt.Sprint(personId), "[]"), " ", "\",\"", -1)
      filterArr = append(filterArr, `{
         "terms": {
            "baseInfo.targetId": ["`+esPersonId+`"]
            }
         }`)
   }
   filterArr = append(filterArr, `{
                    "range": {
                        "picDate": {
                            "gte": "`+startTime+`",
                            "lte": "`+endTime+`"
                        }
                    }
                }`)
   filterArr = append(filterArr, `                {
                    "term": {
                        "targetInfo.targetType.raw": "FaceDetect"
                    }
                }`)
   queryStr := strings.Join(filterArr, ",")

   var buckersUrl = "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
   var buckersBody = `{
    "query": {
        "bool": {
            "filter": [
            ` + queryStr + `
            ]
        }
    },
@@ -270,12 +643,13 @@
                            }
                        }
                    }
                ]
                ],
                "size": 10000000
            },
            "aggs":{
                "top_attention_hits":{
                    "top_hits":{
                        "size": 100,
                        "size": 1000000,
                        "sort": [
                            {
                                "picDate": {
@@ -284,7 +658,7 @@
                            }
                        ],
                        "_source":{
                            "includes":["baseInfo.targetId","cameraId","cameraName","cameraAddr","targetInfo.targetScore","targetInfo.picSmUrl","showLabels","baseInfo.tableId","baseInfo.tableName","baseInfo.bwType","baseInfo.targetName","baseInfo.compareScore","picDate","picMaxUrl"]
                            "includes":["baseInfo.targetId","cameraId","cameraName","cameraAddr","targetInfo.targetScore","targetInfo.picSmUrl","showLabels","baseInfo.tableId","baseInfo.tableName","baseInfo.bwType","baseInfo.targetName","baseInfo.compareScore","picDate","picMaxUrl","picWH"]
                        }
                    }
                }
@@ -292,33 +666,141 @@
        }
    }
}`
   //fmt.Println(buckersUrl)
   //fmt.Println(buckersBody)
   sources := make(map[string]interface{})
   queryStartTime := time.Now()
   buf, err := EsReq("POST", buckersUrl, []byte(buckersBody))
   if err != nil {
      return nil, err
   }

   sources, err := SourceAggregations(buf, thresholdTime)
   queryUseTime := time.Now().Sub(queryStartTime).Seconds() * 1000
   //fmt.Println(queryUseTime)
   tmpSources, err := SourceAggregationsReturnByGrouped(buf, thresholdTime)
   if err != nil {
      return nil, err
   }
   sources["result"] = tmpSources
   sources["total"] = len(tmpSources)
   sources["queryUseTime"] = queryUseTime
   //println(sources)
   return sources, nil
}

//根据时间范围，摄像机列表，分组聚合人脸列表
func GetFaceDataBucketsByCameraIdAndTime(cameraId []string, personId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
   var filterArr []string
   if cameraId != nil && len(cameraId) > 0 {
      esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
      filterArr = append(filterArr, `{
                     "terms": {
                           "cameraId": ["`+esCameraId+`"]
                  }
            }`)
   }
   if personId != nil && len(personId) > 0 {
      esPersonId := strings.Replace(strings.Trim(fmt.Sprint(personId), "[]"), " ", "\",\"", -1)
      filterArr = append(filterArr, `{
         "terms": {
            "baseInfo.targetId": ["`+esPersonId+`"]
            }
         }`)
   }
   filterArr = append(filterArr, `{
                    "range": {
                        "picDate": {
                            "gte": "`+startTime+`",
                            "lte": "`+endTime+`"
                        }
                    }
                }`)
   filterArr = append(filterArr, `                {
                    "term": {
                        "targetInfo.targetType.raw": "FaceDetect"
                    }
                }`)
   queryStr := strings.Join(filterArr, ",")

   var buckersUrl = "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
   var buckersBody = `{
    "query": {
        "bool": {
            "filter": [
            ` + queryStr + `
            ]
        }
    },
    "size": 0,
    "aggs": {
        "buckets_aggs": {
            "composite": {
                "sources": [
                    {
                        "baseInfo.targetId": {
                            "terms": {
                                "field": "baseInfo.targetId"
                            }
                        }
                    },
                    {
                        "cameraId": {
                            "terms": {
                                "field": "cameraId"
                            }
                        }
                    }
                ],
                "size": 10000000
            },
            "aggs":{
                "top_attention_hits":{
                    "top_hits":{
                        "size": 1000000,
                        "sort": [
                            {
                                "picDate": {
                                    "order": "asc"
                                }
                            }
                        ],
                        "_source":{
                            "includes":["baseInfo.targetId","cameraId","cameraName","cameraAddr","targetInfo.targetScore","targetInfo.picSmUrl","showLabels","baseInfo.tableId","baseInfo.tableName","baseInfo.bwType","baseInfo.targetName","baseInfo.compareScore","picDate","picMaxUrl","picWH"]
                        }
                    }
                }
            }
        }
    }
}`
   //fmt.Println(buckersUrl)
   //fmt.Println(buckersBody)
   queryStartTime := time.Now()
   buf, err := EsReq("POST", buckersUrl, []byte(buckersBody))
   if err != nil {
      return nil, err
   }
   queryUseTime := time.Now().Sub(queryStartTime).Seconds() * 1000

   sources, err := SourceAggregations(buf, thresholdTime, queryUseTime)
   if err != nil {
      return nil, err
   }
   return sources, nil
}

//根据抓拍人员id更新（picurl）图片地址
func UpdatePicUrlById(id string, picUrl string, indexName string, serverIp string, serverPort string) (err error) {
    updateTime := time.Now().Format("2006-01-02 15:04:05")
   updateTime := time.Now().Format("2006-01-02 15:04:05")
   tRes, err := AIOceaninfosbyid([]string{id}, indexName, serverIp, serverPort)
   if err != nil || len(tRes) == 0 {
      return err
   }
   picMaxUrls := tRes[0].PicMaxUrl
   sourceStr := `    	
        "lang":"painless",
        "inline": "ctx._source.picMaxUrl.add('` + picUrl + `');ctx._source.updateTime='`+updateTime+`'"
   sourceStr := `
        "source": "ctx._source.picMaxUrl.add('` + picUrl + `');ctx._source.updateTime='` + updateTime + `'"
`
   if len(picMaxUrls) >= 2 {
      sourceStr = `"source": "ctx._source.picMaxUrl[1]='` + picUrl + `';ctx._source.updateTime='`+updateTime+`'"`
      sourceStr = `"source": "ctx._source.picMaxUrl[1]='` + picUrl + `';ctx._source.updateTime='` + updateTime + `'"`
   }
   var info interface{}
   url := "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_update_by_query?refresh=true"
@@ -342,7 +824,7 @@
        }
        `
   //logPrint("url: ", url, videoUrlInfo)
   fmt.Println(url, picUrlInfo)
   //fmt.Println(url, picUrlInfo)
   buf, err := EsReq("POST", url, []byte(picUrlInfo))
   if err != nil {
      logPrint("http request videoUrlInfo info is err!")
@@ -357,7 +839,7 @@
   }
   middle, ok := out["updated"].(float64)
   if !ok {
      logPrint("first updated change error!")
      logPrint("first updated change error!", out)
      return errors.New("first updated change error!")
   }
   if middle == 1 {
@@ -1094,7 +1576,7 @@
   }
   wg.Wait()

   fmt.Println("lenth_all: ", len(dbinfos))
   //fmt.Println("lenth_all: ", len(dbinfos))

   return dbinfos, nil
}