From adbcd8f2d5407cc5351a9fee39f13596f5b6c2c0 Mon Sep 17 00:00:00 2001
From: sunty <1172534965@qq.com>
Date: 星期二, 01 九月 2020 14:09:41 +0800
Subject: [PATCH] add endTime GetFaceDataByTimeAndTotal
---
EsApi.go | 515 +++++++++++++++++++++++++++++++++++++++++++++++---------
1 files changed, 430 insertions(+), 85 deletions(-)
diff --git a/EsApi.go b/EsApi.go
index 875ba9a..73ee338 100644
--- a/EsApi.go
+++ b/EsApi.go
@@ -1,15 +1,15 @@
package esutil
import (
+ "basic.com/pubsub/protomsg.git"
"encoding/json"
"errors"
"fmt"
+ "sort"
"strconv"
"strings"
"sync"
"time"
-
- "basic.com/pubsub/protomsg.git"
)
var logPrint = func(i ...interface{}) {
@@ -159,11 +159,30 @@
}
-//鏍规嵁鎽勫儚鏈哄垪琛ㄥ拰鏃堕棿鏌ヨ浜哄憳娴忚杞ㄨ抗
-func GetPersonDataByCameraIdAndTime(cameraId []string, startTime string, endTime string, serverIp string, ServerPort string, indexName string) (map[string]interface{}, error) {
- esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
- personUrl := "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
- personBody := `{
+/**************************************customer analysis util start**************************************/
+/*******************sort []map util*******************/
+type MapsSort struct {
+ Key string
+ MapList []map[string]interface{}
+}
+
+func (m *MapsSort) Len() int {
+ return len(m.MapList)
+}
+
+func (m *MapsSort) Less(i, j int) bool {
+ return m.MapList[i][m.Key].(string) > m.MapList[j][m.Key].(string)
+}
+
+func (m *MapsSort) Swap(i, j int) {
+ m.MapList[i], m.MapList[j] = m.MapList[j], m.MapList[i]
+}
+
+/*******************sort []map util*******************/
+//鏍规嵁鏃堕棿鑼冨洿鑱氬悎鎵�鏈夊尯鍩熶汉淇℃伅锛岃繑鍥炲浐瀹氭潯鏁�
+func GetFaceDataByTimeAndTotal(startTime string, endTime string, total int, thresholdTime int, thresholdStayTime int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
+ var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ var requestBody = `{
"query": {
"bool": {
"filter": [
@@ -171,68 +190,183 @@
"range": {
"picDate": {
"gte": "` + startTime + `",
- "lte": "` + endTime + `"
+ "lte": "` + endTime + `"
}
}
},
{
- "term": {
- "targetInfo.targetType.raw": "Yolo"
- }
- },
- {
- "terms": {
- "cameraId": [
- "` + esCameraId + `"
- ]
+ "term":{
+ "targetInfo.targetType.raw": "FaceDetect"
}
}
]
}
},
- "size": 2147483647,
- "_source": {
- "includes": [
- "cameraId",
- "cameraName",
- "cameraAddr",
- "targetInfo.targetScore",
- "picDate",
- "updateTime",
- "picMaxUrl",
- "targetInfo.belongsTargetId",
- "targetInfo.targetLocation",
- "picWH.picW"
- ]
+ "size": 0,
+ "aggs": {
+ "buckets_aggs": {
+ "composite": {
+ "sources": [
+ {
+ "faceId": {
+ "terms": {
+ "field": "baseInfo.targetId"
+ }
+ }
+ },
+ {
+ "areaId": {
+ "terms": {
+ "field": "targetInfo.areaId"
+ }
+ }
+ }
+ ],
+ "size": 10000000
+ },
+ "aggs": {
+ "top_attention_hits": {
+ "top_hits": {
+ "size": 1000000,
+ "sort": [
+ {
+ "picDate": {
+ "order": "asc"
+ }
+ }
+ ],
+ "_source": {
+ "includes": [
+ "baseInfo.targetId",
+ "targetInfo.picSmUrl",
+ "targetInfo.areaId",
+ "picDate"
+ ]
+ }
+ }
+ }
+ }
+ }
}
}`
- //fmt.Println(personUrl)
- //fmt.Println(personBody)
- source := make(map[string]interface{})
- queryStartTime := time.Now()
- buf, err := EsReq("POST", personUrl, []byte(personBody))
+ buf, err := EsReq("POST", requestUrl, []byte(requestBody))
if err != nil {
return nil, err
}
- queryUseTime := time.Now().Sub(queryStartTime).Seconds()*1000
- sources, err := Sourcelist(buf)
+ source, err := FaceSourceAggregations(buf, thresholdTime, thresholdStayTime)
if err != nil {
return nil, err
}
- resData,err := PerSonAnalysis(sources)
- source["result"] = resData
- source["total"] = len(resData)
- source["queryUseTime"] = queryUseTime
- //println(sources)
- return source, nil
-
+ faceSource := make([]map[string]interface{}, 0)
+ for index, info := range source {
+ if int(info["stayTime"].(float64)) > thresholdStayTime {
+ faceSource = append(faceSource, source[index])
+ }
+ }
+ if len(faceSource) > total {
+ mapsSort := MapsSort{}
+ mapsSort.Key = "endTime"
+ mapsSort.MapList = faceSource
+ sort.Sort(&mapsSort)
+ return mapsSort.MapList[:total], nil
+ }
+ return faceSource, nil
}
-//鏍规嵁鏃堕棿鑼冨洿锛屾憚鍍忔満鍒楄〃锛屽垎缁勮仛鍚堜汉鑴稿垪琛�,杩斿洖鍒嗙粍鏁版嵁
-func GetFaceDataBucketsByCameraIdAndTimeReturnByGrouped(cameraId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
- esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
- var buckersUrl = "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
- var buckersBody = `{
+func GetFaceDataByTimeAndId(startTime string, endTime string, id string, thresholdTime int, thresholdStayTime int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
+ var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ var requestBody = `{
+ "query": {
+ "bool": {
+ "filter": [
+ {
+ "range": {
+ "picDate": {
+ "gte": "` + startTime + `",
+ "lte": "` + endTime + `"
+ }
+ }
+ },
+ {
+ "term":{
+ "targetInfo.targetType.raw": "FaceDetect"
+ }
+ },
+ {
+ "term":{
+ "baseInfo.targetId": "` + id + `"
+ }
+ }
+ ]
+ }
+ },
+ "size": 0,
+ "aggs": {
+ "buckets_aggs": {
+ "composite": {
+ "sources": [
+ {
+ "faceId": {
+ "terms": {
+ "field": "baseInfo.targetId"
+ }
+ }
+ },
+ {
+ "areaId": {
+ "terms": {
+ "field": "targetInfo.areaId"
+ }
+ }
+ }
+ ],
+ "size": 10000000
+ },
+ "aggs": {
+ "top_attention_hits": {
+ "top_hits": {
+ "size": 1000000,
+ "sort": [
+ {
+ "picDate": {
+ "order": "asc"
+ }
+ }
+ ],
+ "_source": {
+ "includes": [
+ "baseInfo.targetId",
+ "targetInfo.picSmUrl",
+ "targetInfo.areaId",
+ "picDate"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+}`
+ buf, err := EsReq("POST", requestUrl, []byte(requestBody))
+ if err != nil {
+ return nil, err
+ }
+ source, err := FaceSourceAggregations(buf, thresholdTime, thresholdStayTime)
+ if err != nil {
+ return nil, err
+ }
+ faceSource := make([]map[string]interface{}, 0)
+ for index, info := range source {
+ if int(info["stayTime"].(float64)) > thresholdStayTime {
+ faceSource = append(faceSource, source[index])
+ }
+ }
+ return faceSource, nil
+}
+
+func GetFaceIdDeduplication(startTime string, endTime string, serverIp string, serverPort string, indexName string) (ids []string, err error) {
+ var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ var requestBody = `{
"query": {
"bool": {
"filter": [
@@ -248,12 +382,208 @@
"term": {
"targetInfo.targetType.raw": "FaceDetect"
}
+ }
+ ]
+ }
+ },
+ "size": 0,
+ "aggs": {
+ "buckets_aggs": {
+ "composite": {
+ "sources": [
+ {
+ "faceId": {
+ "terms": {
+ "field": "baseInfo.targetId"
+ }
+ }
+ }
+ ],
+ "size": 10000000
+ }
+ }
+ }
+}`
+ //fmt.Println(requestUrl)
+ //fmt.Println(requestBody)
+ buf, err := EsReq("POST", requestUrl, []byte(requestBody))
+ if err != nil {
+ return nil, err
+ }
+ ids, err1 := SourceDeduplication(buf)
+ if err1 != nil {
+ return nil, err1
+ }
+ return ids, nil
+}
+
+//缁熻鍚勪釜鍖哄煙浜烘暟
+func StatisticsEveryAreaPersonsNumber(startTime string, endTime string, serverIp string, serverPort string, indexName string) ([]map[string]interface{}, error) {
+ var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ var requestBody = `{
+ "query": {
+ "bool": {
+ "filter": [
+ {
+ "range": {
+ "picDate": {
+ "gte": "` + startTime + `",
+ "lte": "` + endTime + `"
+ }
+ }
},
{
- "terms": {
- "cameraId": ["` + esCameraId + `"]
+ "term": {
+ "targetInfo.targetType.raw": "Yolo"
}
}
+ ]
+ }
+ },
+ "size": 0,
+ "aggs": {
+ "buckets_aggs": {
+ "composite": {
+ "sources": [
+ {
+ "areaId": {
+ "terms": {
+ "field": "targetInfo.areaId"
+ }
+ }
+ }
+ ],
+ "size": 10000000
+ }
+ }
+ }
+}`
+ buf, err := EsReq("POST", requestUrl, []byte(requestBody))
+ if err != nil {
+ return nil, err
+ }
+ result, err := SourceStatistics(buf)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+/**************************************customer analysis util end**************************************/
+//鏍规嵁鎽勫儚鏈哄垪琛ㄥ拰鏃堕棿鏌ヨ浜哄憳娴忚杞ㄨ抗
+func GetPersonDataByCameraIdAndTime(cameraId []string, startTime string, endTime string, serverIp string, serverPort string, indexName string) (map[string]interface{}, error) {
+
+ var filterArr []string
+ if cameraId != nil && len(cameraId) > 0 {
+ esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
+ filterArr = append(filterArr, `{
+ "terms": {
+ "cameraId": ["`+esCameraId+`"]
+ }
+ }`)
+ }
+ filterArr = append(filterArr, `{
+ "range": {
+ "picDate": {
+ "gte": "`+startTime+`",
+ "lte": "`+endTime+`"
+ }
+ }
+ }`)
+ filterArr = append(filterArr, ` {
+ "term": {
+ "targetInfo.targetType.raw": "Yolo"
+ }
+ }`)
+ queryStr := strings.Join(filterArr, ",")
+
+ personUrl := "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ personBody := `{
+ "query": {
+ "bool": {
+ "filter": [
+ ` + queryStr + `
+ ]
+ }
+ },
+ "size": 2147483647,
+ "_source": {
+ "includes": [
+ "cameraId",
+ "cameraName",
+ "cameraAddr",
+ "targetInfo.targetScore",
+ "picDate",
+ "updateTime",
+ "picMaxUrl",
+ "targetInfo.belongsTargetId",
+ "targetInfo.targetLocation",
+ "picWH"
+ ]
+ }
+}`
+ //fmt.Println(personUrl)
+ //fmt.Println(personBody)
+ source := make(map[string]interface{})
+ queryStartTime := time.Now()
+ buf, err := EsReq("POST", personUrl, []byte(personBody))
+ if err != nil {
+ return nil, err
+ }
+ queryUseTime := time.Now().Sub(queryStartTime).Seconds() * 1000
+ sources, err := Sourcelist(buf)
+ if err != nil {
+ return nil, err
+ }
+ resData, err := PerSonAnalysis(sources)
+ source["result"] = resData
+ source["total"] = len(resData)
+ source["queryUseTime"] = queryUseTime
+ //println(sources)
+ return source, nil
+
+}
+
+//鏍规嵁鏃堕棿鑼冨洿锛屾憚鍍忔満鍒楄〃锛屽垎缁勮仛鍚堜汉鑴稿垪琛�,杩斿洖鍒嗙粍鏁版嵁
+func GetFaceDataBucketsByCameraIdAndTimeReturnByGrouped(cameraId []string, personId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
+ var filterArr []string
+ if cameraId != nil && len(cameraId) > 0 {
+ esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
+ filterArr = append(filterArr, `{
+ "terms": {
+ "cameraId": ["`+esCameraId+`"]
+ }
+ }`)
+ }
+ if personId != nil && len(personId) > 0 {
+ esPersonId := strings.Replace(strings.Trim(fmt.Sprint(personId), "[]"), " ", "\",\"", -1)
+ filterArr = append(filterArr, `{
+ "terms": {
+ "baseInfo.targetId": ["`+esPersonId+`"]
+ }
+ }`)
+ }
+ filterArr = append(filterArr, `{
+ "range": {
+ "picDate": {
+ "gte": "`+startTime+`",
+ "lte": "`+endTime+`"
+ }
+ }
+ }`)
+ filterArr = append(filterArr, ` {
+ "term": {
+ "targetInfo.targetType.raw": "FaceDetect"
+ }
+ }`)
+ queryStr := strings.Join(filterArr, ",")
+
+ var buckersUrl = "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
+ var buckersBody = `{
+ "query": {
+ "bool": {
+ "filter": [
+ ` + queryStr + `
]
}
},
@@ -277,7 +607,7 @@
}
}
],
- "size": 100000000
+ "size": 10000000
},
"aggs":{
"top_attention_hits":{
@@ -299,13 +629,15 @@
}
}
}`
+ //fmt.Println(buckersUrl)
+ //fmt.Println(buckersBody)
sources := make(map[string]interface{})
queryStartTime := time.Now()
buf, err := EsReq("POST", buckersUrl, []byte(buckersBody))
if err != nil {
return nil, err
}
- queryUseTime := time.Now().Sub(queryStartTime).Seconds()*1000
+ queryUseTime := time.Now().Sub(queryStartTime).Seconds() * 1000
//fmt.Println(queryUseTime)
tmpSources, err := SourceAggregationsReturnByGrouped(buf, thresholdTime)
if err != nil {
@@ -319,31 +651,45 @@
}
//鏍规嵁鏃堕棿鑼冨洿锛屾憚鍍忔満鍒楄〃锛屽垎缁勮仛鍚堜汉鑴稿垪琛�
-func GetFaceDataBucketsByCameraIdAndTime(cameraId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
- esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
+func GetFaceDataBucketsByCameraIdAndTime(cameraId []string, personId []string, startTime string, endTime string, thresholdTime float64, serverIp string, ServerPort string, indexName string) (buckersDate map[string]interface{}, err error) {
+ var filterArr []string
+ if cameraId != nil && len(cameraId) > 0 {
+ esCameraId := strings.Replace(strings.Trim(fmt.Sprint(cameraId), "[]"), " ", "\",\"", -1)
+ filterArr = append(filterArr, `{
+ "terms": {
+ "cameraId": ["`+esCameraId+`"]
+ }
+ }`)
+ }
+ if personId != nil && len(personId) > 0 {
+ esPersonId := strings.Replace(strings.Trim(fmt.Sprint(personId), "[]"), " ", "\",\"", -1)
+ filterArr = append(filterArr, `{
+ "terms": {
+ "baseInfo.targetId": ["`+esPersonId+`"]
+ }
+ }`)
+ }
+ filterArr = append(filterArr, `{
+ "range": {
+ "picDate": {
+ "gte": "`+startTime+`",
+ "lte": "`+endTime+`"
+ }
+ }
+ }`)
+ filterArr = append(filterArr, ` {
+ "term": {
+ "targetInfo.targetType.raw": "FaceDetect"
+ }
+ }`)
+ queryStr := strings.Join(filterArr, ",")
+
var buckersUrl = "http://" + serverIp + ":" + ServerPort + "/" + indexName + "/_search"
var buckersBody = `{
"query": {
"bool": {
"filter": [
- {
- "range": {
- "picDate": {
- "gte": "` + startTime + `",
- "lte": "` + endTime + `"
- }
- }
- },
- {
- "term": {
- "targetInfo.targetType.raw": "FaceDetect"
- }
- },
- {
- "terms": {
- "cameraId": ["` + esCameraId + `"]
- }
- }
+ ` + queryStr + `
]
}
},
@@ -367,7 +713,7 @@
}
}
],
- "size": 100000000
+ "size": 10000000
},
"aggs":{
"top_attention_hits":{
@@ -396,9 +742,9 @@
if err != nil {
return nil, err
}
- queryUseTime := time.Now().Sub(queryStartTime).Seconds()*1000
+ queryUseTime := time.Now().Sub(queryStartTime).Seconds() * 1000
- sources, err := SourceAggregations(buf, thresholdTime,queryUseTime)
+ sources, err := SourceAggregations(buf, thresholdTime, queryUseTime)
if err != nil {
return nil, err
}
@@ -407,18 +753,17 @@
//鏍规嵁鎶撴媿浜哄憳id鏇存柊锛坧icurl锛夊浘鐗囧湴鍧�
func UpdatePicUrlById(id string, picUrl string, indexName string, serverIp string, serverPort string) (err error) {
- updateTime := time.Now().Format("2006-01-02 15:04:05")
+ updateTime := time.Now().Format("2006-01-02 15:04:05")
tRes, err := AIOceaninfosbyid([]string{id}, indexName, serverIp, serverPort)
if err != nil || len(tRes) == 0 {
return err
}
picMaxUrls := tRes[0].PicMaxUrl
- sourceStr := `
- "lang":"painless",
- "inline": "ctx._source.picMaxUrl.add('` + picUrl + `');ctx._source.updateTime='`+updateTime+`'"
+ sourceStr := `
+ "source": "ctx._source.picMaxUrl.add('` + picUrl + `');ctx._source.updateTime='` + updateTime + `'"
`
if len(picMaxUrls) >= 2 {
- sourceStr = `"source": "ctx._source.picMaxUrl[1]='` + picUrl + `';ctx._source.updateTime='`+updateTime+`'"`
+ sourceStr = `"source": "ctx._source.picMaxUrl[1]='` + picUrl + `';ctx._source.updateTime='` + updateTime + `'"`
}
var info interface{}
url := "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_update_by_query?refresh=true"
@@ -442,7 +787,7 @@
}
`
//logPrint("url: ", url, videoUrlInfo)
- fmt.Println(url, picUrlInfo)
+ //fmt.Println(url, picUrlInfo)
buf, err := EsReq("POST", url, []byte(picUrlInfo))
if err != nil {
logPrint("http request videoUrlInfo info is err!")
@@ -457,7 +802,7 @@
}
middle, ok := out["updated"].(float64)
if !ok {
- logPrint("first updated change error!")
+ logPrint("first updated change error!", out)
return errors.New("first updated change error!")
}
if middle == 1 {
@@ -1194,7 +1539,7 @@
}
wg.Wait()
- fmt.Println("lenth_all: ", len(dbinfos))
+ //fmt.Println("lenth_all: ", len(dbinfos))
return dbinfos, nil
}
--
Gitblit v1.8.0