From e189b1942c8130d473a1a4128c3d9ef5edfaa564 Mon Sep 17 00:00:00 2001
From: sunty <1172534965@qq.com>
Date: 星期五, 23 十月 2020 10:58:30 +0800
Subject: [PATCH] fix
---
EsApi.go | 317 +++++++++++++++++++++++++++++++++++++++++++++++++---
1 files changed, 297 insertions(+), 20 deletions(-)
diff --git a/EsApi.go b/EsApi.go
index d35c480..96348c7 100644
--- a/EsApi.go
+++ b/EsApi.go
@@ -5,6 +5,7 @@
"encoding/json"
"errors"
"fmt"
+ "sort"
"strconv"
"strings"
"sync"
@@ -159,9 +160,27 @@
}
/**************************************customer analysis util start**************************************/
+/*******************sort []map util*******************/
+type MapsSort struct {
+ Key string
+ MapList []map[string]interface{}
+}
+func (m *MapsSort) Len() int {
+ return len(m.MapList)
+}
+
+func (m *MapsSort) Less(i, j int) bool {
+ return m.MapList[i][m.Key].(string) > m.MapList[j][m.Key].(string)
+}
+
+func (m *MapsSort) Swap(i, j int) {
+ m.MapList[i], m.MapList[j] = m.MapList[j], m.MapList[i]
+}
+
+/*******************sort []map util*******************/
//鏍规嵁鏃堕棿鑼冨洿鑱氬悎鎵�鏈夊尯鍩熶汉淇℃伅锛岃繑鍥炲浐瀹氭潯鏁�
-func GetFaceDataByTimeAnd(startTime string, total int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
+func GetFaceDataByTimeAndTotal(startTime string, endTime string, total int, thresholdTime int, thresholdStayTime int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
var requestBody = `{
"query": {
@@ -170,7 +189,8 @@
{
"range": {
"picDate": {
- "gte": "` + startTime + `"
+ "gte": "` + startTime + `",
+ "lte": "` + endTime + `"
}
}
},
@@ -219,6 +239,7 @@
"includes": [
"baseInfo.targetId",
"targetInfo.picSmUrl",
+ "targetInfo.areaId",
"picDate"
]
}
@@ -232,15 +253,128 @@
if err != nil {
return nil, err
}
- source, err := Sourcelist(buf)
+ source, err := FaceSourceAggregations(buf, thresholdTime, thresholdStayTime)
if err != nil {
return nil, err
}
- fmt.Println(source)
- return resData, nil
+ if len(source) == 0 {
+ return source, nil
+ }
+ faceSource := make([]map[string]interface{}, 0)
+ for index, info := range source {
+ if int(info["stayTime"].(float64)) > thresholdStayTime {
+ faceSource = append(faceSource, source[index])
+ }
+ }
+ mapsSort := MapsSort{}
+ mapsSort.Key = "endTime"
+ mapsSort.MapList = faceSource
+ sort.Sort(&mapsSort)
+ if len(faceSource) > total {
+ return mapsSort.MapList[:total], nil
+ }
+ return mapsSort.MapList, nil
}
-func GetFaceIdDeduplication(startTime string, endTime string, serverIp string, serverPort string, indexName string) (ids []string, err error) {
+func GetFaceDataByTimeAndId(startTime string, endTime string, id string, thresholdTime int, thresholdStayTime int, serverIp string, serverPort string, indexName string) (resData []map[string]interface{}, err error) {
+ var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ var requestBody = `{
+ "query": {
+ "bool": {
+ "filter": [
+ {
+ "range": {
+ "picDate": {
+ "gte": "` + startTime + `",
+ "lte": "` + endTime + `"
+ }
+ }
+ },
+ {
+ "term":{
+ "targetInfo.targetType.raw": "FaceDetect"
+ }
+ },
+ {
+ "term":{
+ "baseInfo.targetId": "` + id + `"
+ }
+ }
+ ]
+ }
+ },
+ "size": 0,
+ "aggs": {
+ "buckets_aggs": {
+ "composite": {
+ "sources": [
+ {
+ "faceId": {
+ "terms": {
+ "field": "baseInfo.targetId"
+ }
+ }
+ },
+ {
+ "areaId": {
+ "terms": {
+ "field": "targetInfo.areaId"
+ }
+ }
+ }
+ ],
+ "size": 10000000
+ },
+ "aggs": {
+ "top_attention_hits": {
+ "top_hits": {
+ "size": 1000000,
+ "sort": [
+ {
+ "picDate": {
+ "order": "asc"
+ }
+ }
+ ],
+ "_source": {
+ "includes": [
+ "baseInfo.targetId",
+ "targetInfo.picSmUrl",
+ "targetInfo.areaId",
+ "picDate"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+}`
+ buf, err := EsReq("POST", requestUrl, []byte(requestBody))
+ if err != nil {
+ return nil, err
+ }
+ source, err := FaceSourceAggregations(buf, thresholdTime, thresholdStayTime)
+ if err != nil {
+ return nil, err
+ }
+ if len(source) == 0 {
+ return source, nil
+ }
+ faceSource := make([]map[string]interface{}, 0)
+ for index, info := range source {
+ if int(info["stayTime"].(float64)) > thresholdStayTime {
+ faceSource = append(faceSource, source[index])
+ }
+ }
+ mapsSort := MapsSort{}
+ mapsSort.Key = "startTime"
+ mapsSort.MapList = faceSource
+ sort.Sort(&mapsSort)
+ return mapsSort.MapList, nil
+}
+
+func GetFaceIdDeduplication(startTime string, endTime string, serverIp string, serverPort string, indexName string) (ids []map[string]interface{}, err error) {
var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
var requestBody = `{
"query": {
@@ -276,6 +410,87 @@
}
],
"size": 10000000
+ },
+ "aggs": {
+ "top_attention_hits": {
+ "top_hits": {
+ "size": 1,
+ "sort": [
+ {
+ "picDate": {
+ "order": "desc"
+ }
+ }
+ ],
+ "_source": {
+ "includes": [
+ "picDate"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}`
+ //fmt.Println(requestUrl)
+ //fmt.Println(requestBody)
+ buf, err := EsReq("POST", requestUrl, []byte(requestBody))
+ if err != nil {
+ return nil, err
+ }
+ ids, err1 := SourceDeduplication(buf)
+ if err1 != nil {
+ return nil, err1
+ }
+ if len(ids) > 1 {
+ mapsSort := MapsSort{}
+ mapsSort.Key = "lastTime"
+ mapsSort.MapList = ids
+ sort.Sort(&mapsSort)
+ return mapsSort.MapList, nil
+ }
+ return ids, nil
+}
+
+//缁熻鍚勪釜鍖哄煙浜烘暟
+func StatisticsEveryAreaPersonsNumber(startTime string, endTime string, serverIp string, serverPort string, indexName string) ([]map[string]interface{}, error) {
+ var requestUrl = "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
+ var requestBody = `{
+ "query": {
+ "bool": {
+ "filter": [
+ {
+ "range": {
+ "picDate": {
+ "gte": "` + startTime + `",
+ "lte": "` + endTime + `"
+ }
+ }
+ },
+ {
+ "term": {
+ "targetInfo.targetType.raw": "Yolo"
+ }
+ }
+ ]
+ }
+ },
+ "size": 0,
+ "aggs": {
+ "buckets_aggs": {
+ "composite": {
+ "sources": [
+ {
+ "areaId": {
+ "terms": {
+ "field": "targetInfo.areaId"
+ }
+ }
+ }
+ ],
+ "size": 10000000
}
}
}
@@ -284,10 +499,11 @@
if err != nil {
return nil, err
}
- fmt.Println(buf)
- //ids, err := SourceDeduplication(buf)
-
- return ids,nil
+ result, err := SourceStatistics(buf)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
}
/**************************************customer analysis util end**************************************/
@@ -580,9 +796,8 @@
return err
}
picMaxUrls := tRes[0].PicMaxUrl
- sourceStr := `
- "lang":"painless",
- "inline": "ctx._source.picMaxUrl.add('` + picUrl + `');ctx._source.updateTime='` + updateTime + `'"
+ sourceStr := `
+ "source": "ctx._source.picMaxUrl.add('` + picUrl + `');ctx._source.updateTime='` + updateTime + `'"
`
if len(picMaxUrls) >= 2 {
sourceStr = `"source": "ctx._source.picMaxUrl[1]='` + picUrl + `';ctx._source.updateTime='` + updateTime + `'"`
@@ -870,7 +1085,7 @@
if category != "all" {
filterArr = append(filterArr, ` {
"term":{
- "targetInfo.targetType":"`+category+`"
+ "targetInfo.targetType.raw":"`+category+`"
}
}`)
@@ -890,7 +1105,7 @@
"sort":[{"picDate":{"order":"desc"}}],
"_source": {"includes":[],"excludes":["*.feature"]}
}`
- logPrint(DSLJson)
+ //logPrint(DSLJson)
buf, err := EsReq("POST", url, []byte(DSLJson))
if err != nil {
return aIOceanInfo, err
@@ -980,7 +1195,6 @@
}
}
}`
- //logPrint(DSLJson)
buf, err := EsReq("POST", url, []byte(DSLJson))
if err != nil {
return total, err
@@ -1018,9 +1232,9 @@
}
},
"aggs":{
- "sdkName_status":{
+ "taskName_status":{
"terms":{
- "field":"sdkName.raw"
+ "field":"taskName.raw"
}
}
}
@@ -1039,11 +1253,11 @@
if !ok {
return nil, errors.New("first hits change error!")
}
- sdkName_status, ok := middle["sdkName_status"].(map[string]interface{})
+ sdkName_status, ok := middle["taskName_status"].(map[string]interface{})
if !ok {
return nil, errors.New("first hits change error!")
}
-
+ //fmt.Println(sdkName_status)
for _, in := range sdkName_status["buckets"].([]interface{}) {
var source = make(map[string]interface{}, 0)
tmpbuf, ok := in.(map[string]interface{})
@@ -1521,3 +1735,66 @@
}
return result, nil
}
+
+type ShardInfo struct {
+ ShardIndex string `json:"shardIndex"` //鍒嗙墖鎵�灞炵储寮曞悕绉�
+ ShardNum int `json:"shardNum"` //鍒嗙墖鍙�
+ ShardRole string `json:"shardRole"` //鍒嗙墖瑙掕壊(涓诲垎鐗囷細primary 鍓湰鍒嗙墖锛歳eplica)
+ ShardState string `json:"shardState"` //鍒嗙墖鐘舵��(鍚敤锛歋TARTED 鏈惎鐢細UNASSIGNED)
+ ShardDocs int `json:"shardDocs"` //鍒嗙墖宸蹭繚瀛樻枃妗ｆ暟
+ ShardStore string `json:"shardStore"` //鍒嗙墖褰撳墠瀛樺偍鏁版嵁澶у皬
+ ShardIp string `json:"shardIp"` //鍒嗙墖鎵�鍦ㄨ妭鐐筰p
+ ShardNode string `json:"shardNode"` //鍒嗙墖鎵�鍦ㄨ妭鐐瑰悕绉�
+}
+
+//鑾峰彇绱㈠紩鍒嗙墖淇℃伅
+func GetShardsByIndex(serverIp string, serverPort string, indexName string) ([]ShardInfo, error) {
+ url := "http://" + serverIp + ":" + serverPort + "/_cat/shards?v"
+ buf, err := EsReq("GET", url, []byte(""))
+ if err != nil {
+ return nil, err
+ }
+ var inf = []ShardInfo{}
+ res := strings.Split(string(buf), "\n")[1:]
+ for _, r := range res {
+ if r != "" {
+
+ inx := strings.Fields(r)
+ index := inx[0]
+ shard, _ := strconv.Atoi(inx[1])
+ prired := inx[2]
+ if prired == "r" {
+ prired = "replica"
+ }
+ if prired == "p" {
+ prired = "primary"
+ }
+ state := inx[3]
+ docs := 0
+ store := ""
+ ip := ""
+ node := ""
+ if state == "STARTED" {
+ docs, _ = strconv.Atoi(inx[4])
+ store = inx[5]
+ ip = inx[6]
+ node = inx[7]
+ }
+ if index == indexName {
+ inf = append(inf, ShardInfo{
+ ShardIndex: index,
+ ShardNum: shard,
+ ShardRole: prired,
+ ShardState: state,
+ ShardDocs: docs,
+ ShardStore: store,
+ ShardIp: ip,
+ ShardNode: node,
+ })
+
+ }
+ }
+
+ }
+ return inf, nil
+}
--
Gitblit v1.8.0