From f6ca7bb43270474fa876ff6ba62c6b2113b045ad Mon Sep 17 00:00:00 2001
From: sunty <1172534965@qq.com>
Date: 星期五, 31 五月 2024 14:03:29 +0800
Subject: [PATCH] Optimize day and night algorithm, format date and time, batch process by date.
---
EsClient.go | 2
EsApi.go | 411 ++++++++++++++++++++++++++++------------------------------
2 files changed, 198 insertions(+), 215 deletions(-)
diff --git a/EsApi.go b/EsApi.go
index c33e153..f27b95c 100644
--- a/EsApi.go
+++ b/EsApi.go
@@ -24,42 +24,6 @@
//***********************閲嶅簡Start**********************************//
-type activeHourFormat struct {
- startTime string
- endTime string
- startHour int
- endHour int
-}
-
-//鎸夐渶姹�(activeHourFormat缁撴瀯浣�)鏍煎紡鍖栨椂闂存暟鎹�
-func formatActiveHour(activeHour string) (activeHourFormat, error) {
- hours := strings.Split(activeHour, "-")
-
- if len(hours) == 2 {
- startHour := hours[0]
- endHour := hours[1]
-
- // 瑙ｆ瀽寮�濮嬫椂闂寸殑灏忔椂鍜屽垎閽�
- startParts := strings.Split(startHour, ":")
- startHourInt, _ := strconv.Atoi(startParts[0])
-
- // 瑙ｆ瀽缁撴潫鏃堕棿鐨勫皬鏃跺拰鍒嗛挓
- endParts := strings.Split(endHour, ":")
- endHourInt, _ := strconv.Atoi(endParts[0])
-
- // 杈撳嚭寮�濮嬫椂闂寸殑灏忔椂
- //fmt.Println("寮�濮嬫椂闂寸殑灏忔椂:", startHourInt)
-
- // 杈撳嚭缁撴潫鏃堕棿鐨勫皬鏃� + 1
- endHourPlusOne := (endHourInt + 1) % 24 // 鍙栦綑纭繚涓嶈秴杩�24灏忔椂
- //fmt.Println("缁撴潫鏃堕棿鐨勫皬鏃� + 1:", endHourPlusOne)
- activeHourFormat := activeHourFormat{startTime: startHour, endTime: endHour, startHour: startHourInt, endHour: endHourPlusOne}
- return activeHourFormat, nil
- }
- return activeHourFormat{}, errors.New("閿欒锛氭棤娉曡В鏋愬紑濮嬫椂闂村拰缁撴潫鏃堕棿")
-
-}
-
//鍒ゆ柇鏃堕棿鏄惁鍐嶈寖鍥翠箣鍐�
func isTimeInRange(timeStr, startStr, endStr string) bool {
layout := "15:04:05"
@@ -178,21 +142,11 @@
// formattedTime := parsedTime.Format("15:04:05")
// return formattedTime, nil
//}
-func resetDataId(dataId []string, id, dDate, dTime string, sDate *string, sTime *string) []string {
- dataId = make([]string, 0)
- *sDate = dDate
- *sTime = dTime
- dataId = append(dataId, id)
- return dataId
-}
-func decodeActivityId(aHFormat activeHourFormat, frequency int, intervalInMinutes int, source []map[string]interface{}) ([]map[string]interface{}, error) {
+func decodeActivityId(frequency int, intervalInMinutes int, source []map[string]interface{}) ([]map[string]interface{}, error) {
docInfo := make([]map[string]interface{}, 0)
for _, info := range source {
tmpInfo := make(map[string]interface{})
- activeId := make([]string, 0)
- sDate := ""
- sTime := ""
documentNumber := info["key"].(string)
tmpInfo["documentNumber"] = documentNumber
//fmt.Println("documentNumber: ", documentNumber)
@@ -213,203 +167,232 @@
picUrl = hitsResult[0].(map[string]interface{})["_source"].(map[string]interface{})["targetInfo"].([]interface{})[0].(map[string]interface{})["picSmUrl"].(string)
}
}
-
- //if hitsResult[0].(map[string]interface{})["baseInfo"] != nil {
- // fmt.Println("picUrl1: ", picUrl)
- // picUrl = hitsResult[0].(map[string]interface{})["baseInfo"].([]interface{})[0].(map[string]interface{})["targetPicUrl"].(string)
- //} else {
- // if hitsResult[0].(map[string]interface{})["targetInfo"] != nil {
- // fmt.Println("picUrl2: ", picUrl)
- // picUrl = hitsResult[0].(map[string]interface{})["targetInfo"].([]interface{})[0].(map[string]interface{})["picSmUrl"].(string)
- // }
- //}
tmpInfo["picUrl"] = picUrl
- for sIndex, sourceInfo := range hitsResult {
+ lastCaptureTime := ""
+ for _, sourceInfo := range hitsResult {
rSourceInfo := sourceInfo.(map[string]interface{})
source := rSourceInfo["_source"].(map[string]interface{})
captureTime := source["picDate"].(string)
- dDate := strings.Split(captureTime, " ")[0]
- dTime := strings.Split(captureTime[:19], " ")[1]
id := source["id"].(string)
- //fmt.Println("sindex: ", sIndex, "documentNumber", tmpInfo["documentNumber"], "id: ", id, "captureTime: ", captureTime)
- if !isTimeInRange(dTime, aHFormat.startTime, aHFormat.endTime) {
- //if sDate != "" && len(dataId) >= frequency {
- // activeId = append(activeId, dataId...)
- // dataId = resetDataId(dataId, id, dDate, dTime, &sDate, &sTime)
- //}
- continue
- }
- if sDate == "" {
- sDate = dDate
- sTime = dTime
- dataId = append(dataId, id)
- if len(dataId) >= frequency {
- activeId = append(activeId, dataId...)
- dataId = resetDataId(dataId, id, dDate, dTime, &sDate, &sTime)
- }
- continue
- }
- if checkTimeDifference(sDate+" "+sTime, captureTime[:19], intervalInMinutes) {
- if len(dataId) >= frequency {
- activeId = append(activeId, dataId...)
- dataId = resetDataId(dataId, id, dDate, dTime, &sDate, &sTime)
- }
- continue
- }
- //fmt.Println(daysBetweenDates(sDate, dDate))
- if aHFormat.startHour < aHFormat.endHour && daysBetweenDates(sDate, dDate) == 0 {
- dataId = append(dataId, id)
- } else if aHFormat.startHour > aHFormat.endHour {
- if daysBetweenDates(sDate, dDate) == 0 {
- if compareTimes(dTime, aHFormat.startTime) == compareTimes(sTime, aHFormat.startTime) {
- // ||compareTimes(dTime,aHFormat.endTime) == compareTimes(sTime, aHFormat.endTime){
- dataId = append(dataId, id)
- }
- } else if daysBetweenDates(sDate, dDate) == 1 {
- //鍒濆鏃堕棿鎴冲湪缁撴潫鑼冨洿涔嬪墠
- if compareTimes(sTime, aHFormat.endTime) == -1 {
- if len(dataId) >= frequency {
- activeId = append(activeId, dataId...)
- dataId = resetDataId(dataId, id, dDate, dTime, &sDate, &sTime)
- }
- //鍒濆鏃堕棿鎴冲湪寮�濮嬭寖鍥翠箣鍚�
- } else if compareTimes(sTime, aHFormat.endTime) == 1 {
- //next鏃堕棿鎴冲湪缁撴潫鑼冨洿涔嬪墠
- if compareTimes(dTime, aHFormat.endTime) == -1 {
- dataId = append(dataId, id)
- //next鏃堕棿鎴冲湪寮�濮嬭寖鍥翠箣鍚�
- } else if compareTimes(dTime, aHFormat.startTime) == 1 {
- if len(dataId) >= frequency {
- activeId = append(activeId, dataId...)
- dataId = resetDataId(dataId, id, dDate, dTime, &sDate, &sTime)
- }
- }
- }
- } else if daysBetweenDates(sDate, dDate) >= 1 {
- //fmt.Println(len(dataId))
- if len(dataId) >= frequency {
- activeId = append(activeId, dataId...)
- dataId = resetDataId(dataId, id, dDate, dTime, &sDate, &sTime)
- }
+ if lastCaptureTime != "" {
+ if checkTimeDifference(lastCaptureTime[:19], captureTime[:19], intervalInMinutes) {
+ //fmt.Println("鏃堕棿灏忎簬杩炵画闃堝��", lastCaptureTime, captureTime)
+ continue
}
}
- if sIndex == len(hitsResult)-1 {
- if len(dataId) >= frequency {
- activeId = append(activeId, dataId...)
- }
- }
+ lastCaptureTime = captureTime
+ dataId = append(dataId, id)
}
- if len(activeId) > 0 {
- tmpInfo["id"] = activeId
+
+ if len(dataId) > frequency {
+ tmpInfo["id"] = dataId
docInfo = append(docInfo, tmpInfo)
}
}
return docInfo, nil
}
-func DayNightActivityQuery(comIds []string, docNumber string, startTime string, endTime string, activeHour string, frequency int,
+func DayNightActivityQuery(comIds []string, cameraIds []string, docNumber string, startDateStr string, endDateStr string, activeHour string, frequency int,
intervalInMinutes int, indexName string, serverIp string, serverPort string) (map[string]interface{}, error) {
esURL := "http://" + serverIp + ":" + serverPort + "/" + indexName + "/_search"
-
- aHFormat, err := formatActiveHour(activeHour)
+ hours := strings.Split(activeHour, "-")
+ startTimeStr := hours[0]
+ endTimeStr := hours[1]
+ layoutTime := "15:04:05"
+ startTime, err := time.Parse(layoutTime, startTimeStr)
if err != nil {
return nil, err
}
+ endTime, err := time.Parse(layoutTime, endTimeStr)
+ if err != nil {
+ return nil, err
+ }
+ layoutDate := "2006-01-02"
+ startDate, err := time.Parse(layoutDate, startDateStr)
+ if err != nil {
+ fmt.Println("Error parsing start time:", err)
+ return nil, err
+ }
+
+ endDate, err := time.Parse(layoutDate, endDateStr)
+ if err != nil {
+ fmt.Println("Error parsing end time:", err)
+ return nil, err
+ }
+ rangePicDate := make([]string, 0)
+ if startTime.Before(endTime) {
+ for !startDate.After(endDate) {
+ sDateTime := startDate.Format(layoutDate) + " " + startTimeStr
+ eDateTime := startDate.Format(layoutDate) + " " + endTimeStr
+ rangePicDate = append(rangePicDate, `
+ {
+ "range": {
+ "picDate": {
+ "gte": "`+sDateTime+`",
+ "lt": "`+eDateTime+`"
+ }
+ }
+ }`)
+ startDate = startDate.AddDate(0, 0, 1)
+ }
+ } else if startTime.After(endTime) {
+ if startDate.Year() == endDate.Year() && startDate.Month() == endDate.Month() && startDate.Day() == endDate.Day() {
+ fmt.Println("璺ㄦ棩鏈熶笉鏀寔鍗曟棩鏌ヨ")
+ return nil, errors.New("璺ㄦ棩鏈熶笉鏀寔鍗曟棩鏌ヨ锛�")
+ }
+ for !startDate.After(endDate) {
+ sDateTime := startDate.Format(layoutDate) + " " + startTimeStr
+ startDate = startDate.AddDate(0, 0, 1)
+ //fendDate := startDate.AddDate(0, 0, 1)
+ if startDate.Year() == endDate.Year() && startDate.Month() == endDate.Month() && startDate.Day() == endDate.Day() {
+ eDateTime := startDate.Format(layoutDate) + " " + endTimeStr
+ rangePicDate = append(rangePicDate, `
+ {
+ "range": {
+ "picDate": {
+ "gte": "`+sDateTime+`",
+ "lt": "`+eDateTime+`"
+ }
+ }
+ }`)
+ //fmt.Println("startDate: ", startDate, "/t endDate: ", endDate)
+ break
+ }
+ eDateTime := startDate.Format(layoutDate) + " " + endTimeStr
+ rangePicDate = append(rangePicDate, `
+ {
+ "range": {
+ "picDate": {
+ "gte": "`+sDateTime+`",
+ "lt": "`+eDateTime+`"
+ }
+ }
+ }`)
+ }
+ }
+ //fmt.Println(rangePicDate)
+ //return nil, nil
filterDocIdAttr := ""
if docNumber != "" {
filterDocIdAttr = "{\"term\": {\"documentNumber\": \"" + docNumber + "\"}},"
}
comIdsStr := ""
- if comIds == nil || len(comIds) > 0 {
+ if comIds != nil && len(comIds) > 0 {
esComIds := strings.Replace(strings.Trim(fmt.Sprint(comIds), "[]"), " ", "\",\"", -1)
comIdsStr = "{\"terms\":{\"communityId\":[\"" + esComIds + "\"]}},"
}
- queryDSL := `
- {
- "size": 0,
- "query": {
- "bool": {
- "filter": [
- {
- "range": {
- "picDate": {
- "gte": "` + startTime + `",
- "lt": "` + endTime + `"
- }
- }
- },
- ` + filterDocIdAttr + `
- ` + comIdsStr + `
- {
- "script": {
- "script": {
- "source": "doc['picDate'].value.hourOfDay >= ` + strconv.Itoa(aHFormat.startHour) + ` || doc['picDate'].value.hourOfDay < ` + strconv.Itoa(aHFormat.endHour) + `",
- "lang": "painless"
- }
- }
- }
- ],
- "must_not": [
- {
- "term": {
- "documentNumber": ""
- }
- }
- ]
- }
- },
- "aggs": {
- "group_by_documentnumber": {
- "terms": {
- "field": "documentNumber",
- "size": 100000
- },
- "aggs": {
- "top_hits": {
- "top_hits": {
- "_source": [
- "id",
- "picDate",
- "baseInfo.targetPicUrl",
- "targetInfo.picSmUrl"
- ],
- "size": 100000,
- "sort": [
- {
- "picDate": {
- "order": "asc"
- }
- }
- ]
- }
- }
- }
- }
- }
- }`
- //fmt.Println(esURL)
- //fmt.Println(queryDSL)
+ cameraIdsStr := ""
+ if comIds != nil && len(cameraIds) > 0 {
+ esCameraIds := strings.Replace(strings.Trim(fmt.Sprint(cameraIds), "[]"), " ", "\",\"", -1)
+ cameraIdsStr = "{\"terms\":{\"cameraId\":[\"" + esCameraIds + "\"]}},"
+ }
+
var result = make(map[string]interface{})
- buf, err := EsReq("POST", esURL, []byte(queryDSL))
- if err != nil {
- return nil, err
+ docInfos := make([]map[string]interface{}, 0)
+ for _, rpd := range rangePicDate {
+ //if comIdsStr != "" || filterDocIdAttr != "" || cameraIdsStr != "" {
+ // rpd += ","
+ //}
+ queryDSL := `
+ {
+ "size": 0,
+ "query": {
+ "bool": {
+ "filter": [
+ ` + comIdsStr + `
+ ` + cameraIdsStr + `
+ ` + filterDocIdAttr + `
+ ` + rpd + `
+ ],
+ "must_not": [
+ {
+ "term": {
+ "documentNumber": ""
+ }
+ }
+ ]
+ }
+ },
+ "aggs": {
+ "group_by_documentnumber": {
+ "terms": {
+ "field": "documentNumber",
+ "size": 100000
+ },
+ "aggs": {
+ "top_hits": {
+ "top_hits": {
+ "_source": [
+ "id",
+ "picDate",
+ "baseInfo.targetPicUrl",
+ "targetInfo.picSmUrl"
+ ],
+ "size": 100000,
+ "sort": [
+ {
+ "picDate": {
+ "order": "asc"
+ }
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }`
+ fmt.Println(esURL)
+ fmt.Println(queryDSL)
+ buf, err := EsReq("POST", esURL, []byte(queryDSL))
+ if err != nil {
+ return nil, err
+ }
+ source, err := SourceAggregationList(buf)
+ if err != nil {
+ return nil, err
+ }
+ //fmt.Println(source)
+ //docResult 鏈鏌ヨ缁撴灉
+ docResult, err := decodeActivityId(frequency, intervalInMinutes, source)
+ if err != nil {
+ return nil, err
+ }
+ if len(docResult) == 0 {
+ continue
+ }
+ if len(docInfos) == 0 {
+ docInfos = append(docInfos, docResult...)
+ continue
+ }
+ for _, dr := range docResult {
+ var found bool
+ for _, di := range docInfos {
+ if dr["documentNumber"].(string) == di["documentNumber"].(string) {
+ // 鎵惧埌浜嗙浉鍚岀殑documentNumber锛屽皢id鍚堝苟鍒癲i["id"]涓�
+ ids, ok := di["id"].([]string)
+ if !ok {
+ // 濡傛灉id涓嶆槸瀛楃涓插垏鐗囩被鍨嬶紝鍒欒繘琛屽垵濮嬪寲
+ ids = []string{}
+ }
+ ids = append(ids, dr["id"].([]string)...)
+ di["id"] = ids
+ //docInfos[diIndex] = di // 鏇存柊鍘熷docInfos涓殑鍊�
+ found = true
+ break
+ }
+ }
+ if !found {
+ // 娌℃湁鎵惧埌鐩稿悓鐨刣ocumentNumber锛屽皢dr娣诲姞鍒癲ocInfos涓�
+ docInfos = append(docInfos, dr)
+ }
+ }
}
- source, err := SourceAggregationList(buf)
- if err != nil {
- return nil, err
+ if len(docInfos) == 0 {
+ return nil, nil
}
- //fmt.Println(source)
- docResult, err := decodeActivityId(aHFormat, frequency, intervalInMinutes, source)
- if err != nil {
- return nil, err
- }
- //result, _ := decodeDocumentInfos(source)
- //return result, nil
- if len(docResult) == 0 {
- return result, nil
- }
- DataInfos, err := GetInfosByIds(docResult[0]["id"].([]string), indexName, serverIp, serverPort)
- result["documentNumbers"] = docResult
+ DataInfos, err := GetInfosByIds(docInfos[0]["id"].([]string), indexName, serverIp, serverPort)
+ result["documentNumbers"] = docInfos
result["datalist"] = DataInfos
return result, nil
}
diff --git a/EsClient.go b/EsClient.go
index 5da48ae..c5b47f1 100644
--- a/EsClient.go
+++ b/EsClient.go
@@ -1056,7 +1056,7 @@
if !ok {
return nil, errors.New("first hits change error!")
}
-
+ //fmt.Println("鏈鍏卞尮閰嶆潯鏁颁负: ", out["hits"].(map[string]interface{})["total"].(float64))
documentAggregations := middle["group_by_documentnumber"].(map[string]interface{})
buckets := documentAggregations["buckets"].([]interface{})
if len(buckets) == 0 {
--
Gitblit v1.8.0