合并插入es分支

panlei

2019-11-13 4c82bdd41470ce7a3690a6fec6317351957e55a4

合并插入es分支

 insertdata/insertDataToEs.go

@@ -40,6 +40,7 @@
    Id              string                `json:"id"`
    CameraId        string                `json:"cameraId"`
    CameraAddr      string                `json:"cameraAddr"`
    CameraName      string                `json:"cameraName"`
    PicDate         string                `json:"picDate"`
    PicMaxUrl       string                `json:"picMaxUrl"`
    TaskId          string                `json:"taskId"`
@@ -61,31 +62,68 @@
    AnalyServerName string                `json:"analyServerName"`
    AnalyServerIp   string                `json:"analyServerIp"`
    ClusterId       string                `json:"clusterId"`
    LinkId          string                `json:"linkId"`
    DetectScore     float64               `json:"detectScore"`
    IsAlarm         int                   `json:"isAlarm"`
    IsAckAlarm      int                   `json:"isAckAlarm"`
    IsCollect       int                   `json:"isCollect"`
    IsDelete        int                   `json:"isDelete"`
    BaseInfo        []*structure.BaseInfo `json:"baseInfo"`
    TargetInfo      Target
    TargetInfo      []Target              `json:"targetInfo"`
    LinkTag         string                `json:"linkTag"`
    LinkTagInfo     []*LinkInfo           `json:"linkTagInfo"`
}

type LinkInfo struct {
    Id              string                `json:"id"`
    CameraId        string                `json:"cameraId"`
    CameraAddr      string                `json:"cameraAddr"`
    CameraName      string                `json:"cameraName"`
    PicDate         string                `json:"picDate"`
    PicMaxUrl       string                `json:"picMaxUrl"`
    TaskId          string                `json:"taskId"`
    TaskName        string                `json:"taskName"`
    SdkName         string                `json:"sdkName"`
    Content         string                `json:"content"`
    AlarmRules      []AlarmRule           `json:"alarmRules"`
    LikeDate        string                `json:"likeDate"`
    Sex             string                `json:"sex"`
    Age             int32                 `json:"age"`
    AgeDescription  string                `json:"ageDescription"`
    Race            string                `json:"race"`
    SmileLevel      int32                 `json:"smileLevel"`
    BeautyLevel     int32                 `json:"beautyLevel"`
    FaceFeature     string                `json:"faceFeature"`
    PicSmUrl        []string              `json:"picSmUrl"`
    VideoUrl        string                `json:"videoUrl"`
    AnalyServerId   string                `json:"analyServerId"`
    AnalyServerName string                `json:"analyServerName"`
    AnalyServerIp   string                `json:"analyServerIp"`
    ClusterId       string                `json:"clusterId"`
    DetectScore     float64               `json:"detectScore"`
    IsAlarm         int                   `json:"isAlarm"`
    IsAckAlarm      int                   `json:"isAckAlarm"`
    IsCollect       int                   `json:"isCollect"`
    IsDelete        int                   `json:"isDelete"`
    BaseInfo        []*structure.BaseInfo `json:"baseInfo"`
    TargetInfo      []Target              `json:"targetInfo"`
}

type Target struct {
    TargetId       string `json:"targetId"`
    TargetId       string  `json:"targetId"`
    TargetScore    float64 `json:"targetScore"`
    TargetLocation Points
    TargetLocation Points  `json:"targetLocation"`
}

type Points struct {
    TopLeft  Point `json:"topLeft"`
    TopLeft     Point `json:"topLeft"`
    BottomRight Point `json:"bottomRight"`
}

type Point struct {
    Lat float64 `json:"lat"`
    Lon float64 `json:"lon"`
    X float64 `json:"x"`
    Y float64 `json:"y"`
}

//  yolo行为的数据结构
type Personaction struct {
    Id              string      `json:"id"`
@@ -103,13 +141,14 @@
    ClusterId       string      `json:"clusterId"`
    PicSmUrl        []string    `json:"picSmUrl"`
    PicDate         string      `json:"picDate"`
    LinkId          string      `json:"linkId"`
    VideoUrl        string      `json:"videoUrl"`
    IsAlarm         int         `json:"isAlarm"`
    IsAckAlarm      int         `json:"isAckAlarm"`
    IsCollect       int         `json:"isCollect"`
    IsDelete        int         `json:"isDelete"`
    TargetInfo        []Target
    TargetInfo      []Target    `json:"targetInfo"`
    LinkTag         string      `json:"linkTag"`
    LinkTagInfo     []*LinkInfo `json:"linkTagInfo"`
}

type AlarmRule struct {
@@ -117,7 +156,6 @@
    AlarmLevel   string `json:"alarmLevel"`
    RuleText     string `json:"ruleText"`
    DefenceState bool   `json:"defenceState"`
    IsLink       bool   `json:"isLink"`
    LinkInfo     string `json:"linkInfo"`
}

@@ -140,12 +178,12 @@
    weedfsUrl = "http://" + localConfig1.WebPicIp + ":" + strconv.Itoa(int(localConfig1.WebPicPort)) + "/submit"
    videoPersonUrl = "http://" + localConfig1.AlarmIp + ":" + strconv.Itoa(int(localConfig1.AlarmPort)) + "/" + EsInfo.EsIndex.VideoPersons.IndexName + "/" + EsInfo.EsIndex.VideoPersons.IndexType
    personAction = "http://" + localConfig1.AlarmIp + ":" + strconv.Itoa(int(localConfig1.AlarmPort)) + "/" + EsInfo.EsIndex.Personaction.IndexName + "/" + EsInfo.EsIndex.Personaction.IndexType
    InsertFace(msg, "")
    InsertYolo(msg, "")
    InsertFace(msg)
    InsertYolo(msg)
}

// 往es中插入人脸数据
func InsertFace(msg structure.ResultMsg, linkId string) {
func InsertFace(msg structure.ResultMsg) {
    if msg.RuleResult["face"] != nil && len(msg.RuleResult["face"].([]structure.TargetResult)) > 0 {
        logger.Info("往ES插人脸数据")
        faces := []*FaceAndRules{}
@@ -165,7 +203,7 @@
                //os.Exit(1)
                for _, faceResult := range face.rules {
                    alarm := ChangeToString(faceResult.DefenceState, faceResult.AlarmLevel)
                    alarmRules = append(alarmRules, AlarmRule{faceResult.RuleGroupId, alarm, faceResult.RuleText, faceResult.DefenceState, faceResult.IsLink, ""})
                    alarmRules = append(alarmRules, AlarmRule{faceResult.RuleGroupId, alarm, faceResult.RuleText, faceResult.DefenceState, ""})
                }
                i := protomsg.Image{}
                err = proto.Unmarshal(bdata, &i)
@@ -193,21 +231,18 @@
                race := getRaceString(face.ThftRes.Race)
                ageDescription := getDescription(face.ThftRes.Age)
                logger.Info(ageDescription)
                //esDataId := uuid.NewV4().String()
                linksId := ""
                if linkId != "" {
                    linksId = linkId
                }
                var target = new(Target)
                target.TargetId = strconv.FormatUint(face.Id, 10)
                target.TargetScore = face.Score
                target.TargetLocation = Points{TopLeft:Point{face.Location.X,face.Location.Y},BottomRight:Point{face.Location.X+face.Location.Width,face.Location.Y+face.Location.Height}}
                target.TargetLocation = Points{TopLeft: Point{face.Location.X, face.Location.Y}, BottomRight: Point{face.Location.X + face.Location.Width, face.Location.Y + face.Location.Height}}
                //logger.Info("人脸的id:",strconv.FormatUint(face.Id, 10))

                var targetInfos []Target
                targetInfos = append(targetInfos, *target)
                pervideo := PerVideoPicture{
                    msg.Push.PushId,
                    face.Uuid,
                    msg.Cid,
                    msg.Push.Cam.Addr,
                    msg.Push.Cam.Name,
                    picTime,
                    imgMaxUrl,
                    msg.Tasklab.Taskid,
@@ -229,14 +264,15 @@
                    msg.Push.ServerName,
                    msg.Push.LocalIp,
                    "",
                    linksId,
                    face.Score,
                    1,
                    0,
                    0,
                    0,
                    face.Liker,
                    *target,
                    targetInfos,
                    "",
                    []*LinkInfo{},
                }
                requstbody, err := json.Marshal(pervideo)

@@ -248,9 +284,9 @@
                if err1 != nil {
                    logger.Error("上传ES出错！---", err1)
                } else {
                    logger.Info("插入es返回的信息：", resp1)
                    logger.Info("插入es返回的信息(人脸)：", resp1)
                    // 发出录像信号
                    ruleserver.AddLxMessage(&protomsg.VideotapeInfo{EsDataId: msg.Push.PushId, CameraId: msg.Cid, TaskId: msg.Tasklab.Taskid, VideoUrl:msg.Push.VideoUrl,ImgId: i.Id, SdkIds: []string{}, Type: 1})
                    ruleserver.AddLxMessage(&protomsg.VideotapeInfo{EsDataId: face.Uuid, CameraId: msg.Cid, TaskId: msg.Tasklab.Taskid, VideoUrl: msg.Push.VideoUrl, ImgId: i.Id, SdkIds: []string{}, Type: 1})
                }
            }
        }
@@ -303,7 +339,7 @@
}

// 往es中插入yolo数据
func InsertYolo(msg structure.ResultMsg, linkId string) {
func InsertYolo(msg structure.ResultMsg) {
    if msg.RuleResult["yolo"] != nil && len(msg.RuleResult["yolo"].([]structure.Result)) > 0 {
        // 先判断一下数据带的规则标签是否有可以插入的
        flag := false
@@ -329,7 +365,7 @@
                    if yoloResult.IsLink {
                        linkInfo = "联动任务"
                    }
                    alarmRules = append(alarmRules, AlarmRule{yoloResult.RuleGroupId, alarm, yoloResult.RuleText, yoloResult.DefenceState, yoloResult.IsLink, linkInfo})
                    alarmRules = append(alarmRules, AlarmRule{yoloResult.RuleGroupId, alarm, yoloResult.RuleText, yoloResult.DefenceState, linkInfo})
                    // 上传缓存数据的图片拿到url
                    if yoloResult.Others.CacheData != nil {
                        //InsertYolo(msg.RuleResult["cacheData"].(ruleserver.ResultMsg))
@@ -355,10 +391,10 @@
                        }
                    }
                    // 装配目标信息数据
                    for _,target := range yoloResult.Location  {
                    for _, target := range yoloResult.Location {
                        // 去重添加
                        var flag = true
                        for _,selectTarget := range targetInfos  {
                        for _, selectTarget := range targetInfos {
                            if strconv.FormatUint(target.TargetId, 10) == selectTarget.TargetId {
                                flag = false
                                break
@@ -368,30 +404,28 @@
                            var target1 = new(Target)
                            target1.TargetId = strconv.FormatUint(target.TargetId, 10)
                            target1.TargetScore = target.TargetScore
                            target1.TargetLocation = Points{TopLeft:Point{target.X,target.Y},BottomRight:Point{target.X+target.Width,target.Y+target.Height}}
                            targetInfos = append(targetInfos,*target1)
                            target1.TargetLocation = Points{TopLeft: Point{target.X, target.Y}, BottomRight: Point{target.X + target.Width, target.Y + target.Height}}
                            targetInfos = append(targetInfos, *target1)
                        }
                    }
                }
            }
            linkFlag := false

            linkTagInfos := []*LinkInfo{}
            for _, yoloResult := range msg.RuleResult["yolo"].([]structure.Result) {
                if (yoloResult.Others.TimeLabel == "01" || yoloResult.Others.TimeLabel == "10") && yoloResult.Others.LinkCache != nil && len(yoloResult.Others.LinkCache) > 1 {
                    linkId := uuid.NewV4().String()
                    for _, msg2 := range yoloResult.Others.LinkCache {
                        logger.Warn("插入联动数据", "此帧数据的id为", msg2.Cid)
                        InsertYolo(msg2, linkId)
                        linkFlag = true
                        // 把msg2的数据装配成一个PerVideoPicture结构体
                        if msg2.Cid != msg.Cid {
                            linkTagInfos = append(linkTagInfos, msg2PersonVideo(msg2))
                        }
                    }
                }
            }
            if linkFlag {
                // 联动数据中包含本帧数据，插入联动数据后就不需要插入本帧数据了
                logger.Warn("成功插入两个联动图片")
                //os.Exit(1)
                return
            linkTag := ""
            if len(linkTagInfos) > 0 {
                linkTag = "联动任务"
            }
            //logger.Info("--------走到这儿就不一样")
            isAlarm := 0
            resp := make(map[string]interface{})
            // 解压缩并上传图片
@@ -419,10 +453,6 @@
            if resp["fileUrl"] != nil {
                url = append(url, resp["fileUrl"].(string))
                //esDataId := uuid.NewV4().String()
                linksId := ""
                if linkId != "" {
                    linksId = linkId
                }
                peraction := Personaction{
                    msg.Push.PushId,
                    msg.Cid,
@@ -439,16 +469,20 @@
                    "",
                    url,
                    i.Timestamp,
                    linksId,
                    "",
                    isAlarm,
                    0,
                    0,
                    0,
                    targetInfos,
                    linkTag,
                    linkTagInfos,
                }
                requstbody, err := json.Marshal(peraction)

                if len(linkTagInfos)>0 {
                    logger.Info("联动任务1111")
                }
                if err != nil {
                    logger.Info("json parse error ", err)
                    return
@@ -457,9 +491,9 @@
                if err2 != nil {
                    logger.Error("往ES插入数据失败", err)
                } else {
                    logger.Debug("插入es返回的数据信息是：", resp1)
                    logger.Debug("插入es返回的数据信息是（yolo）：", resp1)
                    // 发出录像信号
                    ruleserver.AddLxMessage(&protomsg.VideotapeInfo{EsDataId: msg.Push.PushId, CameraId: msg.Cid, TaskId: msg.Tasklab.Taskid,VideoUrl:msg.Push.VideoUrl, ImgId: i.Id, SdkIds: []string{}, Type: 2})
                    ruleserver.AddLxMessage(&protomsg.VideotapeInfo{EsDataId: msg.Push.PushId, CameraId: msg.Cid, TaskId: msg.Tasklab.Taskid, VideoUrl: msg.Push.VideoUrl, ImgId: i.Id, SdkIds: []string{}, Type: 2})
                    logger.Warn("__________________________________________往ES插入yolo数据成功")
                    //os.Exit(1)
                }
@@ -552,3 +586,148 @@
    }
    return race
}

func msg2PersonVideo(msg structure.ResultMsg) *LinkInfo {
    if msg.RuleResult["yolo"] != nil && len(msg.RuleResult["yolo"].([]structure.Result)) > 0 {
        // 先判断一下数据带的规则标签是否有可以插入的
        flag := false
        for _, res := range msg.RuleResult["yolo"].([]structure.Result) {
            //logger.Info("定时器打的数字标签：",res.Others.TimeLabel)
            if res.Others.TimeLabel == "01" || res.Others.TimeLabel == "10" {
                flag = true
            }
        }
        if flag {
            logger.Info("插入Yolo组装数据")
            var sdkNames string = ""
            alarmRules := []AlarmRule{}
            var targetInfos []Target
            url := []string{}
            for _, yoloResult := range msg.RuleResult["yolo"].([]structure.Result) {
                if yoloResult.Others.TimeLabel == "01" || yoloResult.Others.TimeLabel == "10" {
                    // 拼出sdkname
                    //logger.Info("应该进来才对的")
                    sdkNames = sdkNames + yoloResult.SdkName
                    alarm := ChangeToString(yoloResult.DefenceState, yoloResult.AlarmLevel)
                    linkInfo := ""
                    if yoloResult.IsLink {
                        linkInfo = "联动任务"
                    }
                    alarmRules = append(alarmRules, AlarmRule{yoloResult.RuleGroupId, alarm, yoloResult.RuleText, yoloResult.DefenceState, linkInfo})
                    // 上传缓存数据的图片拿到url
                    if yoloResult.Others.CacheData != nil {
                        //InsertYolo(msg.RuleResult["cacheData"].(ruleserver.ResultMsg))
                        // 把缓存的数据上传后得到地址存进去
                        // 解压缩并上传图片
                        msgs := yoloResult.Others.CacheData
                        for _, msg1 := range msgs {
                            bdata, err := util.UnCompress(msg1.Data)
                            if err != nil {
                                panic("解压缩图片时出现错误")
                            }
                            i := protomsg.Image{}
                            err = proto.Unmarshal(bdata, &i)
                            resp1, err1 := util.DrawPolygonOnImageForYolo(msg1.Cid, i, msg1.RuleResult["yolo"].([]structure.Result), weedfsUrl)
                            if err1 != nil {
                                logger.Error("缓存数据画框或上传图片服务器出错", err)
                            } else {
                                logger.Info("上传的图片信息：", resp1)
                            }
                            if resp1["fileUrl"] != nil {
                                url = append(url, resp1["fileUrl"].(string))
                            }
                        }
                    }
                    // 装配目标信息数据
                    for _, target := range yoloResult.Location {
                        // 去重添加
                        var flag = true
                        for _, selectTarget := range targetInfos {
                            if strconv.FormatUint(target.TargetId, 10) == selectTarget.TargetId {
                                flag = false
                                break
                            }
                        }
                        if flag {
                            var target1 = new(Target)
                            target1.TargetId = strconv.FormatUint(target.TargetId, 10)
                            target1.TargetScore = target.TargetScore
                            target1.TargetLocation = Points{TopLeft: Point{target.X, target.Y}, BottomRight: Point{target.X + target.Width, target.Y + target.Height}}
                            targetInfos = append(targetInfos, *target1)
                        }
                    }
                }
            }
            isAlarm := 0
            resp := make(map[string]interface{})
            // 解压缩并上传图片
            bdata, err := util.UnCompress(msg.Data)
            if err != nil {
                panic("解压缩图片时出现错误")
            }
            i := protomsg.Image{}
            err = proto.Unmarshal(bdata, &i)
            if len(alarmRules) > 0 {
                isAlarm = 1
                //resp, err = util.PostFormBufferData(weedfsUrl, i, uuid.NewV4().String())
                resp, err = util.DrawPolygonOnImageForYolo(msg.Cid, i, msg.RuleResult["yolo"].([]structure.Result), weedfsUrl)
                if err != nil {
                    logger.Error("画框或上传图片服务器出错", err)
                    return nil
                } else {
                    logger.Info("上传的图片信息：", resp)
                }
            } else {
                isAlarm = 0
                // 不是报警数据不存
                return nil
            }
            if resp["fileUrl"] != nil {
                url = append(url, resp["fileUrl"].(string))
                //esDataId := uuid.NewV4().String()
                pervideo := &LinkInfo{
                    msg.Push.PushId + "-" + uuid.NewV4().String(),
                    msg.Cid,
                    msg.Push.Cam.Addr,
                    msg.Push.Cam.Name,
                    i.Timestamp,
                    "",
                    msg.Tasklab.Taskid,
                    msg.Tasklab.Taskname,
                    "人脸",
                    "",
                    alarmRules,
                    time.Now().Format("2006-01-02 15:04:05"), // 只检测，没有比对时间
                    "",                                       //暂改为人脸id strconv.FormatUint(face.Id, 10)
                    0,
                    "", // 暂改为分值fmt.Sprintf("%.2f",face.Score)
                    "",
                    0,
                    0,
                    "",
                    []string{resp["fileUrl"].(string)},
                    "",
                    msg.Push.ServerId,
                    msg.Push.ServerName,
                    msg.Push.LocalIp,
                    "",
                    0,
                    isAlarm,
                    0,
                    0,
                    0,
                    nil,
                    targetInfos,
                }
                return pervideo
            } else {
                return nil
            }
        } else {
            logger.Debug("timeLabel条件都不符合！")
            return nil
        }
    } else {
        return nil
    }
}

 ruleserver/readyDataForRule.go

@@ -8,6 +8,7 @@
    "errors"
    "fmt"
    "github.com/golang/protobuf/proto"
    uuid "github.com/satori/go.uuid"
    "math"
    "nanomsg.org/go-mangos"
    "nanomsg.org/go-mangos/protocol/req"
@@ -273,7 +274,8 @@
        if  obj.Score >= threshold && float64(obj.Rects.Width*obj.Rects.Height) >= size && PgsInterPercent(areaPoints, obj.Rects, widthScale, heigthScale) >= intersectionper {
            // 这步要备齐表达式里所需要的所有参数
            a.TargetNum++
            arg1 := structure.Arg{obj.Id,obj.Score, PgsInterPercent(areaPoints, obj.Rects, widthScale, heigthScale), float64(obj.Rects.Width * obj.Rects.Height), a.AreaJson,obj.IsYolo, obj.Rects, obj.Feature, obj.ThftRes, []*structure.BaseInfo{},nil}
            uuid := uuid.NewV4().String()
            arg1 := structure.Arg{obj.Id,uuid,obj.Score, PgsInterPercent(areaPoints, obj.Rects, widthScale, heigthScale), float64(obj.Rects.Width * obj.Rects.Height), a.AreaJson,obj.IsYolo, obj.Rects, obj.Feature, obj.ThftRes, []*structure.BaseInfo{},nil}
            //logger.Println("放进去的arg：-------", arg1)
            a.Args = append(a.Args, &arg1)
            a.FilterData = append(a.FilterData, &arg1)

 structure/rule.go

@@ -1,10 +1,13 @@
package structure

import "basic.com/pubsub/protomsg.git"
import (
    "basic.com/pubsub/protomsg.git"
)

// 每个目标的参数：相似度，占比，尺寸
type Arg struct {
    Id         uint64
    Uuid       string
    Score      float64 // 区域内的目标的相似度
    Proportion float64 // 区域内的目标的占比
    Size       float64 // 区域内的目标的尺寸