添加疑似托管模型

zhangzengfei

2024-12-20 a25ee90c656b39e652f24e3378ad2bf5698b162b

添加疑似托管模型

 cron/cron.go

@@ -46,7 +46,7 @@
}

func Run() (err error) {
    _, err = s.Every(1).Minute().StartImmediately().Do(Dispatch)
    _, err = s.Every(10).Minute().StartImmediately().Do(Dispatch)
    if err != nil {
        return err
    }

 db/db.go

@@ -6,13 +6,12 @@

    "model-engine/config"
    "model-engine/db/es"
    "model-engine/pkg/logger"
    "model-engine/pkg/mysqlx"
)

// Init 初始化mysql和es
func Init() error {
    if err := mysqlx.Init(config.MysqlConf, logger.GetLogger()); err != nil {
    if err := mysqlx.Init(config.MysqlConf, nil); err != nil {
        return err
    }
    if err := es.InitClient([]string{"http://" + config.EsInfo.Ip + ":" + config.EsInfo.Port}); err != nil {
@@ -65,6 +64,15 @@
            Name:        "疑似聚集",
            Description: "通用聚集模型",
            Version:     "v1.0.1",
            Enabled:     false,
        },
        {
            BaseModel: BaseModel{
                ID: ModelIdDisappear,
            },
            Name:        "疑似脱管",
            Description: "通用脱管模型",
            Version:     "v1.0.0",
            Enabled:     false,
        },
    }
@@ -143,6 +151,57 @@
                Value:    "1",
                ValType:  "int",
                Operator: "==",
                Sort:     3,
            },
        },
        {
            Id:      "7a1f0a3a-c207-4d94-bc28-cc9e017b3628",
            ModelId: ModelIdDisappear,
            Scope:   "",
            RuleArg: RuleArg{
                Alias:    "disappearTime",
                Name:     "持续时间",
                Type:     "input",
                Must:     true,
                Unit:     "小时",
                Range:    "1,2400",
                Value:    "24",
                ValType:  "int",
                Operator: ">=",
                Sort:     0,
            },
        },
        {
            Id:      "f1b99f28-1be0-4f78-b7c1-b01b1656b7fa",
            ModelId: ModelIdDisappear,
            Scope:   "",
            RuleArg: RuleArg{
                Alias:    "age",
                Name:     "年龄段",
                Type:     "input",
                Must:     true,
                Unit:     "岁",
                Range:    "1,100",
                Value:    "60, 90",
                ValType:  "int",
                Operator: "==",
                Sort:     1,
            },
        },
        {
            Id:      "47366fa6-2f61-4fe0-957a-b1e0606bb1f0",
            ModelId: ModelIdDisappear,
            Scope:   "",
            RuleArg: RuleArg{
                Alias:    "age",
                Name:     "进出方向",
                Type:     "input",
                Must:     false,
                Unit:     "",
                Range:    "",
                Value:    "",
                ValType:  "string",
                Operator: "==",
                Sort:     2,
            },
        },

 db/model.go

@@ -2,16 +2,18 @@

import (
    "fmt"

    "gorm.io/gorm"

    "model-engine/pkg/mysqlx"
)

type Model struct {
    BaseModel
    Name        string `json:"name" gorm:"type:varchar(255)"`                   //模型名称
    Description string `json:"description,omitempty" gorm:"type:varchar(1000)"` //模型描述
    Version     string `json:"version" gorm:"type:varchar(255)"`                //版本号
    Enabled     bool   `json:"enabled"`                                         //是否开启
    Name        string `json:"name" gorm:"type:varchar(255)"`                   // 模型名称
    Description string `json:"description,omitempty" gorm:"type:varchar(1000)"` // 模型描述
    Version     string `json:"version" gorm:"type:varchar(255)"`                // 版本号
    Enabled     bool   `json:"enabled"`                                         // 是否开启
}

func (m *Model) TableName() string {
@@ -181,6 +183,7 @@
}

const (
    ModelIdDrug   = "drug"   //涉毒
    ModelIdGather = "gather" //聚集
    ModelIdDrug      = "drug"      // 涉毒
    ModelIdGather    = "gather"    // 聚集
    ModelIdDisappear = "disappear" // 失踪
)

 db/persons.go

New file
@@ -0,0 +1,54 @@
package db

import (
    "time"

    "gorm.io/gorm"
)

type BaseEntity struct {
    ID        string `gorm:"primary_key;column:id;type:varchar(255);" json:"id"`
    CreatedAt time.Time
    UpdatedAt time.Time
    DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
}

type DbPersons struct {
    BaseEntity
    TableId            string  `gorm:"column:table_id"`
    FaceFeature        string  `gorm:"column:face_feature"`
    PersonPicUrl       string  `gorm:"column:person_pic_url"`
    PersonName         string  `gorm:"column:person_name"`
    Age                string  `gorm:"column:age"`
    Sex                string  `gorm:"column:sex"`
    IdCard             string  `gorm:"column:id_card"`
    PhoneNum           string  `gorm:"column:phone_num"`
    MonitorLevel       string  `gorm:"column:monitor_level"`
    PicDesc            string  `gorm:"column:pic_desc"`
    Reserved           string  `gorm:"column:reserved"`
    FromServerId       string  `gorm:"column:from_server_id"`
    LastAppearanceTime int64   `gorm:"column:last_appearance_time"`
    SnapshotCount      int     `gorm:"column:snapshot_count"`
    DaysAppeared       int     `gorm:"column:days_appeared"`
    Location           string  `gorm:"column:location"`
    LastLocation       string  `gorm:"column:last_location"`
    FaceAngleYaw       int32   `gorm:"column:face_angle_yaw"`
    FaceAngleRoll      int32   `gorm:"column:face_angle_roll"`
    FaceAnglePitch     int32   `gorm:"column:face_angle_pitch"`
    CommunityID        string  `gorm:"column:community_id"` // 常住小区 domain unit ID
    CommunityName      string  `gorm:"column:community_name"`
    OrgID              string  `gorm:"column:org_id"` // 常住派出所 domain unit ID
    OrgName            string  `gorm:"column:community"`
    FrontFaceScore     float64 `gorm:"column:front_face_score"`
    ImageQuality       float64 `gorm:"column:image_quality"`
    SimilarPersonId    string  `gorm:"column:similar_person_id"`
    PersonalStatus     string  `gorm:"column:personal_status"`
    NonHuman           bool    `gorm:"column:non_human"` // 非活体（1非多体 0活体）
    IdPicUrl           string  `gorm:"column:id_pic_url"`
    IsDelete           int     `gorm:"column:is_delete"` // 0 未删除 1已删除
    FaceUpdateTime     int64   `gorm:"column:face_update_time"`
}

func (dp *DbPersons) TableName() string {
    return "person"
}

 db/summay.go

New file
@@ -0,0 +1,29 @@
package db

type SnapshotCountSummary struct {
    OrgId                    string  `gorm:"column:org_id"`                      // 派出所id
    CommunityID              string  `gorm:"column:community_id"`                // 小区id
    DocumentNumber           string  `gorm:"column:document_number"`             // 档案编号
    DaysAppeared             int     `gorm:"column:days_appeared"`               // 出现天数
    Count                    int     `gorm:"column:count"`                       // 抓拍次数
    Status                   string  `gorm:"column:status"`                      // 标签
    LastAppearanceTime       int64   `gorm:"column:last_appearance_time"`        // 最后出现时间
    LastAppearanceStatusTime int64   `gorm:"column:last_appearance_status_time"` // 最后出现时间
    LastLocation             string  `gorm:"column:last_location"`               // 最后出现地点
    Building                 string  `gorm:"column:building"`                    // 楼栋号
    Floor                    string  `gorm:"column:floor"`                       // 楼层
    FrequentAddress          string  `gorm:"column:frequent_address"`            // 常住地址
    IDStatus                 int     `gorm:"type:tinyint;not null;default:2"`    // 1已实名 2未实名
    KeyStatus                int     `gorm:"type:tinyint;not null;default:2"`    // 1重点人员 2 非重点人员
    StatusScore              float64 `gorm:"column:status_score"`                // 居住属性评分
    StatusInitTime           int64   `gorm:"column:status_init_time"`
    LastDirection            string  `gorm:"column:last_direction;type:varchar(255);"` // 最后出现方向
}

func (sc *SnapshotCountSummary) TableName() string {
    return "snapshot_count_summary"
}

func (sc *SnapshotCountSummary) FindByPersonId(commId, docId string) error {
    return GetDB().Table(sc.TableName()).Where("community_id = ? and document_number = ?", commId, docId).First(sc).Error
}

 db/task_results.go

@@ -2,24 +2,25 @@

import (
    "fmt"
    "gorm.io/gorm"
    "time"

    "gorm.io/gorm"
)

type ModelTaskResults struct {
    BaseModel
    Title         string `json:"name" gorm:"type:varchar(255)"`                                                //预警名称,暂时用任务名称
    Event         string `json:"event" gorm:"type:varchar(255)"`                                               //预警事件
    ModelID       string `json:"modelID" gorm:"type:varchar(255)"`                                             //模型ID
    ModelTaskID   string `json:"modelTaskID" gorm:"type:varchar(255)"`                                         //模型任务ID
    OrgID         string `json:"orgID" gorm:"index;column:org_id;type:varchar(299);not null;"`                 //派出所 domain unit ID
    CommunityId   string `json:"communityID" gorm:"index;column:community_id;type:varchar(299);not null;"`     //小区ID
    PicDate       string `json:"picDate"       gorm:"uniqueIndex:pic_date_first_person_id;type:varchar(255);"` //抓拍时间
    FirstPersonID string `json:"-" gorm:"uniqueIndex:pic_date_first_person_id;type:varchar(255);"`             //第一个人的ID
    ObjectIds     string `json:"objectIds" gorm:"type:text"`                                                   //事件对象，可以是人，多个用逗号分隔
    Location      string `json:"location" gorm:"type:varchar(255)"`                                            //发生地点
    Building      string `json:"building" gorm:"type:varchar(255);"`                                           //楼栋
    Floor         string `json:"floor"       gorm:"type:varchar(255);"`                                        //楼层
    Title         string `json:"name" gorm:"type:varchar(255)"`                                                // 预警名称,暂时用任务名称
    Event         string `json:"event" gorm:"type:varchar(255)"`                                               // 预警事件
    ModelID       string `json:"modelID" gorm:"type:varchar(255)"`                                             // 模型ID
    ModelTaskID   string `json:"modelTaskID" gorm:"type:varchar(255)"`                                         // 模型任务ID
    OrgID         string `json:"orgID" gorm:"index;column:org_id;type:varchar(299);not null;"`                 // 派出所 domain unit ID
    CommunityId   string `json:"communityID" gorm:"index;column:community_id;type:varchar(299);not null;"`     // 小区ID
    PicDate       string `json:"picDate"       gorm:"uniqueIndex:pic_date_first_person_id;type:varchar(255);"` // 抓拍时间
    FirstPersonID string `json:"-" gorm:"uniqueIndex:pic_date_first_person_id;type:varchar(255);"`             // 第一个人的ID
    ObjectIds     string `json:"objectIds" gorm:"type:text"`                                                   // 事件对象，可以是人，多个用逗号分隔
    Location      string `json:"location" gorm:"type:varchar(255)"`                                            // 发生地点
    Building      string `json:"building" gorm:"type:varchar(255);"`                                           // 楼栋
    Floor         string `json:"floor"       gorm:"type:varchar(255);"`                                        // 楼层

}

@@ -69,6 +70,11 @@
    return slf
}

func (slf *ModelTaskResultsSearch) SetTaskId(id string) *ModelTaskResultsSearch {
    slf.ModelTaskID = id
    return slf
}

func (slf *ModelTaskResultsSearch) SetFirstPersonId(firstId string) *ModelTaskResultsSearch {
    slf.FirstPersonID = firstId
    return slf
@@ -97,6 +103,10 @@
        db = db.Where("first_person_id = ?", slf.FirstPersonID)
    }

    if slf.ModelTaskID != "" {
        db = db.Where("model_task_id = ?", slf.ModelTaskID)
    }

    if slf.Keyword != "" {
        kw := "%" + slf.Keyword + "%"
        db = db.Where("name like ?", kw)

 models/disappear.go

New file
@@ -0,0 +1,270 @@
package models

import (
    "errors"
    "fmt"
    "strings"
    "time"

    "model-engine/db"
    "model-engine/pkg/logger"
    "model-engine/service"
)

type DisappearModel struct {
    AreaIds       map[string]struct{}
    Building      string       // 楼栋
    Floor         string       // 楼层
    AlarmType     db.AlarmType // 预警方式
    KeyPersonType string       // 人员类型
    PersonLabel   string
    DisappearTime int    // 消失时间, 单位小时
    AlarmInterval int    // 报警时间间隔, 单位天
    LastDirection string // 最后一次抓拍
    PersonAge     int    // 年龄
    Task          *db.ModelTask
}

func (m *DisappearModel) Init(task *db.ModelTask) error {
    m.AreaIds = make(map[string]struct{})
    for _, a := range task.DomainUnitIds {
        m.AreaIds[a] = struct{}{}
    }

    m.Task = task
    m.Building = task.Building
    m.Floor = task.Floor
    m.AlarmType = task.AlarmType
    m.KeyPersonType = task.PersonType
    m.PersonLabel = task.PersonLabel

    for _, v := range task.Rules {
        if v.Alias == "disappearTime" {
            if val, ok := v.Value.(float64); ok {
                m.DisappearTime = int(val)
            }
        }

        if v.Alias == "alarmInterval" {
            if val, ok := v.Value.(float64); ok {
                m.AlarmInterval = int(val)
            }
        }

        if v.Alias == "age" {
            if val, ok := v.Value.(float64); ok {
                m.PersonAge = int(val)
            }
        }

        if v.Alias == "lastDirection" {
            if val, ok := v.Value.(string); ok {
                m.LastDirection = val
            }
        }

    }

    logger.Debugf("DisappearModel init finish ...task id:%s, name:%s, rule:%+v\n", task.ID, task.Name, m)

    if m.DisappearTime == 0 {
        logger.Warnf("invalid parameters. task id:%s, name:%s\n", task.ID, task.Name)
        return errors.New("invalid parameters")
    }

    return nil
}

type PersonInfo struct {
    DocumentNumber     string `json:"document_number"`
    CommunityId        string `json:"community_id"`
    OrgId              string `json:"org_id"`
    PersonName         string `json:"person_name"`
    IdCard             string `json:"id_card"`
    LastAppearanceTime int64  `json:"last_appearance_time"`
    LastDirection      string `json:"last_direction"`
    LastLocation       string `json:"last_location"`
}

func (m *DisappearModel) Run() error {
    results := make([]*db.ModelTaskResults, 0)
    var ageFilter, labelFilter, keyFilter, lastFilter []PersonInfo

    if m.PersonAge > 0 {
        err := db.GetDB().Raw(`
        SELECT
            s.document_number,
            s.community_id,
            s.org_id,
            p.person_name,
            p.id_card ,
            s.last_appearance_time,
            s.last_direction,
            s.last_location
        FROM
            snapshot_count_summary AS s
            JOIN person AS p ON p.id = s.document_number 
        WHERE
            s.STATUS = 'resident' 
            AND p.id_card != "" 
            AND TIMESTAMPDIFF(
                YEAR,
                STR_TO_DATE( CASE WHEN LENGTH( id_card ) = 18 THEN SUBSTRING( id_card, 7, 8 ) ELSE NULL END, '%Y%m%d' ),
            CURDATE( ) > ?
        `, m.PersonAge).Scan(&ageFilter).Error
        if err != nil {
            logger.Warnf(err.Error())
        }

        if len(ageFilter) == 0 {
            return fmt.Errorf("no results found that match the age condition %d", m.PersonAge)
        }
    }

    if m.PersonLabel != "" {
        labels := strings.Split(m.PersonLabel, ",")
        err := db.GetDB().Raw(`
        SELECT
            s.document_number,
            s.community_id,
            s.org_id,
            p.person_name,
            p.id_card,
            s.last_appearance_time,
            s.last_direction,
            s.last_location 
        FROM
            snapshot_count_summary AS s
            JOIN person AS p ON p.id = s.document_number
            JOIN person_label AS l ON p.id = l.person_id 
        WHERE
            l.label_id IN ?
        `, labels).Scan(&labelFilter).Error
        if err != nil {
            logger.Warnf(err.Error())
        }

        if len(labelFilter) == 0 {
            return fmt.Errorf("no results found that match the label condition %s", m.PersonLabel)
        }
    }

    // 合并一下条件
    if m.PersonAge > 0 && m.PersonLabel != "" {
        lastFilter = intersectPersonInfo(ageFilter, labelFilter)
    } else if m.PersonAge > 0 {
        lastFilter = ageFilter
    } else if m.PersonLabel != "" {
        lastFilter = labelFilter
    }

    if m.KeyPersonType != "" {
        keyTypes := strings.Split(m.KeyPersonType, ",")
        err := db.GetDB().Raw(`
        SELECT
            s.document_number,
            s.community_id,
            s.org_id,
            p.person_name,
            p.id_card,
            s.last_appearance_time,
            s.last_direction,
            s.last_location 
        FROM
            snapshot_count_summary AS s
            JOIN person AS p ON p.id = s.document_number
            JOIN key_person AS k ON k.id_card = p.id_card 
        WHERE
            s.key_status = 1 
            AND p.id_card != "" 
            AND k.person_type IN ?
        `, keyTypes).Scan(&keyFilter).Error
        if err != nil {
            logger.Warnf(err.Error())
        }
    }

    if len(lastFilter) > 0 {
        lastFilter = intersectPersonInfo(lastFilter, keyFilter)
    } else {
        lastFilter = keyFilter
    }

    for _, p := range lastFilter {
        if len(m.AreaIds) > 0 {
            _, o1 := m.AreaIds[p.CommunityId]
            _, o2 := m.AreaIds[p.OrgId]
            if !o1 && !o2 {
                continue
            }
        }

        if m.LastDirection != "" {
            if p.LastDirection != m.LastDirection {
                continue
            }
        }

        if isOlderThanGivenHours(p.LastAppearanceTime, m.DisappearTime) {
            result := &db.ModelTaskResults{
                Title:         m.Task.Name,
                Event:         m.Task.Name,
                ModelID:       m.Task.ModelID,
                ModelTaskID:   m.Task.ID,
                CommunityId:   p.CommunityId,
                OrgID:         p.OrgId,
                ObjectIds:     p.DocumentNumber,
                Location:      p.LastLocation,
                PicDate:       time.Unix(p.LastAppearanceTime, 0).Format("2006-01-02 15:04:05"),
                FirstPersonID: p.DocumentNumber,
            }

            results = append(results, result)
        }
    }

    return service.SaveTaskResults(results)
}

func (m *DisappearModel) Shutdown() error {
    // 清理资源
    fmt.Println("Shutting down Disappear Model")
    return nil
}

func intersectPersonInfo(arr1, arr2 []PersonInfo) []PersonInfo {
    // 使用 map 来存储 arr1 中的 DocumentNumber
    resultMap := make(map[string]PersonInfo)
    var result []PersonInfo

    // 将第一个数组的 DocumentNumber 添加到 map 中
    for _, person := range arr1 {
        resultMap[person.DocumentNumber] = person
    }

    // 遍历第二个数组，检查 DocumentNumber 是否在 map 中
    for _, person := range arr2 {
        if _, exists := resultMap[person.DocumentNumber]; exists {
            // 如果存在交集，将该元素加入结果数组
            result = append(result, person)
            // 可选：如果希望每个交集元素只出现一次，可以删除 map 中的元素
            delete(resultMap, person.DocumentNumber)
        }
    }

    return result
}

func isOlderThanGivenHours(timestamp int64, hours int) bool {
    // 将时间戳转换为时间对象
    timestampTime := time.Unix(timestamp, 0)

    // 获取当前时间
    currentTime := time.Now()

    // 计算当前时间减去指定小时数的时间
    timeThreshold := currentTime.Add(-time.Duration(hours) * time.Hour)

    // 判断时间戳是否早于该时间
    return timestampTime.Before(timeThreshold)
}

 models/gather_model.go

@@ -15,6 +15,7 @@

    "model-engine/config"
    "model-engine/db"
    "model-engine/pkg/logger"
    "model-engine/pkg/set"
    "model-engine/service"
)
@@ -81,10 +82,10 @@
        }
    }

    fmt.Printf("GatherModel init finish ...task id:%s, name:%s, rule:%+v\n", task.ID, task.Name, m)
    logger.Debugf("GatherModel init finish ...task id:%s, name:%s, rule:%+v\n", task.ID, task.Name, m)

    if m.GatherPersons == 0 || m.AppearInterval == 0 || m.DaysWindow == 0 || m.Threshold == 0 {
        fmt.Printf("invalid parameters. task id:%s, name:%s\n", task.ID, task.Name)
        logger.Warnf("invalid parameters. task id:%s, name:%s\n", task.ID, task.Name)
        return errors.New("invalid parameters")
    }


 pkg/mysqlx/mysqlx.go

@@ -1,12 +1,13 @@
package mysqlx

import (
    "time"

    "go.uber.org/zap"
    "gorm.io/driver/mysql"
    "gorm.io/gorm"
    "gorm.io/gorm/logger"
    "gorm.io/gorm/schema"
    "time"
)

type Conf struct {
@@ -29,16 +30,21 @@
        },
        DisableForeignKeyConstraintWhenMigrating: true,
    }
    dbLogger := New(log).LogMode(logger.Info)
    if !conf.LogMode {
        dbLogger = dbLogger.LogMode(logger.Silent)

    if log != nil {
        dbLogger := New(log).LogMode(logger.Info)
        if !conf.LogMode {
            dbLogger = dbLogger.LogMode(logger.Silent)
        }

        gConfig.Logger = dbLogger
    }

    gConfig.Logger = dbLogger
    db, err := gorm.Open(mysql.Open(conf.Dsn), gConfig)
    if err != nil {
        return err
    }

    sqlDb, err := db.DB()
    if err != nil {
        return err

 service/task_results.go

@@ -7,7 +7,7 @@

func SaveTaskResults(results []*db.ModelTaskResults) error {
    for _, v := range results {
        err := db.NewModelTaskResultsSearch().SetPicDate(v.PicDate).SetFirstPersonId(v.FirstPersonID).FirstOrCreate(v)
        err := db.NewModelTaskResultsSearch().SetTaskId(v.ModelTaskID).SetPicDate(v.PicDate).SetFirstPersonId(v.FirstPersonID).FirstOrCreate(v)
        if err != nil {
            logger.Errorf("save task result err:%v", err)
            return err