暂停聚集模型

zhaoqingang

2025-02-19 fca319958029fa924308e50cb61202d7d6ff5008

 models/locationAnalysis.go

@@ -28,6 +28,7 @@
   KeyPersonType  string       // 人员类型
   PersonLabel    string       // 人员标签
   PersonIdentity []string     // 人员身份
   KeyTypes       []string     // 人员身份
   Duration       int          // 时间范围
   Appearances    int          // 出现次数,
   StartTime      int          // 时间范围, 开始时间
@@ -53,16 +54,29 @@
   m.AreaIds = areaIds
   m.Building = task.Building
   m.AlarmType = task.AlarmType
   m.PersonIdentity = []string{"stranger", "visitor", "resident"} //task.IdentityType
   m.KeyTypes = []string{"1", "2", "3", "4", "5"} //task.IdentityType
   m.KeyPersonType = task.PersonType
   m.PersonLabel = task.PersonLabel
   if task.IdentityType != "" {
      m.PersonIdentity = strings.Split(task.IdentityType, ",")
      for _, t := range strings.Split(task.IdentityType, ",") {
         if t == "all" {
            m.PersonIdentity = []string{"stranger", "visitor", "resident"}
            break
         } else {
            m.PersonIdentity = append(m.PersonIdentity, t)
         }
      }
   } else {
      m.PersonIdentity = []string{"stranger", "visitor", "resident"}
   }

   if len(m.AreaIds) == 0 {
      m.AreaIds = m.OrgIds
   if task.PersonType != "" {
      m.KeyTypes = strings.Split(task.PersonType, ",")
   }

   //if len(m.AreaIds) == 0 {
   //   m.AreaIds = m.OrgIds
   //}
   for _, v := range task.Rules {
      if v.Alias == "appearances" {
         if val, ok := v.Value.(float64); ok {
@@ -107,30 +121,23 @@
}

type LocationPersonInfo struct {
   Id             string `json:"id"`
   DocumentNumber string `json:"document_number"`
   PersonType     string `json:"person_type"`
   //CommunityId        string `json:"community_id"`
   //OrgId              string `json:"org_id"`
   //PersonName         string `json:"person_name"`
   //IdCard             string `json:"id_card"`
   LastAppearanceTime int64 `json:"last_appearance_time"`
   //LastDirection      string `json:"last_direction"`
   //LastLocation       string `json:"last_location"`
   Id                 string `json:"id"`
   DocumentNumber     string `json:"document_number"`
   PersonType         string `json:"person_type"`
   LastAppearanceTime int64  `json:"last_appearance_time"`
}

//var (
//   processed        sync.Map                           // 存储已处理记录
//   cleanupThreshold = time.Now().Add(-100 * time.Hour) // 定义一个时间窗口，假设只保存最近100小时的记录
//)

func (m *LocationModel) Run() error {
   // 根据配置的时间段天数, 每天的时间范围内， 重点人员类型或者特定标签人员出现的楼层次数超过阈值

   results := make([]*db.ModelTaskResults, 0)
   var baseFilter, labelFilter, keyFilter, lastFilter []LocationPersonInfo
   var document_number_map map[string]LocationPersonInfo
   var document_number_list []string
   var baseFilter, labelFilter, keyFilter []LocationPersonInfo
   var documentNumberMap = make(map[string]LocationPersonInfo)
   var documentNumberList []string
   communityIds := m.AreaIds
   if len(communityIds) == 0 {
      communityIds = m.OrgIds
   }
   err := db.GetDB().Raw(`
      SELECT
          p.id,
@@ -140,24 +147,24 @@
         snapshot_count_summary AS s
         JOIN person AS p ON p.id = s.document_number
      WHERE
         p.id_card != "" 
         AND (s.community_id IN ? OR s.org_id IN ?)
         (s.community_id IN ? OR s.org_id IN ?)
         AND s.status IN ?
      `, m.AreaIds, m.OrgIds, m.PersonIdentity).Scan(&baseFilter).Error
      `, communityIds, m.OrgIds, m.PersonIdentity).Scan(&baseFilter).Error
   if err != nil {
      logger.Warnf(err.Error())
   }

   if len(baseFilter) == 0 {
      return fmt.Errorf("no results found that match the age condition %s - %s - %s ", m.AreaIds, m.OrgIds, m.PersonIdentity)
      return fmt.Errorf("no results found that match the identity condition %+v", m.Task.Rules)
   }

   logger.Debugf("task %s match age result %d", m.Task.Name, len(baseFilter))
   logger.Debugf("task %s match identity result %d", m.Task.Name, len(baseFilter))
   for _, i := range baseFilter {
      if _, ok := document_number_map[i.DocumentNumber]; !ok {
         document_number_list = append(document_number_list, i.DocumentNumber)
      if _, ok := documentNumberMap[i.DocumentNumber]; !ok {
         documentNumberList = append(documentNumberList, i.DocumentNumber)
         tmp := i
         documentNumberMap[i.DocumentNumber] = tmp
      }
      document_number_map[i.DocumentNumber] = i
   }
   if m.PersonLabel != "" {
      labels := strings.Split(m.PersonLabel, ",")
@@ -170,7 +177,7 @@
      WHERE
         p.id IN ?
         AND   l.label_id IN ?
      `, document_number_list, labels).Scan(&labelFilter).Error
      `, documentNumberList, labels).Scan(&labelFilter).Error
      if err != nil {
         logger.Warnf(err.Error())
      }
@@ -180,17 +187,14 @@
      }

      logger.Debugf("task %s match label result %d", m.Task.Name, len(labelFilter))
      documentNumberList = []string{}
      for _, i := range labelFilter {
         documentNumberList = append(documentNumberList, i.Id)
      }
   }

   document_number_list = []string{}
   for _, i := range labelFilter {

      document_number_list = append(document_number_list, i.Id)

   }

   if m.KeyPersonType != "" {
      keyTypes := strings.Split(m.KeyPersonType, ",")
   if len(m.KeyTypes) > 0 {
      //keyTypes := strings.Split(m.KeyPersonType, ",")
      err := db.GetDB().Raw(`
      SELECT
         p.id,
@@ -201,62 +205,68 @@
      WHERE
         p.id IN ?
         AND k.person_type IN ?
      `, m.StartTime, keyTypes).Scan(&keyFilter).Error
      `, documentNumberList, m.KeyTypes).Scan(&keyFilter).Error
      if err != nil {
         logger.Warnf(err.Error())
      }
      if len(keyFilter) == 0 {
         return fmt.Errorf("no results found that match the key condition %s", m.KeyPersonType)
         return fmt.Errorf("no results found that match the key condition %v------ %s", documentNumberList, m.KeyTypes)
      }

      logger.Debugf("task %s match key person result %d", m.Task.Name, len(keyFilter))
   }

   logger.Debugf("task %s last result %d", m.Task.Name, len(lastFilter))
   document_number_list = []string{}
   //logger.Debugf("task %s last result %d", m.Task.Name, len(lastFilter))
   documentNumberList = []string{}
   for _, i := range keyFilter {
      document_number_list = append(document_number_list, i.Id)
      person := document_number_map[i.DocumentNumber]
      //logger.Debugf("keyFilter %s ------------- %s", i.Id, i.PersonType)
      documentNumberList = append(documentNumberList, i.Id)
      person := documentNumberMap[i.Id]
      person.PersonType = i.PersonType
      document_number_map[i.DocumentNumber] = person
      documentNumberMap[i.Id] = person
   }
   records, err := queryEsLocation(db.GetEsClient(), m, document_number_list)
   //logger.Debugf("document_number_map %s last result %s", m.Task.Name, document_number_map)
   records, err := queryEsLocation(db.GetEsClient(), m, documentNumberList)
   if err != nil {
      return err
   }
   logger.Debugf("records %s last result %d", m.Task.Name, len(records))
   domains, err := domainToLocation(records)
   if err != nil {
      return err
   }
   var tagTypes []string
   var lastAppearanceTime int64
   //var lastAppearanceTime int64
   for _, record := range records {
      tagTypes = []string{}
      for _, personId := range record.DocumentNumbers {
         tagTypes = append(tagTypes, document_number_map[personId].PersonType)
         lastAppearanceTime = document_number_map[personId].LastAppearanceTime
         logger.Debugf("document_number_map[personId] %+v", documentNumberMap[personId])
         tagTypes = append(tagTypes, documentNumberMap[personId].PersonType)
         //lastAppearanceTime = document_number_map[personId].LastAppearanceTime
      }
      //logger.Debugf("tagTypes %s ---------------%s--------", record.PicDate, record.DocumentNumbers)
      _, typeNames, err := service.GetPersonTypeNameByTypes(tagTypes)
      if err != nil {
         return err
      }
      event := strings.Join(typeNames, ",")
      result := &db.ModelTaskResults{
         Title:         m.Task.Name,
         Event:         m.eventFormat(event, record.AppearCount),
         ModelID:       m.Task.ModelID,
         ModelTaskID:   m.Task.ID,
         CommunityId:   record.CommunityId,
         OrgID:         record.OrgId,
         ObjectIds:     strings.Join(record.DocumentNumbers, ","),
         Location:      fmt.Sprintf("%s%s%s", domains[record.CommunityId].Name, record.Building, record.Floor),
         PicDate:       time.Unix(lastAppearanceTime, 0).Format("2006-01-02 15:04:05"),
         Title:       m.Task.Name,
         Event:       m.eventFormat(event, record.AppearCount),
         ModelID:     m.Task.ModelID,
         ModelTaskID: m.Task.ID,
         CommunityId: record.CommunityId,
         OrgID:       record.OrgId,
         ObjectIds:   strings.Join(record.DocumentNumbers, ","),
         Location:    fmt.Sprintf("%s%s%s", domains[record.CommunityId].Name, record.Building, record.Floor),
         Building:    record.Building,
         Floor:       record.Floor,
         //PicDate:     time.Unix(int64(record.PicDate/1000), 0).Format("2006-01-02 15:04:05"),
         PicDate:       record.PicDate,
         FirstPersonID: record.DocumentNumbers[0],
      }
      results = append(results, result)
   }
   logger.Debugf("task %s last filter result %d", m.Task.Name, len(results))
   return service.SaveTaskResults(results)
   return service.SaveLocationTaskResults(results)
}

func (m *LocationModel) KeepAlive() error {
@@ -318,39 +328,26 @@
      }
      filters = append(filters, addrParams)
   }

   //// 重点人员过滤
   //if len(locationModel.KeyPersonType) > 0 {
   //   filters = append(filters, map[string]interface{}{
   //      "terms": map[string]interface{}{
   //         "keyPersonType": strings.Split(locationModel.KeyPersonType, ","),
   //      },
   //   })
   //}

   // 时间范围
   //filters = append(filters, map[string]interface{}{
   //   "range": map[string]interface{}{
   //      "picDate": map[string]interface{}{
   //         "gte": start.Format(time.DateTime),
   //         "lt":  now.Format(time.DateTime),
   //      },
   //   },
   //})
   var queryTimes []map[string]interface{}
   for date := startTime; date.Before(nowTime); date = date.Add(24 * time.Hour) {
      start := time.Date(date.Year(), date.Month(), date.Day(), locationModel.StartTime, 0, 0, 0, date.Location())
      end := time.Date(date.Year(), date.Month(), date.Day(), locationModel.EndTime, 0, 0, 0, date.Location())

      filters = append(filters, map[string]interface{}{
      queryTimes = append(queryTimes, map[string]interface{}{
         "range": map[string]interface{}{
            "picDate": map[string]interface{}{
               "gte": start.Format(time.RFC3339),
               "lte": end.Format(time.RFC3339),
               "gte": start.Format(time.DateTime), // "2006-01-02 15:04:05d"
               "lte": end.Format(time.DateTime),
            },
         },
      })
   }

   filters = append(filters, map[string]interface{}{
      "bool": map[string]interface{}{
         "should": queryTimes,
      },
   })
   //logger.Debugf("filters--------------------------------------- %s", filters)
   query := map[string]interface{}{
      "query": map[string]interface{}{
         "bool": map[string]interface{}{
@@ -401,6 +398,13 @@
                                       "size":  10000,
                                    },
                                 },
                                 "picDate": map[string]interface{}{ // 新增按 picDate 聚合
                                    "date_histogram": map[string]interface{}{
                                       "field":    "picDate",
                                       "interval": "1s", // 按天聚合
                                       //"time_zone": "+08:00", // 设置时区
                                    },
                                 },
                              },
                           },
                        },
@@ -439,29 +443,31 @@
   if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
      return nil, fmt.Errorf("error parsing response body: %s", err)
   }

   //logger.Debugf("result--------------------------------------- %s", result)
   // 解析聚合结果
   var records []*LocationRecord
   if aggs, ok := result["aggregations"].(map[string]interface{}); ok {
      if orgBuckets, ok := aggs["orgs"].(map[string]interface{})["buckets"].([]interface{}); ok {
         for _, orgBucket := range orgBuckets {
            orgId := orgBucket.(map[string]interface{})["key"].(string)

            //logger.Debugf("orgId--------------------------------------- %s", orgId)
            // 解析按communityId的聚合结果
            if communityBuckets, ok := orgBucket.(map[string]interface{})["community"].(map[string]interface{})["buckets"].([]interface{}); ok {
               for _, communityBucket := range communityBuckets {
                  communityId := communityBucket.(map[string]interface{})["key"].(string)

                  //logger.Debugf("communityId--------------------------------------- %s", communityId)
                  // 解析按building的聚合结果
                  if locationBuckets, ok := communityBucket.(map[string]interface{})["location"].(map[string]interface{})["buckets"].([]interface{}); ok {
                     for _, locationBucket := range locationBuckets {
                        building := locationBucket.(map[string]interface{})["key"].(string)

                        //logger.Debugf("building--------------------------------------- %s", building)
                        // 解析按floor的聚合结果
                        if floorBuckets, ok := locationBucket.(map[string]interface{})["floor"].(map[string]interface{})["buckets"].([]interface{}); ok {
                           for _, floorBucket := range floorBuckets {
                              floor := floorBucket.(map[string]interface{})["key"].(string)
                              appearCount := floorBucket.(map[string]interface{})["filter_floor"].(int)
                              //logger.Debugf("floor--------------------------------------- %s -----%s ", floor, floorBucket)
                              appearCount := floorBucket.(map[string]interface{})["doc_count"].(float64)
                              //logger.Debugf("appearCount--------------------------------------- %s- --", appearCount)
                              // 构建 LocationRecord 结构体
                              var persons []string
                              if docNumBuckets, ok := floorBucket.(map[string]interface{})["document_numbers"].(map[string]interface{})["buckets"].([]interface{}); ok {
@@ -469,14 +475,21 @@
                                    persons = append(persons, docNumBucket.(map[string]interface{})["key"].(string))
                                 }
                              }
                              var PicDate string
                              if docNumBuckets, ok := floorBucket.(map[string]interface{})["picDate"].(map[string]interface{})["buckets"].([]interface{}); ok {
                                 for _, docNumBucket := range docNumBuckets {
                                    PicDate = docNumBucket.(map[string]interface{})["key_as_string"].(string)
                                 }
                              }
                              logger.Debugf("floor, persons--,appearCount------------------------------------- %s-- %s --", floor, persons, appearCount)
                              record := &LocationRecord{
                                 //PicDate:        timestamp,
                                 PicDate:         PicDate,
                                 DocumentNumbers: persons,
                                 CommunityId:     communityId,
                                 Building:        building,
                                 Floor:           floor,
                                 OrgId:           orgId,
                                 AppearCount:     appearCount,
                                 AppearCount:     int(appearCount),
                              }

                              records = append(records, record)