| cache/cache.go | ●●●●● 补丁 | 查看 | 原始文档 | blame | 历史 | |
| db/base.go | ●●●●● 补丁 | 查看 | 原始文档 | blame | 历史 | |
| db/person.go | ●●●●● 补丁 | 查看 | 原始文档 | blame | 历史 |
cache/cache.go
@@ -12,12 +12,12 @@ "basic.com/valib/logger.git" ) var querynum = flag.Int("querynum", 3000, "the query number from database") var threadnum = flag.Int("threadnum", 32, "the number of thread to deal data.") var peerSizeNum = flag.Int("peerSize", 3000, "the query number from database") var threadNum = flag.Int("threadNum", 32, "the number of thread to deal data.") var ( Unfiled = "unfiled" PRE_DBTABLE = "dbTable_" Unfilled = "unfilled" PreTableName = "dbTable_" ) type AreaMapItem struct { @@ -26,8 +26,8 @@ } var CaptureDbMap *AreaMapItem var RealNameDbMap = shardmap.New(uint8(*threadnum)) var KeyPersonDbMap = shardmap.New(uint8(*threadnum)) var RealNameDbMap = shardmap.New(uint8(*threadNum)) var KeyPersonDbMap = shardmap.New(uint8(*threadNum)) var doOnce sync.Once @@ -40,7 +40,7 @@ } // 初始化未分类, 没有小区id的档案 CaptureDbMap.Area[Unfiled] = shardmap.New(uint8(*threadnum)) CaptureDbMap.Area[Unfilled] = shardmap.New(uint8(*threadNum)) }) initDbTablePersonsCache() @@ -66,7 +66,7 @@ func initDbTablePersonsCache() { // 缓存底库中的全部人员信息 var dbpApi db.DbPersons total, e := dbpApi.GetPersonTotal("") total, e := dbpApi.GetPersonTotal() // 暂时去掉到访小区过滤 //var psApi db.PersonStatus @@ -74,16 +74,18 @@ logger.Debugf("抓拍档案库共有%d条记录", total) if e == nil && total > 0 { queryEachNum := *querynum qn := int(total) / *threadnum if *querynum < qn { queryEachNum := *peerSizeNum qn := int(total) / *threadNum if *peerSizeNum < qn { queryEachNum = qn } queryT := int(total) / queryEachNum if int(total)%queryEachNum > 0 { queryT++ } temptime := time.Now() startTime := time.Now() var wg sync.WaitGroup for i := 0; i < queryT; i++ { @@ -91,12 +93,13 @@ wg.Add(1) go func(qs int) { defer wg.Done() dbPersons, err := dbpApi.GetPersonsCompareCacheBase(j, queryEachNum) dbPersons, err := dbpApi.GetPersonsCacheBase(j, queryEachNum) if err != nil { logger.Error(err) return } logger.Debugf("eachNum:%d, 获取%d条人员信息", queryEachNum, len(dbPersons)) logger.Debugf("thread:%d, 获取%d条人员信息", queryEachNum, len(dbPersons)) CaptureDbMap.Lock() areaId := "" @@ -104,14 +107,14 @@ areaId = value.AreaId // 没有小区id的人员 if areaId == "" { CaptureDbMap.Area[Unfiled].Set(value.Id, value) CaptureDbMap.Area[Unfiled].Settime() CaptureDbMap.Area[Unfilled].Set(value.Id, value) CaptureDbMap.Area[Unfilled].Settime() continue } //for _, areaId := range accessAreas[value.Id] { if _, ok := CaptureDbMap.Area[areaId]; !ok { CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadnum)) CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadNum)) } CaptureDbMap.Area[areaId].Set(value.Id, value) @@ -124,10 +127,11 @@ }(j) } wg.Wait() logger.Debug("抓拍档案库人员缓存完成用时:", time.Since(temptime)) logger.Debug("抓拍档案库人员缓存完成用时:", time.Since(startTime)) for k, v := range CaptureDbMap.Area { logger.Debugf("Cache area %s item len %d ", k, v.GetLen()) logger.Debugf("Cache area %s items len %d ", k, v.GetLen()) } } } @@ -163,7 +167,7 @@ // UpdateDbPersonsCacheById 更新缓存中的全部人员信息 func UpdateDbPersonsCacheById(id string) { var dbpApi db.DbPersons info, err := dbpApi.GetPersonsCompareCacheById(id) info, err := dbpApi.GetPersonsById(id) if err != nil { logger.Error(err) return @@ -172,7 +176,7 @@ if _, ok := CaptureDbMap.Area[info.AreaId]; !ok { CaptureDbMap.Lock() defer CaptureDbMap.Unlock() CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadnum)) CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadNum)) } CaptureDbMap.Area[info.AreaId].Set(info.Id, info) CaptureDbMap.Area[info.AreaId].Settime() @@ -189,7 +193,7 @@ CaptureDbMap.Lock() defer CaptureDbMap.Unlock() if _, ok := CaptureDbMap.Area[tableId]; !ok { CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadnum)) CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadNum)) } var ei = protomsg.Esinfo{ Id: id, @@ -203,7 +207,7 @@ } func RealTimeDelPersonFromCache(tableId string, id string) { logger.Debug("DelPersonFromCache,tableId:", tableId, ",id:", id) logger.Debug("Delete person from cache, tableId:", tableId, ",id:", id) CaptureDbMap.Lock() defer CaptureDbMap.Unlock() if _, ok := CaptureDbMap.Area[tableId]; ok { @@ -219,7 +223,7 @@ CaptureDbMap.Lock() defer CaptureDbMap.Unlock() if dtM, ok := CaptureDbMap.Area[PRE_DBTABLE]; ok { if dtM, ok := CaptureDbMap.Area[PreTableName]; ok { dtM.Del(tableId) } if _, ok := CaptureDbMap.Area[tableId]; ok { @@ -233,10 +237,10 @@ CaptureDbMap.Lock() defer CaptureDbMap.Unlock() if _, ok := CaptureDbMap.Area[PRE_DBTABLE]; !ok { CaptureDbMap.Area[PRE_DBTABLE] = shardmap.New(uint8(*threadnum)) if _, ok := CaptureDbMap.Area[PreTableName]; !ok { CaptureDbMap.Area[PreTableName] = shardmap.New(uint8(*threadNum)) } CaptureDbMap.Area[PRE_DBTABLE].Set(tableId, enable == 1) CaptureDbMap.Area[PreTableName].Set(tableId, enable == 1) } func UpdateCache(changeMsg *protomsg.EsPersonCacheChange) { db/base.go
@@ -1,12 +1,15 @@ package db import ( "gorm.io/gorm" "time" ) type BaseEntity struct { Id string `gorm:"primary_key;column:id" json:"id" example:""` CreateTime string `gorm:"column:createTime" json:"createTime,omitempty" example:""` UpdateTime string `gorm:"column:updateTime" json:"updateTime,omitempty" example:""` CreateBy string `gorm:"column:createBy" json:"createBy,omitempty" example:""` IsDelete int `gorm:"column:isDelete" json:"isDelete" example:"0 未删除 1已删除"` Enable int `gorm:"column:enable" json:"enable" example:" 1生效 0未生效"` ID string `gorm:"primary_key;column:id;type:varchar(255);" json:"id"` CreatedAt time.Time UpdatedAt time.Time DeletedAt gorm.DeletedAt `gorm:"index" json:"-"` } type FeatureCacheBase struct { @@ -14,5 +17,4 @@ AreaId string TableId string FaceFeature []float32 Enable int32 } db/person.go
@@ -9,48 +9,26 @@ type DbPersons struct { BaseEntity TableId string `gorm:"column:tableId" json:"tableId" example:"库表id"` FaceFeature string `gorm:"column:faceFeature" json:"faceFeature" example:"人脸特征值（车主特征）"` PersonPicUrl string `gorm:"column:personPicUrl" json:"personPicUrl" example:"图片路径,（车主照片）"` PersonName string `gorm:"column:personName" json:"personName" example:"人员姓名，（车主姓名）"` Age string `gorm:"column:age" json:"age" example:"年龄"` Sex string `gorm:"column:sex" json:"sex" example:"性别 男 女（车主性别）"` IdCard string `gorm:"column:idCard" json:"idCard" example:"身份证（车主身份证）"` PhoneNum string `gorm:"column:phoneNum" json:"phoneNum" example:"手机号码"` MonitorLevel string `gorm:"column:monitorLevel" json:"monitorLevel" example:"等级"` PicDesc string `gorm:"column:picDesc" json:"picDesc" example:"照片标识"` Reserved string `gorm:"column:reserved" json:"reserved" example:"其他"` FromServerId string `gorm:"column:fromServerId" json:"fromServerId,omitempty" example:"入库serverId"` ResidentialArea string `gorm:"column:residential_area;type:varchar(255)" json:"residentialArea" example:"小区"` Community string `gorm:"column:community;type:varchar(255)" json:"community" example:"社区"` LastAppearanceTime int64 `gorm:"column:last_appearance_time;type:int;not null;default:0" json:"lastAppearanceTime" example:"123456789"` SnapshotCount int `gorm:"column:snapshot_count;type:varchar(255)" json:"snapshotCount" example:"10" comment:"抓拍次数"` DaysAppeared int `gorm:"column:days_appeared;type:int(11);not null;default:0" json:"daysAppeared" example:"5" comment:"出现天数"` Location string `gorm:"column:location;type:varchar(255)" json:"location" example:"建档地点" comment:"建档地点"` LastLocation string `gorm:"column:last_location;type:varchar(255)" json:"lastLocation" example:"最后出现地点" comment:"最后出现地点"` FaceAngleYaw int `gorm:"column:face_angle_yaw;type:int(11)" json:"faceAngleYaw" example:"15" comment:"人脸角度偏航角"` FaceAngleRoll int `gorm:"column:face_angle_roll;type:int(11)" json:"faceAngleRoll" example:"16" comment:"人脸角度滚转角"` FaceAnglePitch int `gorm:"column:face_angle_pitch;type:int(11)" json:"faceAnglePitch" example:"15" comment:"人脸角度俯仰角"` PersonalStatusName string `gorm:"column:personal_status;type:varchar(31);comment:AI标签" json:"-"` //AI标签 PersonalStatus string `gorm:"-" json:"personalStatus"` //AI标签对应名称 Labels []string `gorm:"-" json:"labels"` //手动添加的标签对应名称 AreaID string `json:"areaID" gorm:"index;column:communityID;type:varchar(299);"` //常驻小区 domain unit ID OrgID string `json:"orgID" gorm:"index;column:org_id;type:varchar(299);"` //常驻派出所 domain unit ID TableId string `gorm:"column:tableId"` FaceFeature string `gorm:"column:face_feature"` CommunityID string `gorm:"column:community_id"` // 常住小区 domain unit ID OrgID string `gorm:"column:org_id"` // 常住派出所 domain unit ID OrgName string `gorm:"column:community"` } func (dbp *DbPersons) GetPersonTotal(tableId string) (total int64, err error) { sql := "select count(1) as total from dbtablepersons where isDelete=0 and tableId in (select id from dbtables where isDelete=0)" if tableId != "" { sql += " and tableId='" + tableId + "'" } func (dbp *DbPersons) GetPersonTotal() (total int64, err error) { sql := "select id from person where is_delete = 0 and tableId = 'system'" err = db.Raw(sql).Count(&total).Error return } func (dbp *DbPersons) GetPersonsCompareCacheBase(from int, size int) (arr []*FeatureCacheBase, err error) { func (dbp *DbPersons) GetPersonsCacheBase(from int, size int) (arr []*FeatureCacheBase, err error) { var persons []DbPersons sql := "select id, tableId, faceFeature, communityID, enable from dbtablepersons where isDelete=0 and tableId in (select id from dbtables where isDelete=0)" sql := "select id, tableId, face_feature, community_id from person where is_delete = 0 and tableId = 'system'" sql += " order by id asc limit " + strconv.Itoa(from) + "," + strconv.Itoa(size) err = db.Raw(sql).Find(&persons).Error if err != nil { return nil, err @@ -60,28 +38,30 @@ if p.FaceFeature != "" { byteFeat, err := base64.StdEncoding.DecodeString(p.FaceFeature) if err != nil { logger.Errorf("Person %s feature is invalid", p.Id) logger.Errorf("Person %s feature is invalid", p.ID) continue } arr = append(arr, &FeatureCacheBase{ Id: p.Id, Id: p.ID, TableId: p.TableId, AreaId: p.AreaID, AreaId: p.CommunityID, FaceFeature: util.ByteSlice2float32Slice(byteFeat), Enable: int32(p.Enable), }) } else { logger.Errorf("Person %s feature is empty %s", p.Id) logger.Warnf("Person %s feature is empty %s", p.ID) } } return } func (dbp *DbPersons) GetPersonsCompareCacheById(id string) (info *FeatureCacheBase, err error) { sql := "select id, tableId, faceFeature, communityID, enable from dbtablepersons where id = \"" + id + "\"" func (dbp *DbPersons) GetPersonsById(id string) (info *FeatureCacheBase, err error) { var p DbPersons err = db.Raw(sql).First(&p).Error err = db.Table("person"). Select("id", "tableId", "face_feature", "community_id"). First(&p, "id = ?", id).Error if err != nil { return nil, err } @@ -92,12 +72,12 @@ return nil, err } info = &FeatureCacheBase{ Id: p.Id, Id: p.ID, TableId: p.TableId, AreaId: p.AreaID, AreaId: p.CommunityID, FaceFeature: util.ByteSlice2float32Slice(byteFeat), Enable: int32(p.Enable), } } return }
