From 250f2e68becab56cc009216496dfa0d7d8d1a22f Mon Sep 17 00:00:00 2001
From: zhangzengfei <zhangzengfei@smartai.com>
Date: 星期日, 07 七月 2024 15:15:20 +0800
Subject: [PATCH] 添加常住人口和重点人员比对
---
compare/capture.go | 75 +++++++++
compare/compare.go | 89 ++--------
serve/serve.go | 4
compare/keyPerson.go | 18 ++
cache/cache.go | 150 +++++++++++-------
db/realPersons.go | 75 +++++++++
main.go | 3
compare/realName.go | 18 ++
8 files changed, 298 insertions(+), 134 deletions(-)
diff --git a/cache/cache.go b/cache/cache.go
index bfaceb8..f0b725c 100644
--- a/cache/cache.go
+++ b/cache/cache.go
@@ -20,44 +20,49 @@
PRE_DBTABLE = "dbTable_"
)
-type CmapItem struct {
+type AreaMapItem struct {
sync.Mutex
Area map[string]*shardmap.ShardMap
}
-var CacheMap *CmapItem
+var CaptureDbMap *AreaMapItem
+var RealNameDbMap = shardmap.New(uint8(*threadnum))
+var KeyPersonDbMap = shardmap.New(uint8(*threadnum))
+
var doOnce sync.Once
-func ReInitDbTablePersonsCache() {
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if CacheMap == nil {
- CacheMap = &CmapItem{
- Area: make(map[string]*shardmap.ShardMap),
- }
- }
- for tableId, _ := range CacheMap.Area {
- delete(CacheMap.Area, tableId)
- }
-
- initDbTablePersonsCache()
-}
-
-func InitDbTablePersons() {
+func InitCache() {
doOnce.Do(func() {
flag.Parse()
- CacheMap = &CmapItem{
+ CaptureDbMap = &AreaMapItem{
Area: make(map[string]*shardmap.ShardMap),
}
// 鍒濆鍖栨湭鍒嗙被, 娌℃湁灏忓尯id鐨勬。妗�
- CacheMap.Area[Unfiled] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[Unfiled] = shardmap.New(uint8(*threadnum))
})
initDbTablePersonsCache()
+ initRealNamePersonsCache()
+ initKeyPersonsCache()
}
+func ReInitDbTablePersonsCache() {
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+
+ if CaptureDbMap == nil {
+ CaptureDbMap = &AreaMapItem{
+ Area: make(map[string]*shardmap.ShardMap),
+ }
+ }
+ for tableId, _ := range CaptureDbMap.Area {
+ delete(CaptureDbMap.Area, tableId)
+ }
+
+ initDbTablePersonsCache()
+}
func initDbTablePersonsCache() {
// 缂撳瓨搴曞簱涓殑鍏ㄩ儴浜哄憳淇℃伅
var dbpApi db.DbPersons
@@ -66,7 +71,7 @@
var psApi db.PersonStatus
accessAreas, _ := psApi.GetPersonAccessedAreas()
- logger.Debugf("鎵�鏈夊簳搴撳叡鏈�%d鏉¤褰�", total)
+ logger.Debugf("鎶撴媿妗ｆ搴撳叡鏈�%d鏉¤褰�", total)
if e == nil && total > 0 {
queryEachNum := *querynum
qn := int(total) / *threadnum
@@ -91,35 +96,63 @@
return
}
logger.Debugf(" eachNum:%d, 鑾峰彇%d鏉′汉鍛樹俊鎭�", queryEachNum, len(dbPersons))
- CacheMap.Lock()
+ CaptureDbMap.Lock()
areaId := ""
for _, value := range dbPersons {
areaId = value.AreaId
// 娌℃湁灏忓尯id鐨勪汉鍛�
if areaId == "" {
- CacheMap.Area[Unfiled].Set(value.Id, value)
- CacheMap.Area[Unfiled].Settime()
+ CaptureDbMap.Area[Unfiled].Set(value.Id, value)
+ CaptureDbMap.Area[Unfiled].Settime()
continue
}
for _, areaId := range accessAreas[value.Id] {
- if _, ok := CacheMap.Area[areaId]; !ok {
- CacheMap.Area[areaId] = shardmap.New(uint8(*threadnum))
+ if _, ok := CaptureDbMap.Area[areaId]; !ok {
+ CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadnum))
}
- CacheMap.Area[areaId].Set(value.Id, value)
- CacheMap.Area[areaId].Settime()
+ CaptureDbMap.Area[areaId].Set(value.Id, value)
+ CaptureDbMap.Area[areaId].Settime()
}
}
- CacheMap.Unlock()
+ CaptureDbMap.Unlock()
}(j)
}
wg.Wait()
- logger.Debug("搴曞簱浜哄憳缂撳瓨瀹屾垚鐢ㄦ椂:", time.Since(temptime))
+ logger.Debug("鎶撴媿妗ｆ搴撲汉鍛樼紦瀛樺畬鎴愮敤鏃�:", time.Since(temptime))
}
+}
+
+func initRealNamePersonsCache() {
+ var dbApi db.Layouts
+ dbPersons, err := dbApi.GetRealNamePersonList()
+ if err != nil {
+ logger.Error("init real-name persons error,", err.Error())
+ }
+
+ for _, value := range dbPersons {
+ RealNameDbMap.Set(value.Id, value)
+ }
+
+ logger.Debugf("甯镐綇浜哄彛鍏辨湁%d鏉¤褰�", len(dbPersons))
+}
+
+func initKeyPersonsCache() {
+ var dbApi db.Layouts
+ dbPersons, err := dbApi.GetKeyPersonList()
+ if err != nil {
+ logger.Error("init real-name persons error,", err.Error())
+ }
+
+ for _, value := range dbPersons {
+ KeyPersonDbMap.Set(value.Id, value)
+ }
+
+ logger.Debugf("閲嶇偣浜哄憳鍏辨湁%d鏉¤褰�", len(dbPersons))
}
// UpdateDbPersonsCacheById 鏇存柊缂撳瓨涓殑鍏ㄩ儴浜哄憳淇℃伅
@@ -131,30 +164,27 @@
return
}
if info != nil && info.AreaId != "" {
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if _, ok := CacheMap.Area[info.AreaId]; !ok {
- CacheMap.Area[info.AreaId] = shardmap.New(uint8(*threadnum))
+ if _, ok := CaptureDbMap.Area[info.AreaId]; !ok {
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+ CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadnum))
}
- CacheMap.Area[info.AreaId].Set(info.Id, info)
- CacheMap.Area[info.AreaId].Settime()
+ CaptureDbMap.Area[info.AreaId].Set(info.Id, info)
+ CaptureDbMap.Area[info.AreaId].Settime()
}
}
func DeleteDbPersonsCacheById(id string) {
- CacheMap.Lock()
- defer CacheMap.Unlock()
-
- for key, _ := range CacheMap.Area {
- CacheMap.Area[key].Del(id)
+ for key, _ := range CaptureDbMap.Area {
+ CaptureDbMap.Area[key].Del(id)
}
}
func RealTimeAddPersonInfoToCache(tableId string, id string, faceFeature string, enable int32, carNo string) {
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if _, ok := CacheMap.Area[tableId]; !ok {
- CacheMap.Area[tableId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+ if _, ok := CaptureDbMap.Area[tableId]; !ok {
+ CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadnum))
}
var ei = protomsg.Esinfo{
Id: id,
@@ -163,16 +193,16 @@
Enable: enable,
CarNo: carNo,
}
- CacheMap.Area[tableId].Set(id, &ei)
+ CaptureDbMap.Area[tableId].Set(id, &ei)
logger.Debug("id:", id, ",tableId:", ",len(faceFeature):", len(faceFeature), ",tableId:", tableId, ",enable:", enable)
}
func RealTimeDelPersonFromCache(tableId string, id string) {
logger.Debug("DelPersonFromCache,tableId:", tableId, ",id:", id)
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if _, ok := CacheMap.Area[tableId]; ok {
- CacheMap.Area[tableId].Del(id)
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+ if _, ok := CaptureDbMap.Area[tableId]; ok {
+ CaptureDbMap.Area[tableId].Del(id)
logger.Debug("DelPerson ok success")
} else {
logger.Error("tableId:", tableId, " not exist")
@@ -181,27 +211,27 @@
func RealTimeDelTable(tableId string) {
logger.Debug("RealTimeDelTable tableId:", tableId)
- CacheMap.Lock()
- defer CacheMap.Unlock()
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
- if dtM, ok := CacheMap.Area[PRE_DBTABLE]; ok {
+ if dtM, ok := CaptureDbMap.Area[PRE_DBTABLE]; ok {
dtM.Del(tableId)
}
- if _, ok := CacheMap.Area[tableId]; ok {
- delete(CacheMap.Area, tableId)
+ if _, ok := CaptureDbMap.Area[tableId]; ok {
+ delete(CaptureDbMap.Area, tableId)
}
}
// 浣垮簳搴撶敓鏁�,灏嗗簳搴撲腑鐨勬墍鏈夌敓鏁堢姸鎬佺殑浜虹壒寰佹坊鍔犲埌缂撳瓨
func RealTimeUpdateTable(tableId string, enable int32) {
logger.Debug("RealTimeUpdateTable tableId:", tableId, ",enable:", enable)
- CacheMap.Lock()
- defer CacheMap.Unlock()
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
- if _, ok := CacheMap.Area[PRE_DBTABLE]; !ok {
- CacheMap.Area[PRE_DBTABLE] = shardmap.New(uint8(*threadnum))
+ if _, ok := CaptureDbMap.Area[PRE_DBTABLE]; !ok {
+ CaptureDbMap.Area[PRE_DBTABLE] = shardmap.New(uint8(*threadnum))
}
- CacheMap.Area[PRE_DBTABLE].Set(tableId, enable == 1)
+ CaptureDbMap.Area[PRE_DBTABLE].Set(tableId, enable == 1)
}
func UpdateCache(changeMsg *protomsg.EsPersonCacheChange) {
diff --git a/compare/capture.go b/compare/capture.go
new file mode 100644
index 0000000..71956f7
--- /dev/null
+++ b/compare/capture.go
@@ -0,0 +1,75 @@
+package compare
+
+import (
+ "sdkCompare/cache"
+
+ "basic.com/pubsub/protomsg.git"
+)
+
+func capturePersonsCompere(args protomsg.CompareArgs, fFeature []float32, baseScore float32) protomsg.SdkCompareResult {
+ var scr protomsg.SdkCompareResult
+ var walkedArea = make(map[string]struct{}, 0)
+
+ // 姣斿浼犲叆鐨勫皬鍖篿d
+ if args.TreeNodes != nil && len(args.TreeNodes) > 0 {
+ for _, id := range args.TreeNodes {
+ if _, ok := cache.CaptureDbMap.Area[id]; !ok {
+ continue
+ }
+
+ targets := cache.CaptureDbMap.Area[id].Walk(DoSdkCompare, fFeature, baseScore)
+ if len(targets) > 0 {
+ // 姣斿缁撴灉鍘婚噸, 鍚屼竴涓汉鍒拌杩囧涓皬鍖�, 缂撳瓨鏁版嵁鍐呬細鏈夊鏉¤褰�
+ for idx, t := range targets {
+ var isRepeat bool
+ for _, r := range scr.CompareResult {
+ if t.Id == r.Id {
+ isRepeat = true
+ break
+ }
+ }
+ if !isRepeat {
+ scr.CompareResult = append(scr.CompareResult, targets[idx])
+ }
+ }
+ }
+
+ walkedArea[id] = struct{}{}
+ }
+
+ if len(scr.CompareResult) > 0 || !args.IsCompareAll {
+ return scr
+ }
+ }
+
+ // 姣斿鍏ㄩ儴灏忓尯
+ if !args.IsCompareAll && len(args.TreeNodes) > 0 {
+ baseScore += 20
+ }
+
+ for key, val := range cache.CaptureDbMap.Area {
+ if _, ok := walkedArea[key]; ok {
+ continue
+ }
+
+ targets := val.Walk(DoSdkCompare, fFeature, baseScore)
+ if len(targets) > 0 {
+ // 姣斿缁撴灉鍘婚噸, 鍚屼竴涓汉鍒拌杩囧涓皬鍖�, 缂撳瓨鏁版嵁鍐呬細鏈夊鏉¤褰�
+ for idx, t := range targets {
+ var isRepeat bool
+ for _, r := range scr.CompareResult {
+ if t.Id == r.Id {
+ isRepeat = true
+ break
+ }
+ }
+ if !isRepeat {
+ scr.CompareResult = append(scr.CompareResult, targets[idx])
+ }
+ }
+ }
+ // todo 娣诲姞灏忓尯澶栫殑鍏宠仈鍏崇郴, 涓嬫浼樺厛姣斿
+ }
+
+ return scr
+}
diff --git a/compare/compare.go b/compare/compare.go
index 59f989a..a1fa2e1 100644
--- a/compare/compare.go
+++ b/compare/compare.go
@@ -2,10 +2,9 @@
import (
"fmt"
- "sdkCompare/util"
"strconv"
- "sdkCompare/cache"
+ "sdkCompare/util"
"basic.com/pubsub/protomsg.git"
"basic.com/valib/logger.git"
@@ -14,12 +13,12 @@
const thresholdLimit = float32(30)
-func GetComparePersonBaseInfo(args protomsg.CompareArgs) []byte {
+func Walk(args protomsg.CompareArgs) []byte {
if args.FaceFeature == nil {
return nil
}
- floatFeat := util.ByteSlice2float32Slice(args.FaceFeature)
+ fFeature := util.ByteSlice2float32Slice(args.FaceFeature)
//鎸囧畾鏈�浣庡垎
baseScore := thresholdLimit
@@ -31,79 +30,27 @@
baseScore = 0
}
- var scResult protomsg.SdkCompareResult
+ var response protomsg.SdkCompareResult
- var walkedArea = make(map[string]struct{}, 0)
- // 浼樺厛姣斿浼犲叆鐨勫皬鍖篿d
- if args.TreeNodes != nil && len(args.TreeNodes) > 0 {
- for _, id := range args.TreeNodes {
- if _, ok := cache.CacheMap.Area[id]; ok {
- targets := cache.CacheMap.Area[id].Walk(DoSdkCompare, floatFeat, baseScore)
- if len(targets) > 0 {
- // 姣斿缁撴灉鍘婚噸, 鍚屼竴涓汉鍒拌杩囧涓皬鍖�, 缂撳瓨鏁版嵁鍐呬細鏈夊鏉¤褰�
- for idx, t := range targets {
- var isRepeat bool
- for _, r := range scResult.CompareResult {
- if t.Id == r.Id {
- isRepeat = true
- break
- }
- }
- if !isRepeat {
- scResult.CompareResult = append(scResult.CompareResult, targets[idx])
- }
- }
- }
-
- walkedArea[id] = struct{}{}
- }
- }
-
- if len(scResult.CompareResult) > 0 {
- goto done
- }
+ // 鎶撴媿搴撴瘮瀵�
+ switch args.CompareTarget {
+ case "realName":
+ response = realNamePersonsCompere(args, fFeature, baseScore)
+ case "keyPerson":
+ response = keyPersonsCompere(args, fFeature, baseScore)
+ default:
+ response = capturePersonsCompere(args, fFeature, baseScore)
}
- // 姣斿浠ュ鐨勫皬鍖�
- if !args.IsCompareAll && len(args.TreeNodes) > 0 {
- baseScore += 20
+ logger.Debugf("Compare Target %s result len %d", args.CompareTarget, len(response.CompareResult))
+
+ if len(response.CompareResult) > 0 {
+ logger.Debugf("Compare result %+v", response.CompareResult)
}
- for key, val := range cache.CacheMap.Area {
- if _, ok := walkedArea[key]; ok {
- continue
- }
-
- targets := val.Walk(DoSdkCompare, floatFeat, baseScore)
- if len(targets) > 0 {
- if len(targets) > 0 {
- // 姣斿缁撴灉鍘婚噸, 鍚屼竴涓汉鍒拌杩囧涓皬鍖�, 缂撳瓨鏁版嵁鍐呬細鏈夊鏉¤褰�
- for idx, t := range targets {
- var isRepeat bool
- for _, r := range scResult.CompareResult {
- if t.Id == r.Id {
- isRepeat = true
- break
- }
- }
- if !isRepeat {
- scResult.CompareResult = append(scResult.CompareResult, targets[idx])
- }
- }
- }
- // todo 娣诲姞灏忓尯澶栫殑鍏宠仈鍏崇郴, 涓嬫浼樺厛姣斿
- }
- }
-
-done:
- logger.Debugf("姣斿缁撴灉 %d鏉�", len(scResult.CompareResult))
- if len(scResult.CompareResult) > 0 {
- logger.Debugf("姣斿缁撴灉%+v", scResult.CompareResult)
- }
-
- buf, err := proto.Marshal(&scResult)
+ buf, err := proto.Marshal(&response)
if err != nil {
- logger.Error("scResult Marshal error!", err)
+ logger.Error("response Marshal error!", err)
return nil
}
diff --git a/compare/keyPerson.go b/compare/keyPerson.go
new file mode 100644
index 0000000..00c761c
--- /dev/null
+++ b/compare/keyPerson.go
@@ -0,0 +1,18 @@
+package compare
+
+import (
+ "basic.com/pubsub/protomsg.git"
+ "sdkCompare/cache"
+)
+
+func keyPersonsCompere(args protomsg.CompareArgs, fFeature []float32, baseScore float32) protomsg.SdkCompareResult {
+ var scr protomsg.SdkCompareResult
+ targets := cache.KeyPersonDbMap.Walk(DoSdkCompare, fFeature, baseScore)
+ if len(targets) > 0 {
+ for idx, _ := range targets {
+ scr.CompareResult = append(scr.CompareResult, targets[idx])
+ }
+ }
+
+ return scr
+}
diff --git a/compare/realName.go b/compare/realName.go
new file mode 100644
index 0000000..4753655
--- /dev/null
+++ b/compare/realName.go
@@ -0,0 +1,18 @@
+package compare
+
+import (
+ "basic.com/pubsub/protomsg.git"
+ "sdkCompare/cache"
+)
+
+func realNamePersonsCompere(args protomsg.CompareArgs, fFeature []float32, baseScore float32) protomsg.SdkCompareResult {
+ var scr protomsg.SdkCompareResult
+ targets := cache.RealNameDbMap.Walk(DoSdkCompare, fFeature, baseScore)
+ if len(targets) > 0 {
+ for idx, _ := range targets {
+ scr.CompareResult = append(scr.CompareResult, targets[idx])
+ }
+ }
+
+ return scr
+}
diff --git a/db/realPersons.go b/db/realPersons.go
new file mode 100644
index 0000000..8bfee85
--- /dev/null
+++ b/db/realPersons.go
@@ -0,0 +1,75 @@
+package db
+
+import (
+ "encoding/base64"
+ "sdkCompare/util"
+)
+
+type Layouts struct {
+ Id string `gorm:"column:id"`
+ TableId string `gorm:"column:tableId"`
+ Name string `gorm:"column:name"`
+ Phone string `gorm:"column:phone"`
+ Address string `gorm:"column:address"`
+ Age string `gorm:"column:age"`
+ Gender string `gorm:"column:gender"`
+ IdCard string `gorm:"column:id_card"`
+ PersonType string `gorm:"column:person_type"` //'1锛氬父椹讳汉鍙ｏ紝2锛氬墠绉戜汉鍛橈紝3锛氭秹姣掍汉鍛橈紝4锛氭秹绋充汉鍛橈紱5锛氬湪閫冧汉鍛橈紝6:娴佸姩浜哄彛锛�7:瀵勪綇浜哄彛锛�',
+ Feature string `gorm:"column:feature"`
+}
+
+func (l *Layouts) TableName() string {
+ return "layouts"
+}
+
+func (l *Layouts) GetRealNamePersonList() (arr []*FeatureCacheBase, err error) {
+ var persons []Layouts
+ sql := "select id, tableId, person_type, feature from layouts where person_type = '1'"
+ err = db.Raw(sql).Find(&persons).Error
+ if err != nil {
+ return nil, nil
+ }
+
+ for _, p := range persons {
+ if p.Feature != "" {
+ byteFeat, err := base64.StdEncoding.DecodeString(p.Feature)
+ if err != nil {
+ continue
+ }
+
+ arr = append(arr, &FeatureCacheBase{
+ Id: p.Id,
+ TableId: p.TableId,
+ FaceFeature: util.ByteSlice2float32Slice(byteFeat),
+ })
+ }
+ }
+
+ return
+}
+
+func (l *Layouts) GetKeyPersonList() (arr []*FeatureCacheBase, err error) {
+ var persons []Layouts
+ sql := "select id, id_card, person_type, feature from layouts where person_type != '1'"
+ err = db.Raw(sql).Find(&persons).Error
+ if err != nil {
+ return nil, nil
+ }
+
+ for _, p := range persons {
+ if p.Feature != "" {
+ byteFeat, err := base64.StdEncoding.DecodeString(p.Feature)
+ if err != nil {
+ continue
+ }
+
+ arr = append(arr, &FeatureCacheBase{
+ Id: p.Id,
+ TableId: p.IdCard,
+ FaceFeature: util.ByteSlice2float32Slice(byteFeat),
+ })
+ }
+ }
+
+ return
+}
diff --git a/main.go b/main.go
index b735628..580fc7c 100644
--- a/main.go
+++ b/main.go
@@ -42,7 +42,7 @@
return
}
- cache.InitDbTablePersons()
+ cache.InitCache()
serveUrl := "tcp://0.0.0.0:"
serveUrl = serveUrl + strconv.Itoa(config.MainConf.ServePort)
@@ -50,7 +50,6 @@
logger.Infof("%s serve url:%s", procName, serveUrl)
var ctx, cancel = context.WithCancel(context.Background())
-
serve.Start(ctx, serveUrl, config.MainConf.WorkerNum)
quit := make(chan os.Signal)
diff --git a/serve/serve.go b/serve/serve.go
index d3584d7..031c02c 100644
--- a/serve/serve.go
+++ b/serve/serve.go
@@ -41,7 +41,7 @@
var compareArgInfo protomsg.CompareArgs
if err = proto.Unmarshal(request.Payload, &compareArgInfo); err == nil {
timeStart := time.Now()
- result = compare.GetComparePersonBaseInfo(compareArgInfo)
+ result = compare.Walk(compareArgInfo)
logger.Debug("鐢ㄦ椂锛�", time.Since(timeStart))
} else {
logger.Warn("CompareArgs or EsPersonCacheChange json unmarshal error")
@@ -83,10 +83,12 @@
logger.Error("can't get new rep socket: %s", err)
return
}
+
if err = sock.SetOption(mangos.OptionRaw, true); err != nil {
logger.Error("can't set raw mode: %s", err)
return
}
+
sock.AddTransport(ipc.NewTransport())
sock.AddTransport(tcp.NewTransport())
if err = sock.Listen(url); err != nil {
--
Gitblit v1.8.0