From 366e2ff546092d9be26411fb698b3ddd8e834a11 Mon Sep 17 00:00:00 2001
From: zhangzengfei <zhangzengfei@smartai.com>
Date: 星期一, 09 十二月 2024 15:38:02 +0800
Subject: [PATCH] 更新实名比对
---
cache/cache.go | 182 ++++++++++++++++++++++++++------------------
1 files changed, 107 insertions(+), 75 deletions(-)
diff --git a/cache/cache.go b/cache/cache.go
index aee3708..09ef7e6 100644
--- a/cache/cache.go
+++ b/cache/cache.go
@@ -1,6 +1,7 @@
package cache
import (
+ "basic.com/pubsub/protomsg.git"
"flag"
"sync"
"time"
@@ -8,76 +9,81 @@
"sdkCompare/cache/shardmap"
"sdkCompare/db"
- "basic.com/pubsub/protomsg.git"
"basic.com/valib/logger.git"
)
-var querynum = flag.Int("querynum", 3000, "the query number from database")
-var threadnum = flag.Int("threadnum", 32, "the number of thread to deal data.")
+var peerSizeNum = flag.Int("peerSize", 3000, "the query number from database")
+var threadNum = flag.Int("threadNum", 32, "the number of thread to deal data.")
var (
- Unfiled = "unfiled"
- PRE_DBTABLE = "dbTable_"
+ Unfilled = "unfilled"
+ PreTableName = "dbTable_"
)
-type CmapItem struct {
+type AreaMapItem struct {
sync.Mutex
Area map[string]*shardmap.ShardMap
}
-var CacheMap *CmapItem
+var CaptureDbMap *AreaMapItem
+var RealNameDbMap = shardmap.New(uint8(*threadNum))
+
var doOnce sync.Once
-func ReInitDbTablePersonsCache() {
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if CacheMap == nil {
- CacheMap = &CmapItem{
- Area: make(map[string]*shardmap.ShardMap),
- }
- }
- for tableId, _ := range CacheMap.Area {
- delete(CacheMap.Area, tableId)
- }
-
- initDbTablePersonsCache()
-}
-
-func InitDbTablePersons() {
+func InitCache() {
doOnce.Do(func() {
flag.Parse()
- CacheMap = &CmapItem{
+ CaptureDbMap = &AreaMapItem{
Area: make(map[string]*shardmap.ShardMap),
}
// 鍒濆鍖栨湭鍒嗙被, 娌℃湁灏忓尯id鐨勬。妗�
- CacheMap.Area[Unfiled] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[Unfilled] = shardmap.New(uint8(*threadNum))
})
initDbTablePersonsCache()
+ initRealNamePersonsCache()
}
+func ReInitDbTablePersonsCache() {
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+
+ if CaptureDbMap == nil {
+ CaptureDbMap = &AreaMapItem{
+ Area: make(map[string]*shardmap.ShardMap),
+ }
+ }
+ for tableId, _ := range CaptureDbMap.Area {
+ delete(CaptureDbMap.Area, tableId)
+ }
+
+ initDbTablePersonsCache()
+}
func initDbTablePersonsCache() {
// 缂撳瓨搴曞簱涓殑鍏ㄩ儴浜哄憳淇℃伅
var dbpApi db.DbPersons
- total, e := dbpApi.GetPersonTotal("")
+ total, e := dbpApi.GetPersonTotal()
- var psApi db.PersonStatus
- accessAreas, _ := psApi.GetPersonAccessedAreas()
+ // 鏆傛椂鍘绘帀鍒拌灏忓尯杩囨护
+ //var psApi db.PersonStatus
+ //accessAreas, _ := psApi.GetPersonAccessedAreas()
- logger.Debugf("鎵�鏈夊簳搴撳叡鏈�%d鏉¤褰�", total)
+ logger.Debugf("鎶撴媿妗ｆ搴撳叡鏈�%d鏉¤褰�", total)
if e == nil && total > 0 {
- queryEachNum := *querynum
- qn := int(total) / *threadnum
- if *querynum < qn {
+ queryEachNum := *peerSizeNum
+ qn := int(total) / *threadNum
+ if *peerSizeNum < qn {
queryEachNum = qn
}
+
queryT := int(total) / queryEachNum
if int(total)%queryEachNum > 0 {
queryT++
}
- temptime := time.Now()
+
+ startTime := time.Now()
var wg sync.WaitGroup
for i := 0; i < queryT; i++ {
@@ -85,67 +91,93 @@
wg.Add(1)
go func(qs int) {
defer wg.Done()
- dbPersons, err := dbpApi.GetPersonsCompareCacheBase(j, queryEachNum)
+ dbPersons, err := dbpApi.GetPersonsCacheBase(j, queryEachNum)
if err != nil {
logger.Error(err)
return
}
- logger.Debugf("鑾峰彇%d鏉′汉鍛樹俊鎭�", len(dbPersons))
- CacheMap.Lock()
+
+ logger.Debugf("thread:%d, 鑾峰彇%d鏉′汉鍛樹俊鎭�", queryEachNum, len(dbPersons))
+ CaptureDbMap.Lock()
areaId := ""
for _, value := range dbPersons {
areaId = value.AreaId
// 娌℃湁灏忓尯id鐨勪汉鍛�
if areaId == "" {
- CacheMap.Area[Unfiled].Set(value.Id, value)
- CacheMap.Area[Unfiled].Settime()
+ CaptureDbMap.Area[Unfilled].Set(value.Id, value)
+ CaptureDbMap.Area[Unfilled].Settime()
continue
}
- for _, areaId := range accessAreas[value.Id] {
- if _, ok := CacheMap.Area[areaId]; !ok {
- CacheMap.Area[areaId] = shardmap.New(uint8(*threadnum))
- }
-
- CacheMap.Area[areaId].Set(value.Id, value)
- CacheMap.Area[areaId].Settime()
+ //for _, areaId := range accessAreas[value.Id] {
+ if _, ok := CaptureDbMap.Area[areaId]; !ok {
+ CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadNum))
}
+
+ CaptureDbMap.Area[areaId].Set(value.Id, value)
+ CaptureDbMap.Area[areaId].Settime()
+ //}
}
- CacheMap.Unlock()
+ CaptureDbMap.Unlock()
}(j)
}
wg.Wait()
- logger.Debug("搴曞簱浜哄憳缂撳瓨瀹屾垚鐢ㄦ椂:", time.Since(temptime))
+
+ logger.Debug("鎶撴媿妗ｆ搴撲汉鍛樼紦瀛樺畬鎴愮敤鏃�:", time.Since(startTime))
+
+ for k, v := range CaptureDbMap.Area {
+ logger.Debugf("Cache area %s items len %d ", k, v.GetLen())
+ }
}
+}
+
+func initRealNamePersonsCache() {
+ var dbApi db.RealNamePerson
+ dbPersons, err := dbApi.GetRealNamePersonList()
+ if err != nil {
+ logger.Error("init real-name persons error,", err.Error())
+ }
+
+ for _, value := range dbPersons {
+ RealNameDbMap.Set(value.Id, value)
+ }
+
+ logger.Debugf("甯镐綇浜哄彛鍏辨湁%d鏉¤褰�", len(dbPersons))
}
// UpdateDbPersonsCacheById 鏇存柊缂撳瓨涓殑鍏ㄩ儴浜哄憳淇℃伅
func UpdateDbPersonsCacheById(id string) {
var dbpApi db.DbPersons
- info, err := dbpApi.GetPersonsCompareCacheById(id)
+ info, err := dbpApi.GetPersonsById(id)
if err != nil {
logger.Error(err)
return
}
- if info.AreaId != "" {
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if _, ok := CacheMap.Area[info.AreaId]; !ok {
- CacheMap.Area[info.AreaId] = shardmap.New(uint8(*threadnum))
+ if info != nil && info.AreaId != "" {
+ if _, ok := CaptureDbMap.Area[info.AreaId]; !ok {
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+ CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadNum))
}
- CacheMap.Area[info.AreaId].Set(info.Id, info)
- CacheMap.Area[info.AreaId].Settime()
+ CaptureDbMap.Area[info.AreaId].Set(info.Id, info)
+ CaptureDbMap.Area[info.AreaId].Settime()
+ }
+}
+
+func DeleteDbPersonsCacheById(id string) {
+ for key, _ := range CaptureDbMap.Area {
+ CaptureDbMap.Area[key].Del(id)
}
}
func RealTimeAddPersonInfoToCache(tableId string, id string, faceFeature string, enable int32, carNo string) {
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if _, ok := CacheMap.Area[tableId]; !ok {
- CacheMap.Area[tableId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+ if _, ok := CaptureDbMap.Area[tableId]; !ok {
+ CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadNum))
}
var ei = protomsg.Esinfo{
Id: id,
@@ -154,16 +186,16 @@
Enable: enable,
CarNo: carNo,
}
- CacheMap.Area[tableId].Set(id, &ei)
+ CaptureDbMap.Area[tableId].Set(id, &ei)
logger.Debug("id:", id, ",tableId:", ",len(faceFeature):", len(faceFeature), ",tableId:", tableId, ",enable:", enable)
}
func RealTimeDelPersonFromCache(tableId string, id string) {
- logger.Debug("DelPersonFromCache,tableId:", tableId, ",id:", id)
- CacheMap.Lock()
- defer CacheMap.Unlock()
- if _, ok := CacheMap.Area[tableId]; ok {
- CacheMap.Area[tableId].Del(id)
+ logger.Debug("Delete person from cache, tableId:", tableId, ",id:", id)
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
+ if _, ok := CaptureDbMap.Area[tableId]; ok {
+ CaptureDbMap.Area[tableId].Del(id)
logger.Debug("DelPerson ok success")
} else {
logger.Error("tableId:", tableId, " not exist")
@@ -172,27 +204,27 @@
func RealTimeDelTable(tableId string) {
logger.Debug("RealTimeDelTable tableId:", tableId)
- CacheMap.Lock()
- defer CacheMap.Unlock()
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
- if dtM, ok := CacheMap.Area[PRE_DBTABLE]; ok {
+ if dtM, ok := CaptureDbMap.Area[PreTableName]; ok {
dtM.Del(tableId)
}
- if _, ok := CacheMap.Area[tableId]; ok {
- delete(CacheMap.Area, tableId)
+ if _, ok := CaptureDbMap.Area[tableId]; ok {
+ delete(CaptureDbMap.Area, tableId)
}
}
// 浣垮簳搴撶敓鏁�,灏嗗簳搴撲腑鐨勬墍鏈夌敓鏁堢姸鎬佺殑浜虹壒寰佹坊鍔犲埌缂撳瓨
func RealTimeUpdateTable(tableId string, enable int32) {
logger.Debug("RealTimeUpdateTable tableId:", tableId, ",enable:", enable)
- CacheMap.Lock()
- defer CacheMap.Unlock()
+ CaptureDbMap.Lock()
+ defer CaptureDbMap.Unlock()
- if _, ok := CacheMap.Area[PRE_DBTABLE]; !ok {
- CacheMap.Area[PRE_DBTABLE] = shardmap.New(uint8(*threadnum))
+ if _, ok := CaptureDbMap.Area[PreTableName]; !ok {
+ CaptureDbMap.Area[PreTableName] = shardmap.New(uint8(*threadNum))
}
- CacheMap.Area[PRE_DBTABLE].Set(tableId, enable == 1)
+ CaptureDbMap.Area[PreTableName].Set(tableId, enable == 1)
}
func UpdateCache(changeMsg *protomsg.EsPersonCacheChange) {
--
Gitblit v1.8.0