From 366e2ff546092d9be26411fb698b3ddd8e834a11 Mon Sep 17 00:00:00 2001
From: zhangzengfei <zhangzengfei@smartai.com>
Date: 星期一, 09 十二月 2024 15:38:02 +0800
Subject: [PATCH] 更新实名比对
---
cache/cache.go | 95 ++++++++++++++++++++++-------------------------
1 files changed, 44 insertions(+), 51 deletions(-)
diff --git a/cache/cache.go b/cache/cache.go
index f0b725c..09ef7e6 100644
--- a/cache/cache.go
+++ b/cache/cache.go
@@ -1,6 +1,7 @@
package cache
import (
+ "basic.com/pubsub/protomsg.git"
"flag"
"sync"
"time"
@@ -8,16 +9,15 @@
"sdkCompare/cache/shardmap"
"sdkCompare/db"
- "basic.com/pubsub/protomsg.git"
"basic.com/valib/logger.git"
)
-var querynum = flag.Int("querynum", 3000, "the query number from database")
-var threadnum = flag.Int("threadnum", 32, "the number of thread to deal data.")
+var peerSizeNum = flag.Int("peerSize", 3000, "the query number from database")
+var threadNum = flag.Int("threadNum", 32, "the number of thread to deal data.")
var (
- Unfiled = "unfiled"
- PRE_DBTABLE = "dbTable_"
+ Unfilled = "unfilled"
+ PreTableName = "dbTable_"
)
type AreaMapItem struct {
@@ -26,8 +26,7 @@
}
var CaptureDbMap *AreaMapItem
-var RealNameDbMap = shardmap.New(uint8(*threadnum))
-var KeyPersonDbMap = shardmap.New(uint8(*threadnum))
+var RealNameDbMap = shardmap.New(uint8(*threadNum))
var doOnce sync.Once
@@ -40,12 +39,11 @@
}
// 鍒濆鍖栨湭鍒嗙被, 娌℃湁灏忓尯id鐨勬。妗�
- CaptureDbMap.Area[Unfiled] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[Unfilled] = shardmap.New(uint8(*threadNum))
})
initDbTablePersonsCache()
initRealNamePersonsCache()
- initKeyPersonsCache()
}
func ReInitDbTablePersonsCache() {
@@ -66,23 +64,26 @@
func initDbTablePersonsCache() {
// 缂撳瓨搴曞簱涓殑鍏ㄩ儴浜哄憳淇℃伅
var dbpApi db.DbPersons
- total, e := dbpApi.GetPersonTotal("")
+ total, e := dbpApi.GetPersonTotal()
- var psApi db.PersonStatus
- accessAreas, _ := psApi.GetPersonAccessedAreas()
+ // 鏆傛椂鍘绘帀鍒拌灏忓尯杩囨护
+ //var psApi db.PersonStatus
+ //accessAreas, _ := psApi.GetPersonAccessedAreas()
logger.Debugf("鎶撴媿妗ｆ搴撳叡鏈�%d鏉¤褰�", total)
if e == nil && total > 0 {
- queryEachNum := *querynum
- qn := int(total) / *threadnum
- if *querynum < qn {
+ queryEachNum := *peerSizeNum
+ qn := int(total) / *threadNum
+ if *peerSizeNum < qn {
queryEachNum = qn
}
+
queryT := int(total) / queryEachNum
if int(total)%queryEachNum > 0 {
queryT++
}
- temptime := time.Now()
+
+ startTime := time.Now()
var wg sync.WaitGroup
for i := 0; i < queryT; i++ {
@@ -90,12 +91,13 @@
wg.Add(1)
go func(qs int) {
defer wg.Done()
- dbPersons, err := dbpApi.GetPersonsCompareCacheBase(j, queryEachNum)
+ dbPersons, err := dbpApi.GetPersonsCacheBase(j, queryEachNum)
if err != nil {
logger.Error(err)
return
}
- logger.Debugf(" eachNum:%d, 鑾峰彇%d鏉′汉鍛樹俊鎭�", queryEachNum, len(dbPersons))
+
+ logger.Debugf("thread:%d, 鑾峰彇%d鏉′汉鍛樹俊鎭�", queryEachNum, len(dbPersons))
CaptureDbMap.Lock()
areaId := ""
@@ -103,19 +105,19 @@
areaId = value.AreaId
// 娌℃湁灏忓尯id鐨勪汉鍛�
if areaId == "" {
- CaptureDbMap.Area[Unfiled].Set(value.Id, value)
- CaptureDbMap.Area[Unfiled].Settime()
+ CaptureDbMap.Area[Unfilled].Set(value.Id, value)
+ CaptureDbMap.Area[Unfilled].Settime()
continue
}
- for _, areaId := range accessAreas[value.Id] {
- if _, ok := CaptureDbMap.Area[areaId]; !ok {
- CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadnum))
- }
-
- CaptureDbMap.Area[areaId].Set(value.Id, value)
- CaptureDbMap.Area[areaId].Settime()
+ //for _, areaId := range accessAreas[value.Id] {
+ if _, ok := CaptureDbMap.Area[areaId]; !ok {
+ CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadNum))
}
+
+ CaptureDbMap.Area[areaId].Set(value.Id, value)
+ CaptureDbMap.Area[areaId].Settime()
+ //}
}
CaptureDbMap.Unlock()
@@ -123,12 +125,17 @@
}(j)
}
wg.Wait()
- logger.Debug("鎶撴媿妗ｆ搴撲汉鍛樼紦瀛樺畬鎴愮敤鏃�:", time.Since(temptime))
+
+ logger.Debug("鎶撴媿妗ｆ搴撲汉鍛樼紦瀛樺畬鎴愮敤鏃�:", time.Since(startTime))
+
+ for k, v := range CaptureDbMap.Area {
+ logger.Debugf("Cache area %s items len %d ", k, v.GetLen())
+ }
}
}
func initRealNamePersonsCache() {
- var dbApi db.Layouts
+ var dbApi db.RealNamePerson
dbPersons, err := dbApi.GetRealNamePersonList()
if err != nil {
logger.Error("init real-name persons error,", err.Error())
@@ -141,24 +148,10 @@
logger.Debugf("甯镐綇浜哄彛鍏辨湁%d鏉¤褰�", len(dbPersons))
}
-func initKeyPersonsCache() {
- var dbApi db.Layouts
- dbPersons, err := dbApi.GetKeyPersonList()
- if err != nil {
- logger.Error("init real-name persons error,", err.Error())
- }
-
- for _, value := range dbPersons {
- KeyPersonDbMap.Set(value.Id, value)
- }
-
- logger.Debugf("閲嶇偣浜哄憳鍏辨湁%d鏉¤褰�", len(dbPersons))
-}
-
// UpdateDbPersonsCacheById 鏇存柊缂撳瓨涓殑鍏ㄩ儴浜哄憳淇℃伅
func UpdateDbPersonsCacheById(id string) {
var dbpApi db.DbPersons
- info, err := dbpApi.GetPersonsCompareCacheById(id)
+ info, err := dbpApi.GetPersonsById(id)
if err != nil {
logger.Error(err)
return
@@ -167,7 +160,7 @@
if _, ok := CaptureDbMap.Area[info.AreaId]; !ok {
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
- CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadNum))
}
CaptureDbMap.Area[info.AreaId].Set(info.Id, info)
CaptureDbMap.Area[info.AreaId].Settime()
@@ -184,7 +177,7 @@
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
if _, ok := CaptureDbMap.Area[tableId]; !ok {
- CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadNum))
}
var ei = protomsg.Esinfo{
Id: id,
@@ -198,7 +191,7 @@
}
func RealTimeDelPersonFromCache(tableId string, id string) {
- logger.Debug("DelPersonFromCache,tableId:", tableId, ",id:", id)
+ logger.Debug("Delete person from cache, tableId:", tableId, ",id:", id)
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
if _, ok := CaptureDbMap.Area[tableId]; ok {
@@ -214,7 +207,7 @@
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
- if dtM, ok := CaptureDbMap.Area[PRE_DBTABLE]; ok {
+ if dtM, ok := CaptureDbMap.Area[PreTableName]; ok {
dtM.Del(tableId)
}
if _, ok := CaptureDbMap.Area[tableId]; ok {
@@ -228,10 +221,10 @@
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
- if _, ok := CaptureDbMap.Area[PRE_DBTABLE]; !ok {
- CaptureDbMap.Area[PRE_DBTABLE] = shardmap.New(uint8(*threadnum))
+ if _, ok := CaptureDbMap.Area[PreTableName]; !ok {
+ CaptureDbMap.Area[PreTableName] = shardmap.New(uint8(*threadNum))
}
- CaptureDbMap.Area[PRE_DBTABLE].Set(tableId, enable == 1)
+ CaptureDbMap.Area[PreTableName].Set(tableId, enable == 1)
}
func UpdateCache(changeMsg *protomsg.EsPersonCacheChange) {
--
Gitblit v1.8.0