From 1e180d22fdb13399e9caf31da97a1b5554123102 Mon Sep 17 00:00:00 2001
From: zhangzengfei <zhangzengfei@smartai.com>
Date: 星期四, 26 九月 2024 21:09:27 +0800
Subject: [PATCH] fix db
---
cache/cache.go | 58 +++++++++++++++++++++++++++++++---------------------------
1 files changed, 31 insertions(+), 27 deletions(-)
diff --git a/cache/cache.go b/cache/cache.go
index 91dd9df..72c8d62 100644
--- a/cache/cache.go
+++ b/cache/cache.go
@@ -12,12 +12,12 @@
"basic.com/valib/logger.git"
)
-var querynum = flag.Int("querynum", 3000, "the query number from database")
-var threadnum = flag.Int("threadnum", 32, "the number of thread to deal data.")
+var peerSizeNum = flag.Int("peerSize", 3000, "the query number from database")
+var threadNum = flag.Int("threadNum", 32, "the number of thread to deal data.")
var (
- Unfiled = "unfiled"
- PRE_DBTABLE = "dbTable_"
+ Unfilled = "unfilled"
+ PreTableName = "dbTable_"
)
type AreaMapItem struct {
@@ -26,8 +26,8 @@
}
var CaptureDbMap *AreaMapItem
-var RealNameDbMap = shardmap.New(uint8(*threadnum))
-var KeyPersonDbMap = shardmap.New(uint8(*threadnum))
+var RealNameDbMap = shardmap.New(uint8(*threadNum))
+var KeyPersonDbMap = shardmap.New(uint8(*threadNum))
var doOnce sync.Once
@@ -40,7 +40,7 @@
}
// 鍒濆鍖栨湭鍒嗙被, 娌℃湁灏忓尯id鐨勬。妗�
- CaptureDbMap.Area[Unfiled] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[Unfilled] = shardmap.New(uint8(*threadNum))
})
initDbTablePersonsCache()
@@ -66,7 +66,7 @@
func initDbTablePersonsCache() {
// 缂撳瓨搴曞簱涓殑鍏ㄩ儴浜哄憳淇℃伅
var dbpApi db.DbPersons
- total, e := dbpApi.GetPersonTotal("")
+ total, e := dbpApi.GetPersonTotal()
// 鏆傛椂鍘绘帀鍒拌灏忓尯杩囨护
//var psApi db.PersonStatus
@@ -74,16 +74,18 @@
logger.Debugf("鎶撴媿妗ｆ搴撳叡鏈�%d鏉¤褰�", total)
if e == nil && total > 0 {
- queryEachNum := *querynum
- qn := int(total) / *threadnum
- if *querynum < qn {
+ queryEachNum := *peerSizeNum
+ qn := int(total) / *threadNum
+ if *peerSizeNum < qn {
queryEachNum = qn
}
+
queryT := int(total) / queryEachNum
if int(total)%queryEachNum > 0 {
queryT++
}
- temptime := time.Now()
+
+ startTime := time.Now()
var wg sync.WaitGroup
for i := 0; i < queryT; i++ {
@@ -91,12 +93,13 @@
wg.Add(1)
go func(qs int) {
defer wg.Done()
- dbPersons, err := dbpApi.GetPersonsCompareCacheBase(j, queryEachNum)
+ dbPersons, err := dbpApi.GetPersonsCacheBase(j, queryEachNum)
if err != nil {
logger.Error(err)
return
}
- logger.Debugf("eachNum:%d, 鑾峰彇%d鏉′汉鍛樹俊鎭�", queryEachNum, len(dbPersons))
+
+ logger.Debugf("thread:%d, 鑾峰彇%d鏉′汉鍛樹俊鎭�", queryEachNum, len(dbPersons))
CaptureDbMap.Lock()
areaId := ""
@@ -104,14 +107,14 @@
areaId = value.AreaId
// 娌℃湁灏忓尯id鐨勪汉鍛�
if areaId == "" {
- CaptureDbMap.Area[Unfiled].Set(value.Id, value)
- CaptureDbMap.Area[Unfiled].Settime()
+ CaptureDbMap.Area[Unfilled].Set(value.Id, value)
+ CaptureDbMap.Area[Unfilled].Settime()
continue
}
//for _, areaId := range accessAreas[value.Id] {
if _, ok := CaptureDbMap.Area[areaId]; !ok {
- CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[areaId] = shardmap.New(uint8(*threadNum))
}
CaptureDbMap.Area[areaId].Set(value.Id, value)
@@ -124,10 +127,11 @@
}(j)
}
wg.Wait()
- logger.Debug("鎶撴媿妗ｆ搴撲汉鍛樼紦瀛樺畬鎴愮敤鏃�:", time.Since(temptime))
+
+ logger.Debug("鎶撴媿妗ｆ搴撲汉鍛樼紦瀛樺畬鎴愮敤鏃�:", time.Since(startTime))
for k, v := range CaptureDbMap.Area {
- logger.Debugf("Cache area %s item len %d ", k, v.GetLen())
+ logger.Debugf("Cache area %s items len %d ", k, v.GetLen())
}
}
}
@@ -163,7 +167,7 @@
// UpdateDbPersonsCacheById 鏇存柊缂撳瓨涓殑鍏ㄩ儴浜哄憳淇℃伅
func UpdateDbPersonsCacheById(id string) {
var dbpApi db.DbPersons
- info, err := dbpApi.GetPersonsCompareCacheById(id)
+ info, err := dbpApi.GetPersonsById(id)
if err != nil {
logger.Error(err)
return
@@ -172,7 +176,7 @@
if _, ok := CaptureDbMap.Area[info.AreaId]; !ok {
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
- CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[info.AreaId] = shardmap.New(uint8(*threadNum))
}
CaptureDbMap.Area[info.AreaId].Set(info.Id, info)
CaptureDbMap.Area[info.AreaId].Settime()
@@ -189,7 +193,7 @@
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
if _, ok := CaptureDbMap.Area[tableId]; !ok {
- CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadnum))
+ CaptureDbMap.Area[tableId] = shardmap.New(uint8(*threadNum))
}
var ei = protomsg.Esinfo{
Id: id,
@@ -203,7 +207,7 @@
}
func RealTimeDelPersonFromCache(tableId string, id string) {
- logger.Debug("DelPersonFromCache,tableId:", tableId, ",id:", id)
+ logger.Debug("Delete person from cache, tableId:", tableId, ",id:", id)
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
if _, ok := CaptureDbMap.Area[tableId]; ok {
@@ -219,7 +223,7 @@
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
- if dtM, ok := CaptureDbMap.Area[PRE_DBTABLE]; ok {
+ if dtM, ok := CaptureDbMap.Area[PreTableName]; ok {
dtM.Del(tableId)
}
if _, ok := CaptureDbMap.Area[tableId]; ok {
@@ -233,10 +237,10 @@
CaptureDbMap.Lock()
defer CaptureDbMap.Unlock()
- if _, ok := CaptureDbMap.Area[PRE_DBTABLE]; !ok {
- CaptureDbMap.Area[PRE_DBTABLE] = shardmap.New(uint8(*threadnum))
+ if _, ok := CaptureDbMap.Area[PreTableName]; !ok {
+ CaptureDbMap.Area[PreTableName] = shardmap.New(uint8(*threadNum))
}
- CaptureDbMap.Area[PRE_DBTABLE].Set(tableId, enable == 1)
+ CaptureDbMap.Area[PreTableName].Set(tableId, enable == 1)
}
func UpdateCache(changeMsg *protomsg.EsPersonCacheChange) {
--
Gitblit v1.8.0