From 1c8c14cb32884c1a0885e669509e2152f1d3e5e8 Mon Sep 17 00:00:00 2001
From: 554325746@qq.com <554325746@qq.com>
Date: 星期一, 08 七月 2019 09:48:02 +0800
Subject: [PATCH] struct to map[string]float32]
---
shardmap/shardmap.go | 79 ++++++++++++-------
dbpersoninfo.go | 66 ++++++++++------
esutil/EsApi.go | 36 +++-----
esutil/EsClient.go | 46 +++++------
4 files changed, 126 insertions(+), 101 deletions(-)
diff --git a/dbpersoninfo.go b/dbpersoninfo.go
index b42ab0d..7846cd9 100644
--- a/dbpersoninfo.go
+++ b/dbpersoninfo.go
@@ -7,9 +7,11 @@
"sync"
"encoding/base64"
- "basic.com/pubsub/cache.git/esutil"
- "basic.com/pubsub/cache.git/shardmap"
- "basic.com/pubsub/protomsg.git"
+ "test/cache/esutil"
+ "test/cache/shardmap"
+// "basic.com/pubsub/cache.git/esutil"
+// "basic.com/pubsub/cache.git/shardmap"
+// "basic.com/pubsub/protomsg.git"
"basic.com/valib/gosdk.git"
)
@@ -17,34 +19,50 @@
var querynum = flag.Int("querynum", 10, "the query number from database")
var threadnum = flag.Int("threadnum",32, "the number of thread to deal data.")
-var Cmap *shardmap.ShardMap
+type CmapItem struct {
+ sync.Mutex
+ cam map[string]*shardmap.ShardMap
+}
+
+var Cmap *CmapItem
+
func Init(){
flag.Parse()
gosdk.InitFaceExtractor(16, 0)
- Cmap = shardmap.New(uint8(*threadnum))
+ Cmap = &CmapItem{
+ cam: make(map[string]*shardmap.ShardMap),
+ }
temptime := time.Now()
var wg sync.WaitGroup
+
for i:=0; i<*threadnum; i++ {
j := i*(*querynum)
wg.Add(1)
go func(qs int){
defer wg.Done()
-
- escache, err := esutil.DbPersoninfos(qs, *querynum)
+ escache, err := esutil.DbPersoninfos(qs, *querynum)
if err != nil {
fmt.Println(err)
return
}
- for _, value := range escache {
- Cmap.Set(value.FaceFeature,value)
+ Cmap.Lock()
+
+ for _, value := range escache{
+ if _, ok :=Cmap.cam[value.Tableid]; !ok {
+ Cmap.cam[value.Tableid]=shardmap.New(uint8(*threadnum))
+ }
+
+ Cmap.cam[value.Tableid].Set(value.Id,value.FaceFeature)
}
+
+ Cmap.Unlock()
+
}(j)
}
wg.Wait()
- fmt.Println("get number of es: ", Cmap.GetLen())
fmt.Println("time of get data from es.", time.Since(temptime))
fmt.Println()
}
@@ -54,34 +72,30 @@
// Getdbpersonmsg("")
//}
-func Getdbpersonmsg(teststring string, IsCompare bool) ([]byte) {
+func Getdbpersonmsg(tableid, teststring string, IsCompare bool) ([]byte) {
- var binfos protomsg.Binfos
-
+ var buf []byte
if !IsCompare {
return nil
}
if teststring == "" {
- testcache, _:= esutil.DbPersoninfos(0,1)
- teststring = testcache[0].FaceFeature
+ return nil
}
+ for id, value := range Cmap.cam{
+ if id == tableid{
+ fmt.Println(id)
+ buf =value.Walk(Printest, teststring)
+ }
+ }
+
fmt.Println()
- fmt.Println("======鍗庝附鐨勫垎鍓茬嚎============")
fmt.Println()
firsttime := time.Now()
- Rscore := Cmap.Walk(Printest, teststring)
-
- binfos.Infos = append(binfos.Infos, Rscore...)
- binfosbytes, err := binfos.Marshal()
- if err != nil {
- fmt.Println(err)
- return nil
- }
fmt.Println(time.Since(firsttime))
- return binfosbytes
+ return buf
}
func Printest(ci, co string ) (float32){
@@ -92,7 +106,7 @@
// return -1
//}
- co_d, err := base64.StdEncoding.DecodeString(co)
+ co_d, err := base64.StdEncoding.DecodeString(co)
if err != nil {
fmt.Println("co_d : error : ", err)
return -1
diff --git a/esutil/EsApi.go b/esutil/EsApi.go
index 835ff21..3d92d2b 100644
--- a/esutil/EsApi.go
+++ b/esutil/EsApi.go
@@ -8,7 +8,8 @@
)
// 鏌ヨ搴曞簱浜哄憳淇℃伅
-func DbPersoninfos( queryindex int, querynums int ) (dbpersoninfos []*protomsg.Baseinfo, err error){
+func DbPersoninfos( queryindex int, querynums int ) ([]*protomsg.Esinfo, error){
+ var dbinfos []*protomsg.Esinfo
point := strconv.Itoa(queryindex)
number := strconv.Itoa(querynums)
@@ -17,38 +18,27 @@
"query": {
"match_all": {}
},
- "size":`+ number +`,
- "_source": [
+ "size":`+ number +`,
+ "_source": [
+ "id",
"tableId",
- "tableName",
- "compareScore",
- "personId",
- "personName",
- "personPicUrl",
- "phoneNum",
- "sex",
- "idCard",
- "monitorLevel",
- "faceFeature",
- "content",
- "id"
+ "faceFeature"
]
- }`
+ }`
- // 杩斿洖璇锋眰鐨刡uf
- buf, err := GetDbinfo(dbinfoRequest )
- if err != nil {
- return nil ,errors.New("http request get dbperson info is err!")
- }
+ buf, err := EsReq("POST", "http://192.168.1.182:9200/dbtablepersons/dbpersons/_search",[]byte(dbinfoRequest))
+ if err != nil {
+ return dbinfos ,errors.New("http request dbtablename info is err!")
+ }
// 杩斿洖 _source 鏁扮粍
sources, err := Sourcelist(buf)
if err != nil {
- return nil,err
+ return dbinfos,err
}
// 杩斿洖鎵�鏈夋煡璇㈢殑鏁版嵁
- dbpersoninfos = Parsesources(sources)
+ dbpersoninfos := Parsesources(sources)
return dbpersoninfos, nil
}
diff --git a/esutil/EsClient.go b/esutil/EsClient.go
index 4381757..212dbb9 100644
--- a/esutil/EsClient.go
+++ b/esutil/EsClient.go
@@ -12,40 +12,38 @@
"basic.com/pubsub/protomsg.git"
)
-func GetDbinfo(dbinforequest string)([]byte, error) {
- buf , err := EsReq("POST", "http://192.168.1.182:9200/dbtablepersons/dbpersons/_search", []byte(dbinforequest))
- return buf, err
-}
-func Parsesources(sources []map[string]interface{}) []*protomsg.Baseinfo {
- var baseinfos []*protomsg.Baseinfo
+func Parsesources(sources []map[string]interface{}) (esinfos []*protomsg.Esinfo) {
var ok bool
for _, source := range sources {
- var tmpinfo protomsg.Baseinfo
+ var tmpinfo protomsg.Esinfo
tmpinfo.FaceFeature, ok = source["faceFeature"].(string)
Isnil("faceFeature", ok)
- tmpinfo.PersonId, ok = source["id"].(string)
+ tmpinfo.Id, ok = source["id"].(string)
Isnil("id", ok)
- tmpinfo.TableId, ok = source["tableId"].(string)
+ tmpinfo.Tableid, ok = source["tableId"].(string)
Isnil("tableId", ok)
- tmpinfo.PersonName,ok = source["personName"].(string)
- Isnil("personName", ok)
- tmpinfo.PersonPicUrl,ok = source["personPicUrl"].(string)
- Isnil("personPicUrl", ok)
- tmpinfo.PhoneNum,ok = source["phoneNum"].(string)
- Isnil("phoneNum", ok)
- tmpinfo.Sex,ok = source["sex"].(string)
- Isnil("sex", ok)
- tmpinfo.Idcard,ok = source["idCard"].(string)
- Isnil("idCard", ok)
- tmpinfo.MonitorLevel,ok = source["monitorLevel"].(string)
+ esinfos = append(esinfos, &tmpinfo)
+
+// tmpinfo.PersonName,ok = source["personName"].(string)
+// Isnil("personName", ok)
+// tmpinfo.PersonPicUrl,ok = source["personPicUrl"].(string)
+// Isnil("personPicUrl", ok)
+// tmpinfo.PhoneNum,ok = source["phoneNum"].(string)
+// Isnil("phoneNum", ok)
+// tmpinfo.Sex,ok = source["sex"].(string)
+// Isnil("sex", ok)
+// tmpinfo.Idcard,ok = source["idCard"].(string)
+// Isnil("idCard", ok)
+// tmpinfo.MonitorLevel,ok = source["monitorLevel"].(string)
// 鏍规嵁 tableid 鑾峰彇 tablename
- name, _:= Dbtablename(tmpinfo.TableId)
- tmpinfo.TableName= name
- baseinfos = append(baseinfos, &tmpinfo)
+// name, _:= Dbtablename(tmpinfo.TableId)
+// tmpinfo.TableName= name
+// binfos.Infos = append(binfos.Infos, &tmpinfo)
+
}
- return baseinfos
+ return
}
func Sourcelist(buf []byte)(sources []map[string]interface{}, err error){
diff --git a/shardmap/shardmap.go b/shardmap/shardmap.go
index c19edd7..9f1dc6c 100644
--- a/shardmap/shardmap.go
+++ b/shardmap/shardmap.go
@@ -4,20 +4,18 @@
import (
"sync"
+ "encoding/json"
"fmt"
-
- "basic.com/pubsub/protomsg.git"
)
+var commonmux sync.Mutex
const (
- DEF_SHARD_CNT = 32
BKDR_SEED = 131 // 31 131 1313 13131 131313 etc...
)
type shardItem struct {
sync.RWMutex
-
data map[string]interface{}
}
@@ -34,9 +32,6 @@
* @param uint8, shardCnt must be pow of two
*/
func New(shardCnt uint8) *ShardMap {
- if !isPowOfTwo(shardCnt) {
- shardCnt = DEF_SHARD_CNT
- }
s := &ShardMap{
shardCnt: shardCnt,
@@ -83,33 +78,65 @@
}
// modify by long.
-func (s *ShardMap) Walk(wf wfOp, cfrom string) []*protomsg.Baseinfo{
+func (s *ShardMap) Walk(wf wfOp, cfrom string) ([]byte){
var wg sync.WaitGroup
var second float32
+ var buf []byte
+ var err error
- var baseinfos []*protomsg.Baseinfo
+ ids := make(map[string]float32)
+ for _, si := range s.shards {
+ var tempsi shardItem
+ tempsi = *si
+
+
+ wg.Add(1)
+
+ go func(st *shardItem, fw wfOp, cf string) {
+ defer wg.Done()
+ commonmux.Lock()
+ for id, feature := range st.data {
+ if str, ok := feature.(string); ok {
+ second = fw(cf,str)
+ if second == -1 {
+ continue
+ }
+ ids[id]=second
+ }
+ }
+ commonmux.Unlock()
+ }(&tempsi, wf, cfrom)
+ }
+
+ wg.Wait()
+
+ buf, err = json.Marshal(ids)
+ if err != nil {
+ fmt.Println("compare json err")
+ buf = nil
+
+ }
+ return buf
+}
+
+// print all
+func (s *ShardMap) Printall() (infos []interface{}){
+ var wg sync.WaitGroup
for _, si := range s.shards {
wg.Add(1)
- go func(s *shardItem, fw wfOp, cf string) {
+ go func(s *shardItem) {
defer wg.Done()
s.RLock()
- for key, value := range s.data {
- second = fw(cf,key)
- if second == -1 {
- continue
- }
- if info, ok := value.(*protomsg.Baseinfo) ;ok {
- fmt.Println("姣斿鍒嗘暟: ", second)
- info.CompareScore = second
- baseinfos = append(baseinfos,info)
- }
- }
- s.RUnlock()
- }(si,wf, cfrom)
+ for _, value := range s.data {
+ infos = append(infos, value)
+ }
+
+ s.RUnlock()
+ }(si)
}
wg.Wait()
- return baseinfos
+ return
}
func (s * ShardMap)GetLen() int {
@@ -124,10 +151,6 @@
i := bkdrHash(key) & uint32(s.shardCnt-1)
return s.shards[i]
-}
-
-func isPowOfTwo(x uint8) bool {
- return x != 0 && (x&(x-1) == 0)
}
//https://www.byvoid.com/blog/string-hash-compare/
--
Gitblit v1.8.0