struct to map[string]float32]

554325746@qq.com

2019-07-08 1c8c14cb32884c1a0885e669509e2152f1d3e5e8

 struct to map[string]float32]

 dbpersoninfo.go

@@ -7,9 +7,11 @@
    "sync"
    "encoding/base64"

    "basic.com/pubsub/cache.git/esutil"
    "basic.com/pubsub/cache.git/shardmap"
    "basic.com/pubsub/protomsg.git"
    "test/cache/esutil"
    "test/cache/shardmap"
//    "basic.com/pubsub/cache.git/esutil"
//  "basic.com/pubsub/cache.git/shardmap"
//  "basic.com/pubsub/protomsg.git"

    "basic.com/valib/gosdk.git"
)
@@ -17,34 +19,50 @@
var querynum = flag.Int("querynum", 10, "the query number from database")
var threadnum = flag.Int("threadnum",32, "the number of thread to deal data.")

var Cmap *shardmap.ShardMap
type CmapItem struct {
    sync.Mutex
    cam map[string]*shardmap.ShardMap
}

var Cmap *CmapItem

func Init(){
    flag.Parse()
    gosdk.InitFaceExtractor(16, 0)

    Cmap = shardmap.New(uint8(*threadnum))
    Cmap = &CmapItem{
        cam: make(map[string]*shardmap.ShardMap),
    }

    temptime := time.Now()
    var wg sync.WaitGroup

    for i:=0; i<*threadnum; i++ {
        j := i*(*querynum)
        wg.Add(1)
        go func(qs int){
            defer wg.Done()

            escache, err := esutil.DbPersoninfos(qs, *querynum)
            escache, err := esutil.DbPersoninfos(qs, *querynum) 
            if err != nil {
                fmt.Println(err)
                return
            }

            for _, value := range escache {
                 Cmap.Set(value.FaceFeature,value) 
            Cmap.Lock()           

            for _, value := range escache{
                if _, ok :=Cmap.cam[value.Tableid]; !ok {
                   Cmap.cam[value.Tableid]=shardmap.New(uint8(*threadnum))        
                }
                
                Cmap.cam[value.Tableid].Set(value.Id,value.FaceFeature) 
            }

            Cmap.Unlock()

        }(j)
    }
    wg.Wait()
    fmt.Println("get number of es: ", Cmap.GetLen())
    fmt.Println("time of get data from es.", time.Since(temptime))
    fmt.Println()
}
@@ -54,34 +72,30 @@
//    Getdbpersonmsg("")
//}

func Getdbpersonmsg(teststring string, IsCompare bool) ([]byte) {
func Getdbpersonmsg(tableid, teststring string, IsCompare bool) ([]byte) {

    var binfos protomsg.Binfos

    var buf []byte
    if !IsCompare {
        return nil
    }

    if teststring == "" {
    testcache, _:= esutil.DbPersoninfos(0,1)
    teststring = testcache[0].FaceFeature
        return nil
    }

   for id, value := range Cmap.cam{
        if id == tableid{
            fmt.Println(id)
            buf =value.Walk(Printest, teststring)          
        }
   }

    fmt.Println()
    fmt.Println("======华丽的分割线============")
    fmt.Println()

    firsttime := time.Now()
    Rscore := Cmap.Walk(Printest, teststring)
    
    binfos.Infos = append(binfos.Infos, Rscore...)
    binfosbytes, err := binfos.Marshal()
    if err != nil  {
        fmt.Println(err)
        return nil 
    }
    fmt.Println(time.Since(firsttime))
    return binfosbytes
    return  buf
}

func Printest(ci, co string ) (float32){
@@ -92,7 +106,7 @@
    //    return -1
    //}

    co_d, err := base64.StdEncoding.DecodeString(co)
   co_d, err := base64.StdEncoding.DecodeString(co)
    if err != nil {
        fmt.Println("co_d : error : ", err)
        return -1

 esutil/EsApi.go

@@ -8,7 +8,8 @@
        )

// 查询底库人员信息
func DbPersoninfos( queryindex int, querynums int ) (dbpersoninfos []*protomsg.Baseinfo, err error){
func DbPersoninfos( queryindex int, querynums int ) ([]*protomsg.Esinfo, error){
            var dbinfos []*protomsg.Esinfo
            point   := strconv.Itoa(queryindex)
            number   := strconv.Itoa(querynums)

@@ -17,38 +18,27 @@
                    "query": {
                        "match_all": {}
                    },
                        "size":`+ number +`,
                        "_source": [
                     "size":`+ number +`,
                     "_source": [
                       "id",
                       "tableId",
                       "tableName",
                       "compareScore",
                       "personId",
                       "personName",
                       "personPicUrl",
                       "phoneNum",
                       "sex",
                       "idCard",
                       "monitorLevel",
                       "faceFeature",
                       "content",
                       "id"
                       "faceFeature"
                       ]
                }`
                    }`

      // 返回请求的buf
      buf, err := GetDbinfo(dbinfoRequest )
      if err != nil {
        return nil ,errors.New("http request get dbperson info is err!")
      }
    buf, err := EsReq("POST", "http://192.168.1.182:9200/dbtablepersons/dbpersons/_search",[]byte(dbinfoRequest))
    if err != nil {
        return dbinfos ,errors.New("http request dbtablename info is err!")
    }

      // 返回 _source 数组
      sources, err := Sourcelist(buf)
      if err != nil {
            return nil,err 
            return dbinfos,err 
      }

      // 返回所有查询的数据
      dbpersoninfos =  Parsesources(sources)
     dbpersoninfos :=  Parsesources(sources)
      return dbpersoninfos, nil
}


 esutil/EsClient.go

@@ -12,40 +12,38 @@
    "basic.com/pubsub/protomsg.git"
)

func GetDbinfo(dbinforequest string)([]byte, error) {
    buf , err := EsReq("POST", "http://192.168.1.182:9200/dbtablepersons/dbpersons/_search", []byte(dbinforequest))
    return buf, err
}

func Parsesources(sources []map[string]interface{}) []*protomsg.Baseinfo {
    var baseinfos []*protomsg.Baseinfo
func Parsesources(sources []map[string]interface{}) (esinfos []*protomsg.Esinfo) {
    var ok bool
    for _, source := range sources {
        var tmpinfo protomsg.Baseinfo
       var tmpinfo protomsg.Esinfo
        tmpinfo.FaceFeature, ok =   source["faceFeature"].(string)
            Isnil("faceFeature", ok)
        tmpinfo.PersonId, ok    =   source["id"].(string)  
        tmpinfo.Id, ok    =   source["id"].(string)  
            Isnil("id", ok)
        tmpinfo.TableId, ok     =   source["tableId"].(string)
        tmpinfo.Tableid, ok     =   source["tableId"].(string)
            Isnil("tableId", ok)
        tmpinfo.PersonName,ok  =   source["personName"].(string)
            Isnil("personName", ok)
        tmpinfo.PersonPicUrl,ok =  source["personPicUrl"].(string)
            Isnil("personPicUrl", ok)
        tmpinfo.PhoneNum,ok     =  source["phoneNum"].(string)
            Isnil("phoneNum", ok)
        tmpinfo.Sex,ok          =  source["sex"].(string)
            Isnil("sex", ok)
        tmpinfo.Idcard,ok       =  source["idCard"].(string)
            Isnil("idCard", ok)
        tmpinfo.MonitorLevel,ok =  source["monitorLevel"].(string)
        esinfos = append(esinfos, &tmpinfo)

//        tmpinfo.PersonName,ok  =   source["personName"].(string)
//            Isnil("personName", ok)
//        tmpinfo.PersonPicUrl,ok =  source["personPicUrl"].(string)
//            Isnil("personPicUrl", ok)
//        tmpinfo.PhoneNum,ok     =  source["phoneNum"].(string)
//            Isnil("phoneNum", ok)
//        tmpinfo.Sex,ok          =  source["sex"].(string)
//            Isnil("sex", ok)
//        tmpinfo.Idcard,ok       =  source["idCard"].(string)
//            Isnil("idCard", ok)
//        tmpinfo.MonitorLevel,ok =  source["monitorLevel"].(string)

        // 根据 tableid 获取 tablename
        name, _:= Dbtablename(tmpinfo.TableId) 
        tmpinfo.TableName= name 
        baseinfos = append(baseinfos, &tmpinfo)
//        name, _:= Dbtablename(tmpinfo.TableId) 
//        tmpinfo.TableName= name 
//        binfos.Infos = append(binfos.Infos, &tmpinfo)

    }
    return baseinfos
    return
}

func Sourcelist(buf []byte)(sources []map[string]interface{}, err error){

 shardmap/shardmap.go

@@ -4,20 +4,18 @@

import (
    "sync"
    "encoding/json"
    "fmt"

    "basic.com/pubsub/protomsg.git"
)

var commonmux sync.Mutex
const (
    DEF_SHARD_CNT = 32

    BKDR_SEED = 131 // 31 131 1313 13131 131313 etc...
)

type shardItem struct {
    sync.RWMutex

    data map[string]interface{}
}

@@ -34,9 +32,6 @@
* @param uint8, shardCnt must be pow of two
*/
func New(shardCnt uint8) *ShardMap {
    if !isPowOfTwo(shardCnt) {
        shardCnt = DEF_SHARD_CNT
    }

    s := &ShardMap{
        shardCnt: shardCnt,
@@ -83,33 +78,65 @@
}

// modify by long.
func (s *ShardMap) Walk(wf wfOp, cfrom string) []*protomsg.Baseinfo{
func (s *ShardMap) Walk(wf wfOp, cfrom string) ([]byte){
    var wg sync.WaitGroup
    var second float32 
    var buf []byte
    var err error

    var baseinfos []*protomsg.Baseinfo
    ids := make(map[string]float32)
    for _, si := range s.shards {
        var tempsi shardItem
            tempsi = *si


        wg.Add(1)
        
        go func(st *shardItem, fw wfOp, cf string) {
            defer wg.Done()
            commonmux.Lock()
            for id, feature := range st.data {
                 if str, ok := feature.(string); ok {
                    second = fw(cf,str)
                    if second == -1 {
                        continue 
                     }
                    ids[id]=second
                 }
            }
         commonmux.Unlock()
        }(&tempsi, wf, cfrom)
    }

    wg.Wait()

    buf, err = json.Marshal(ids)
    if err != nil {
        fmt.Println("compare json err")
            buf = nil 
        
    }
    return buf
}

// print all
func (s *ShardMap) Printall() (infos []interface{}){
    var wg sync.WaitGroup

    for _, si := range s.shards {
        wg.Add(1)
        go func(s *shardItem, fw wfOp, cf string) {
        go func(s *shardItem) {
            defer wg.Done()
            s.RLock()
            for key, value := range s.data {
                 second = fw(cf,key)
                 if second == -1 {
                        continue 
                 }
                 if info, ok := value.(*protomsg.Baseinfo) ;ok {
                    fmt.Println("比对分数: ", second)
                    info.CompareScore = second
                    baseinfos = append(baseinfos,info) 
                 } 
            }
            s.RUnlock()
        }(si,wf, cfrom)
            for _, value := range s.data {
                infos = append(infos, value)
            } 
    	
          s.RUnlock()
        }(si)
    }
    wg.Wait()
    return baseinfos
    return
}

func (s * ShardMap)GetLen() int {
@@ -124,10 +151,6 @@
    i := bkdrHash(key) & uint32(s.shardCnt-1)

    return s.shards[i]
}

func isPowOfTwo(x uint8) bool {
    return x != 0 && (x&(x-1) == 0)
}

//https://www.byvoid.com/blog/string-hash-compare/