优化缓存的数据, 取消多次的base64计算和float32转换

zhangzengfei

2024-04-18 bdcc9624341ee34298be74a706b09f12f8306456

优化缓存的数据, 取消多次的base64计算和float32转换

 cache/shardmap/shardmap.go

@@ -29,7 +29,7 @@

var Count = make(chan int)

type wfOp func(a []byte, b string) float32
type wfOp func(a, b []float32) float32

/**
* @param uint8, shardCnt must be pow of two
@@ -90,7 +90,7 @@
    value interface{}
}

func (s *ShardMap) Walk(wf wfOp, sourceFea []byte, baseScore float32) (targets []*protomsg.SdkCompareEach) {
func (s *ShardMap) Walk(wf wfOp, sourceFea []float32, baseScore float32) (targets []*protomsg.SdkCompareEach) {
    var wg sync.WaitGroup
    var lock sync.Mutex
    for _, si := range s.shards {
@@ -102,18 +102,18 @@

        wg.Add(1)

        go func(st *shardItem, fw wfOp, sf []byte, baseSec float32) {
        go func(st *shardItem, fn wfOp, srcFeat []float32, baseSec float32) {
            defer wg.Done()
            for _, feature := range st.data {
                if eif, ok := feature.(*db.FeatureCacheBase); ok {
                if item, ok := feature.(*db.FeatureCacheBase); ok {
                    score := float32(0)
                    score = fw(sf, eif.FaceFeature)
                    score = fn(srcFeat, item.FaceFeature)
                    if score > 0 && score >= baseScore {
                        lock.Lock()
                        targets = append(targets, &protomsg.SdkCompareEach{
                            Id:           eif.Id,
                            Id:           item.Id,
                            CompareScore: score,
                            Tableid:      eif.TableId,
                            Tableid:      item.TableId,
                        })
                        lock.Unlock()
                    }

 compare/compare.go

@@ -1,7 +1,6 @@
package compare

import (
    "encoding/base64"
    "fmt"
    "strconv"

@@ -18,6 +17,8 @@
    if args.FaceFeature == nil {
        return nil
    }

    floatFeat := ByteSlice2float32Slice(args.FaceFeature)

    //指定最低分
    baseScore := thresholdLimit
@@ -36,7 +37,7 @@
    if args.TreeNodes != nil && len(args.TreeNodes) > 0 {
        for _, id := range args.TreeNodes {
            if _, ok := cache.CacheMap.Area[id]; ok {
                targets := cache.CacheMap.Area[id].Walk(DoSdkCompare, args.FaceFeature, baseScore)
                targets := cache.CacheMap.Area[id].Walk(DoSdkCompare, floatFeat, baseScore)
                if len(targets) > 0 {
                    scResult.CompareResult = append(scResult.CompareResult, targets...)
                }
@@ -60,7 +61,7 @@
            continue
        }

        targets := val.Walk(DoSdkCompare, args.FaceFeature, baseScore)
        targets := val.Walk(DoSdkCompare, floatFeat, baseScore)
        if len(targets) > 0 {
            scResult.CompareResult = append(scResult.CompareResult, targets...)
            // todo 添加小区外的关联关系, 下次优先比对
@@ -78,13 +79,8 @@
    return buf
}

func DoSdkCompare(ci []byte, co string) float32 {
    co_d, err := base64.StdEncoding.DecodeString(co)
    if err != nil {
        logger.Error("DoSdkCompare err:", err)
        return -1
    }
    sec := DecCompare(ci, co_d)
func DoSdkCompare(ci, co []float32) float32 {
    sec := DirectCompare(ci, co)
    //logger.Debug("比对得分为：", sec)

    sec = ParseScore(sec)

 compare/faceSdk.go

@@ -62,10 +62,31 @@

//        return  fscore;
//    }
func DecCompare(feat1 []byte, feat2 []byte) float32 {
    ffeat1 := byteSlice2float32Slice(feat1)
    ffeat2 := byteSlice2float32Slice(feat2)
    if len(ffeat1) != len(ffeat2) {
func DirectCompare(feat1 []float32, feat2 []float32) float32 {
    if len(feat1) != len(feat2) {
        return 0
    }

    var score float32
    for i := 0; i < 1536; i++ {
        score += feat1[i] * feat2[i]
    }
    score += 0.05
    if score > 0.9999 {
        score = 0.9999
    }
    if score < 0.0001 {
        score = 0.0001
    }

    //fmt.Println("score:", score)
    return score
}

func DecCompare(feat1, feat2 []byte) float32 {
    ffeat1 := ByteSlice2float32Slice(feat1)
    ffeat2 := ByteSlice2float32Slice(feat2)
    if len(feat1) != len(feat2) {
        return 0
    }
    //fmt.Println("len:", len(ffeat1), len(feat2))
@@ -86,7 +107,7 @@
    return score
}

func byteSlice2float32Slice(src []byte) []float32 {
func ByteSlice2float32Slice(src []byte) []float32 {
    if len(src) == 0 {
        return nil
    }

 db/base.go

@@ -13,6 +13,6 @@
    Id          string
    AreaId      string
    TableId     string
    FaceFeature string
    FaceFeature []float32
    Enable      int32
}

 db/person.go

@@ -1,6 +1,8 @@
package db

import (
    "encoding/base64"
    "sdkCompare/compare"
    "strconv"
)

@@ -55,11 +57,16 @@

    for _, p := range persons {
        if p.FaceFeature != "" {
            byteFeat, err := base64.StdEncoding.DecodeString(p.FaceFeature)
            if err != nil {
                continue
            }

            arr = append(arr, &FeatureCacheBase{
                Id:          p.Id,
                TableId:     p.TableId,
                AreaId:      p.AreaID,
                FaceFeature: p.FaceFeature,
                FaceFeature: compare.ByteSlice2float32Slice(byteFeat),
                Enable:      int32(p.Enable),
            })
        }
@@ -74,12 +81,17 @@
    if err != nil {
        return nil, err
    }

    if p.FaceFeature != "" {
        byteFeat, err := base64.StdEncoding.DecodeString(p.FaceFeature)
        if err != nil {
            return nil, err
        }
        info = &FeatureCacheBase{
            Id:          p.Id,
            TableId:     p.TableId,
            AreaId:      p.AreaID,
            FaceFeature: p.FaceFeature,
            FaceFeature: compare.ByteSlice2float32Slice(byteFeat),
            Enable:      int32(p.Enable),
        }
    }