添加失败缓存重发的功能

zhangzengfei

2024-04-16 e4840c7de5ff5f1a816a4fd023ba5bf797a4fa9e

添加失败缓存重发的功能

 config/config.go

@@ -69,5 +69,9 @@
        LogConf.Level = "info"
    }

    if ForwardConf.ReportInterval == 0 {
        ForwardConf.ReportInterval = 10
    }

    logger.SetLogLevel(LogConf.Level)
}

 cron/cron.go

New file
@@ -0,0 +1,21 @@
package cron

import (
    "gat1400Exchange/config"
    "gat1400Exchange/service"
    "github.com/go-co-op/gocron"
    "time"
)

var s *gocron.Scheduler

func init() {
    s = gocron.NewScheduler(time.UTC)
}

func Init() {
    s.Every(10).Minutes().StartImmediately().Do(service.ResendImageData)
    s.Every(config.ForwardConf.ReportInterval).Second().StartImmediately().Do(service.DeviceInfoReportTask)

    s.StartAsync()
}

 go.mod

@@ -23,9 +23,11 @@
    github.com/chenzhuoyu/iasm v0.9.1 // indirect
    github.com/gabriel-vasile/mimetype v1.4.3 // indirect
    github.com/gin-contrib/sse v0.1.0 // indirect
    github.com/go-co-op/gocron v1.37.0 // indirect
    github.com/go-sql-driver/mysql v1.7.0 // indirect
    github.com/goccy/go-json v0.10.2 // indirect
    github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
    github.com/google/uuid v1.4.0 // indirect
    github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
    github.com/hashicorp/hcl v1.0.0 // indirect
    github.com/jinzhu/inflection v1.0.0 // indirect
@@ -40,6 +42,7 @@
    github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
    github.com/modern-go/reflect2 v1.0.2 // indirect
    github.com/pelletier/go-toml/v2 v2.1.1 // indirect
    github.com/robfig/cron/v3 v3.0.1 // indirect
    github.com/sagikazarmark/locafero v0.4.0 // indirect
    github.com/sagikazarmark/slog-shim v0.1.0 // indirect
    github.com/satori/go.uuid v1.2.0 // indirect

 go.sum

@@ -14,6 +14,7 @@
github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -28,6 +29,8 @@
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0=
github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@@ -43,6 +46,8 @@
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -59,8 +64,14 @@
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
@@ -78,8 +89,13 @@
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
@@ -105,6 +121,7 @@
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
@@ -138,7 +155,10 @@
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

 main.go

@@ -3,6 +3,7 @@
import (
    "context"
    "fmt"
    "gat1400Exchange/cron"
    "net/http"
    "os"
    "os/signal"
@@ -38,7 +39,9 @@

    // 启动网络视频字符叠加器服务
    go service.NVCSServer()
    go service.DeviceInfoReportTask()

    // 启动定时任务
    cron.Init()

    // Initializing the server in a goroutine so that
    // it won't block the graceful shutdown handling below

 models/cache.go

New file
@@ -0,0 +1,29 @@
package models

type Cache struct {
    Id         uint   `gorm:"column:id;primary_key;auto_increment;unique;not null;"`
    Data       string `gorm:"column:data;type:text"`
    CreateTime int64  `gorm:"column:create_time;"`
    Retry      int    `gorm:"column:retry;"`
}

func (c *Cache) TableName() string {
    return "caches"
}

func (c *Cache) FindAll() ([]Cache, error) {
    var caches []Cache
    if err := db.Table(c.TableName()).Find(&caches).Error; err != nil {
        return nil, err
    }

    return caches, nil
}

func (c *Cache) Update() error {
    return db.Table(c.TableName()).Save(&c).Error
}

func (c *Cache) Delete() error {
    return db.Table(c.TableName()).Delete("id = ?", c.Id).Error
}

 models/db.go

@@ -20,7 +20,7 @@
        logger.Debug("db open error ", err)
        return err
    }
    _ = db.AutoMigrate(&Device{}, &Positions{})
    _ = db.AutoMigrate(&Device{}, &Positions{}, &Cache{})

    // 添加默认数据
    InitData()

 repository/captureRepo.go

@@ -1,15 +1,14 @@
package repository

import (
    "bytes"
    "encoding/base64"
    "encoding/json"
    "net/http"
    "time"

    "gat1400Exchange/config"
    "gat1400Exchange/models"
    "gat1400Exchange/pkg/logger"
    "gat1400Exchange/util"
    "gat1400Exchange/vo"

    uuid "github.com/satori/go.uuid"
@@ -46,18 +45,24 @@
        }

        // 转发图像
        logger.Debug("准备转发,deviceId:%s, image len:%d, server:%s", deviceId, len(faceImageStr), config.ForwardConf.SyncServer)
        logger.Debug("Prepare forward image, deviceId:%s, image len:%d, server:%s", deviceId, len(faceImageStr), config.ForwardConf.SyncServer)
        if deviceId != "" && faceImageStr != "" && config.ForwardConf.SyncServer != "" {
            pd := c.PackPushDataV2(deviceId, faceId, faceImageStr, face.FaceAppearTime)
            if pd == nil {
                return
            }

            rsp, err := c.SendData(pd, config.ForwardConf.SyncServer)
            payload, err := json.Marshal(pd)
            if err != nil {
                logger.Warn("数据转发失败:%s", err.Error())
                logger.Warn("Marshal error, %s", err.Error())
                return
            }

            if !c.SendData(payload, config.ForwardConf.SyncServer) {
                c.CacheData(payload)
                logger.Warn("数据转发失败, 添加缓存")
            } else {
                logger.Debug("数据转发成功,id:%s", rsp)
                logger.Debug("数据转发成功")
            }
        }
    }
@@ -77,7 +82,7 @@
    // 匹配楼层
    faceAppearTime, err := time.ParseInLocation("20060102150405", appearTime, time.Local)
    if err != nil {
        logger.Warn("Parse face appear time error,%s", err.Error())
        logger.Warn("Parse face appear time error, %s", err.Error())
        faceAppearTime = time.Now()
    }

@@ -86,7 +91,7 @@

    imageBytes, err := base64.StdEncoding.DecodeString(faceImage)
    if err != nil {
        logger.Warn("Decode Image Base64 String failure,%s", err.Error())
        logger.Warn("Decode Image Base64 String failure, %s", err.Error())
        return pd
    }

@@ -112,7 +117,7 @@
    // 匹配楼层
    faceAppearTime, err := time.ParseInLocation("20060102150405", appearTime, time.Local)
    if err != nil {
        logger.Warn("Parse face appear time error,%s", err.Error())
        logger.Warn("Parse face appear time error, %s", err.Error())
        faceAppearTime = time.Now()
    }

@@ -126,7 +131,7 @@

    imageBytes, err := base64.StdEncoding.DecodeString(faceImage)
    if err != nil {
        logger.Warn("Decode Image Base64 String failure,%s", err.Error())
        logger.Warn("Decode Image Base64 String failure, %s", err.Error())
        return pd
    }

@@ -154,35 +159,30 @@
    return pd
}

func (c CaptureRepository) SendData(pushDataInfo interface{}, url string) (id string, err error) {
    id = ""
    payload, err := json.Marshal(pushDataInfo)
func (c CaptureRepository) SendData(payload []byte, url string) bool {
    body, err := util.HttpPost(url, nil, payload)
    if err != nil {
        return id, err
        logger.Error("Post request failure, url:%s, err:%s", err.Error())
        return false
    }

    req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
    var rsp vo.SyncServerResponse
    err = json.Unmarshal(body, &rsp)
    if err != nil {
        return id, err
        logger.Error("Can't parse response, url:%s, rsp: %v", url, body)
        return false
    }

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        return id, err
    }
    defer resp.Body.Close()
    return rsp.Success
}

    var responseData map[string]interface{}
    err = json.NewDecoder(resp.Body).Decode(&responseData)
    if err != nil {
        return id, err
func (c CaptureRepository) CacheData(payload []byte) {
    var cacheItem = models.Cache{
        Data:       string(payload),
        CreateTime: time.Now().Unix(),
        Retry:      0,
    }

    id = responseData["data"].(string)
    // 处理响应
    // 这里可以根据实际需求进行处理，例如读取响应内容或检查状态码等

    return id, nil
    models.GetDB().Table(cacheItem.TableName()).Create(&c)

}

 service/report.go

@@ -11,6 +11,7 @@
    "gat1400Exchange/config"
    "gat1400Exchange/pkg/logger"
    "gat1400Exchange/models"
    "gat1400Exchange/util"

    "github.com/hashicorp/golang-lru/v2/expirable"
)
@@ -39,9 +40,11 @@
    IpAddr      string                 `json:"ip"`
}

func reportDeviceInfo() error {
func DeviceInfoReportTask() error {
    logger.Info("Start device info report task, server:%s.", config.ForwardConf.ReportServer)

    if config.ForwardConf.ReportServer == "" {
        return errors.New("server addr is empty !")
        return errors.New("Server addr is empty!")
    }

    var d models.Device
@@ -71,7 +74,7 @@

        logger.Info("Report device info. %+v", dev)

        _, err = HttpPost(config.ForwardConf.ReportServer, nil, data)
        _, err = util.HttpPost(config.ForwardConf.ReportServer, nil, data)
        if err != nil {
            return err
        }
@@ -82,51 +85,4 @@

func KeepDeviceAlive(id string) {
    deviceAliveCache.Add(id, true)
}

func DeviceInfoReportTask() {
    interval := config.ForwardConf.ReportInterval
    if interval == 0 {
        interval = 10
    }

    logger.Info("start device info report task, server:%s interval:%d.", config.ForwardConf.ReportServer, interval)

    ticker := time.NewTicker(1 * time.Second)
    for {
        select {
        case <-ticker.C:
            err := reportDeviceInfo()
            if err != nil {
                logger.Warn(err.Error())
            }

            ticker.Reset(time.Duration(interval) * time.Second)
        }
    }
}


func HttpPost(url string, header map[string]string, data []byte) ([]byte, error) {
    req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
    if err != nil {
        return nil, err
    }
    req.Header.Set("Content-Type", "applicaiton/json; charset=UTF-8")
    if header != nil {
        for k, v := range header {
            req.Header.Set(k, v)
        }
    }
    cli := &http.Client{}
    resp, err := cli.Do(req)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()
    body, err := io.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }
    return body, nil
}

 service/resend.go

New file
@@ -0,0 +1,45 @@
package service

import (
    "encoding/json"
    "gat1400Exchange/config"
    "gat1400Exchange/models"
    "gat1400Exchange/pkg/logger"
    "gat1400Exchange/util"
    "gat1400Exchange/vo"
)

func ResendImageData() {
    var cacheMod models.Cache
    cacheItems, _ := cacheMod.FindAll()

    for _, c := range cacheItems {
        if !SendData([]byte(c.Data), config.ForwardConf.SyncServer) {
            logger.Warn("The data resend failed. retry count %d", c.Retry)

            c.Retry = c.Retry + 1
            c.Update()
        } else {
            c.Delete()
            logger.Debug("The data resend successful.")
        }
    }

}

func SendData(payload []byte, url string) bool {
    body, err := util.HttpPost(url, nil, payload)
    if err != nil {
        logger.Error("Post request failure, url:%s, err:%s", err.Error())
        return false
    }

    var rsp vo.SyncServerResponse
    err = json.Unmarshal(body, &rsp)
    if err != nil {
        logger.Error("Can't parse response, url:%s, rsp: %v", url, body)
        return false
    }

    return rsp.Success
}

 util/http.go

New file
@@ -0,0 +1,35 @@
package util

import (
    "bytes"
    "io"
    "net/http"
)

func HttpPost(url string, header map[string]string, data []byte) ([]byte, error) {
    req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
    if err != nil {
        return nil, err
    }

    req.Header.Set("Content-Type", "applicaiton/json; charset=UTF-8")
    if header != nil {
        for k, v := range header {
            req.Header.Set(k, v)
        }
    }

    cli := &http.Client{}
    resp, err := cli.Do(req)
    if err != nil {
        return nil, err
    }

    defer resp.Body.Close()
    body, err := io.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }

    return body, nil
}

 vo/server.go

New file
@@ -0,0 +1,8 @@
package vo

type SyncServerResponse struct {
    Code    int
    Success bool
    Msg     string
    Data    interface{}
}