get sdks from remote shop

liuxiaolong

2020-06-05 7277a23ca81123a95cbf60cbda65ca2fd6e27c17

get sdks from remote shop

 controllers/sdk.go

@@ -81,8 +81,10 @@
// @Router /data/api-v/sdk/findAllSdk [GET]
func (sc SdkController) FindAllSdk(c *gin.Context) {
    sdkName := c.Query("sdkName")
    logger.Debug("FindAllSdk sdkName:", sdkName)

    sdks := service.GetSdkList(sdkName)
    sdks := service.GetSdkList("") //本机已安装的算法

    util.ResponseFormat(c, code.Success, sdks)
}


 go.mod

@@ -8,7 +8,7 @@
    basic.com/gb28181api.git v0.0.0-20191028082253-472438a8407b
    basic.com/pubsub/cache.git v0.0.0-20190718093725-6a413e1d7d48
    basic.com/pubsub/esutil.git v0.0.0-20200114073900-ad9de8362777
    basic.com/pubsub/protomsg.git v0.0.0-20200306071852-76af8d2ca810
    basic.com/pubsub/protomsg.git v0.0.0-20200605082339-fe3f28d45337
    basic.com/valib/capture.git v0.0.0-20191204103802-89c923cf2abe
    basic.com/valib/deliver.git v0.0.0-20190531095353-25d8c3b20051
    basic.com/valib/godraw.git v0.0.0-20191122082247-26e9987cd183
@@ -17,7 +17,7 @@
    basic.com/valib/gopherdiscovery.git v0.0.0-20190605034340-15d89d8b4e28
    basic.com/valib/logger.git v0.0.0-20190928113028-4907b08c4159
    github.com/Microsoft/go-winio v0.4.12 // indirect
    github.com/ajg/form v1.5.1 // indirect
    github.com/ajg/form v1.5.1
    github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc
    github.com/dgrijalva/jwt-go v3.2.0+incompatible
    github.com/disintegration/imaging v1.6.2 // indirect

 go.sum

@@ -10,9 +10,8 @@
basic.com/pubsub/esutil.git v0.0.0-20191120125514-865efa73a9ae/go.mod h1:yIvppFPFGC61DOdm71ujnsxZBMFUu2yKjr5O43bMWCw=
basic.com/pubsub/esutil.git v0.0.0-20200114073900-ad9de8362777 h1:gTeuhepfLgOchD6bqydsGGV6KCj/UaseQQgo4DFyhGQ=
basic.com/pubsub/esutil.git v0.0.0-20200114073900-ad9de8362777/go.mod h1:yIvppFPFGC61DOdm71ujnsxZBMFUu2yKjr5O43bMWCw=
basic.com/pubsub/protomsg.git v0.0.0-20200304101716-2f4c0110fab2/go.mod h1:un5NV5VWQoblVLZfx1Rt5vyLgwR0jI92d3VJhfrJhWU=
basic.com/pubsub/protomsg.git v0.0.0-20200306071852-76af8d2ca810 h1:xGOKeo8E89NxioWCFySRhom9i8zWvu3Gsi9IYu6LYJw=
basic.com/pubsub/protomsg.git v0.0.0-20200306071852-76af8d2ca810/go.mod h1:un5NV5VWQoblVLZfx1Rt5vyLgwR0jI92d3VJhfrJhWU=
basic.com/pubsub/protomsg.git v0.0.0-20200605082339-fe3f28d45337 h1:6LTdIfHUZkfemhkuQnXXaeXRdnm4459PyxMd36AXV20=
basic.com/pubsub/protomsg.git v0.0.0-20200605082339-fe3f28d45337/go.mod h1:un5NV5VWQoblVLZfx1Rt5vyLgwR0jI92d3VJhfrJhWU=
basic.com/valib/capture.git v0.0.0-20191204103802-89c923cf2abe h1:uh3u7DuSOw6AwzvPC1EM19sw1Skks1EUJddcbHDKI9M=
basic.com/valib/capture.git v0.0.0-20191204103802-89c923cf2abe/go.mod h1:y+h7VUnoSQ3jOtf2K3twXNA8fYDfyUsifSswcyKLgNw=
basic.com/valib/deliver.git v0.0.0-20190531095353-25d8c3b20051/go.mod h1:bkYiTUGzckyNOjAgn9rB/DOjFzwoSHJlruuWQ6hu6IY=

 service/SdkDownLoad.go

@@ -12,6 +12,9 @@
    "os"
    "path/filepath"
    "strings"
    "webserver/extend/config"

    reqUtil "webserver/extend/util"

    "webserver/util"

@@ -127,30 +130,59 @@
    }
}

func GetSdkList(sdkName string) []map[string]interface{} {
    // 待返回的数据
    sdks, idsCache := queryDatabase(sdkName)

    // 查询本地已经存在的算法和远端的全部算法, 然后和已安装的比对, 返回全部可用的算法组. 未安装的算法在前端走下载流程
    localSdks := GetLocalSdks()
    for _, sdk := range localSdks {
        id := sdk["id"].(string)
        if !idsCache[id] {
            idsCache[id] = true

            sdk["enable"] = false
            sdk["installed"] = false

            // 不返回参数信息
            delete(sdk, "args")
            sdks = append(sdks, sdk)
func GetSdkList(sdkName string) []SdkInsOrUpgrade {
    var api dbapi.SdkApi
    localSdks := api.FindAll(sdkName) //本地已安装所有算法
    localSdkM := make(map[string]SdkInsOrUpgrade)
    for _,ls :=range localSdks {
        siou := SdkInsOrUpgrade{
            Installed: true,
        }
        siou.Sdk = Sdk{
            Id: ls.Id,
            IpcId: ls.IpcId,
            SdkType: ls.SdkType,
            SdkName: ls.SdkName,
            Icon: ls.Icon,
            Url: ls.Url,
            CreateTime: ls.CreateTime,
            CreateBy: ls.CreateBy,
            UpdateTime: ls.UpdateTime,
            Enable: ls.Enable,
            DelFlag: int(ls.DelFlag),
            IconBlob: ls.IconBlob,
            Version: ls.Version,
        }
        localSdkM[siou.Id] = siou
        }

    //远端可用的算法
    remoteSdks := findAllMySdk()

    for sdkId,sdk := range remoteSdks {
        if v,ok := localSdkM[sdkId];ok { //本地已安装
            ls := v
            if shouldVersionBeUpgrade(ls.Version, sdk.Version) { //判断版本号是否需要升级
                ls.Installed = true
                ls.IsUpgrade = true //需要升级
                ls.RemoteVersion = sdk.Version //远端的版本号
            }
            localSdkM[sdkId] = ls
        } else { //本地未安装，需要安装
            bIns := SdkInsOrUpgrade{
                RemoteVersion: sdk.Version,
                IsUpgrade: false,
                Installed: false,
            }
            bIns.Sdk = sdk
            localSdkM[sdkId] = bIns
        }
    }

    // 目前还没有远端算法库,
    // remoteSdks := getRemoteSdks()

    sdks := make([]SdkInsOrUpgrade, 0)
    for _,v := range localSdkM {
        sdks = append(sdks, v)
    }
    return sdks
}

@@ -177,6 +209,42 @@
    return sdks, ids
}


type shopSdks struct {
    Sdks []Sdk `json:"sdks"`
}

func findAllMySdk() map[string]Sdk {
    token := "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1OTEzODA3ODksInVzZXIiOiJ7XCJpZFwiOlwiZTZjY2QzNmQtNGYxNi00NmZjLTg4ZDUtMDczNjU4NjZkMjA1XCIsXCJwZXJtaXNzaW9uc1wiOltcIm9yZGVyTWFuZ2U6ZG93bmxvYWRcIixcImRldmljZU1hbmFnZTp1bmJpbmRcIixcImNvZGVNYW5nZVwiLFwidmlwTWFuYWdlOmFkZFwiLFwiYWRtaW5NYW5hZ2U6ZWRpdFwiLFwiY29kZU1hbmdlOnZpZXdcIixcImNvZGVNYW5nZTphZGRcIixcImFkbWluTWFuYWdlXCIsXCJvcmRlck1hbmdlOmNhbmNlbFwiLFwicHJvZHVjdENlbnRlcjpidXlcIixcInByb2R1Y3RDZW50ZXJcIixcInByb2R1Y3RNYW5nZTp2aWV3XCIsXCJ2aXBNYW5hZ2U6dmlld1wiLFwib3JkZXJNYW5nZVwiLFwicHJvZHVjdENlbnRlcjp2aWV3XCIsXCJkZXZpY2VNYW5hZ2VcIixcImFkbWluTWFuYWdlOmFkZFwiLFwicHJvZHVjdE1hbmdlOmFkZFwiLFwiYWRtaW5NYW5hZ2U6dmlld1wiLFwicHJvZHVjdE1hbmdlXCIsXCJvcmRlck1hbmdlOnZpZXdcIixcImRldmljZU1hbmFnZTp2aWV3XCIsXCJvcmRlck1hbmdlOnBheVwiLFwidmlwTWFuYWdlOmVkaXRcIixcInZpcE1hbmFnZVwiLFwicHJvZHVjdE1hbmdlOmVkaXRcIixcInByb2R1Y3RNYW5nZTpwdWJsaXNoXCIsXCJkZXZpY2VNYW5hZ2U6YWRkXCIsXCJhcGlcIixcImhvbWVcIixcInByb2R1Y3RNYW5nZTpvZmZTYWxlXCIsXCJwcm9kdWN0Q2VudGVyOmRvd25sb2FkXCJdLFwidXNlcm5hbWVcIjpcImJhc2ljXCJ9In0.HwRobdFLtMK7ni5OKk4_NAyqpKGuUlUbqF3HBJMJuOk"
    url := "http://192.168.20.10:7004/data/api-s/sdk/findAllMySdk"
    fmt.Println("token:", token, "url:", url)
    paramBody := map[string]interface{} {
        "serverId": config.Server.AnalyServerId,
        "machineCode": "",
    }
    header := map[string]string {
        "Authorization": token,
    }
    respBody, err := reqUtil.DoPostRequest(url, reqUtil.CONTENT_TYPE_JSON, paramBody, nil, header)
    if err != nil {
        return nil
    }
    var res dbapi.Result
    if err = json.Unmarshal(respBody, &res); err != nil {
        return nil
    }
    bytes, _ := json.Marshal(res.Data)
    var ss shopSdks
    if err := json.Unmarshal(bytes, &ss);err != nil {
        return nil
    }
    m := make(map[string]Sdk)
    for _,s := range ss.Sdks {
        m[s.Id] = s
    }
    return m
}

func GetLocalSdks() []map[string]interface{} {
    var algos = []map[string]interface{}{}


 service/SdkDownLoad_test.go

@@ -1 +1,11 @@
package service

import (
    "fmt"
    "testing"
)

func TestGetSdkList(t *testing.T) {
    list := GetSdkList("")
    fmt.Println("sdks:", list)
}

 service/SdkInstall.go

@@ -9,6 +9,8 @@
    "os"
    "path"
    "path/filepath"
    "strconv"
    "strings"
    "webserver/extend/config"
    "webserver/extend/util"
)
@@ -23,17 +25,21 @@
    if config.Server.PatchPath != "" {
        configPatchPath = config.Server.PatchPath
    } else {
        configPatchPath = "/opt/vasystem/patch"
        configPatchPath = "../patch"
    }

    filenameWithSuffix := path.Base(filename)
    ext := path.Ext(filenameWithSuffix)

    zipFilePath := configPatchPath + "/"+identifier+ext
    unPackTargetPath := configPatchPath+"/"+identifier+"_basic/"
    if util.Exists(zipFilePath) {
        strMd5, e := util.FileMd5(zipFilePath)
        if e !=nil || strMd5 == "" {
            go os.Remove(zipFilePath)
            if util.DirExists(unPackTargetPath) {
                go os.RemoveAll(unPackTargetPath)
            }
            return false,errors.New("获取安装包md5失败")
        }
        if strMd5 == identifier {
@@ -44,6 +50,9 @@

        } else {
            go os.Remove(zipFilePath)
            if util.DirExists(unPackTargetPath) {
                go os.RemoveAll(unPackTargetPath)
            }
            logger.Debug("strMd5 is", strMd5,"identifier is",identifier,"not equal")
            return false,errors.New("校验安装文件失败")
        }
@@ -58,7 +67,7 @@
    if config.Server.PatchPath != "" {
        configPatchPath = config.Server.PatchPath
    } else {
        configPatchPath = "/opt/vasystem/patch"
        configPatchPath = "../patch"
    }
    //1.解压缩更新包
    unPackTargetPath := configPatchPath+"/"+identifier+"_basic/"
@@ -122,19 +131,50 @@
                        //3.将算法so、依赖文件、zconf、
                        soM := make(map[string]SdkDef)
                        var skDefArr []SdkDef
                        nInsM := make(map[string]string)
                        aInsM := make(map[string]string)
                        if err = json.Unmarshal(defB, &skDefArr);err == nil {
                            //注册算法信息和算法参数到dbserver
                            var sdkApi dbapi.SdkApi
                            for _,skd := range skDefArr {
                                //注册算法信息和算法参数到dbserver
                                //先看此算法有没有注册，如果已安装，比对版本号
                                bf, curDbSdk := sdkApi.GetById(skd.Def.Id)
                                if bf {
                                    bytes, _ := json.Marshal(curDbSdk)
                                    var sm map[string]interface{}
                                    unE :=  json.Unmarshal(bytes, &sm)
                                    if unE != nil {
                                        continue
                                    }
                                    sdkVersion, ok := sm["version"]
                                    if ok { //有版本号
                                        vStr := sdkVersion.(string)
                                        if shouldVersionBeUpgrade(vStr, skd.Def.Version) {
                                            nInsM[skd.Def.SdkType] = skd.Def.SdkType
                                            aInsM[skd.Def.Id] = skd.Def.Id
                                        }
                                    } else {
                                        nInsM[skd.Def.SdkType] = skd.Def.SdkType
                                        aInsM[skd.Def.Id] = skd.Def.Id
                                    }
                                } else {
                                    nInsM[skd.Def.SdkType] = skd.Def.SdkType
                                    aInsM[skd.Def.Id] = skd.Def.Id
                                }


                                if _,ok := soM[skd.Def.SdkType];!ok {
                                    soM[skd.Def.SdkType] = skd
                                }
                                if _,ok := aInsM[skd.Def.Id]; ok {
                                srv := SdkRegisterVo{
                                    Id: skd.Def.Id,
                                    SdkType: skd.Def.SdkType,
                                    SdkName: skd.Def.SdkName,
                                    Icon: skd.Def.Icon,
                                    Url: skd.Def.Url,
                                        IconBlob: skd.Def.IconBlob,
                                        Version: skd.Def.Version,
                                }
                                for _,ag := range skd.Args {
                                    sra := SdkRegisterArgVo{
@@ -158,8 +198,9 @@
                                logger.Debug("sdkApi.Register paramBody:", paramBody)
                                sdkApi.Register(paramBody) //将算法注册到数据库中
                            }
                            zconfPath := "/opt/vasystem/bin/zconf/"
                            libPath := "/opt/vasystem/libs/"
                            }
                            zconfPath := "./zconf/"
                            libPath := "../libs/"
                            if !util.DirExists(zconfPath) {
                                os.MkdirAll(zconfPath, 0777)
                            }
@@ -167,6 +208,7 @@
                                os.MkdirAll(libPath, 0777)
                            }
                            for sdkType,_ := range soM {
                                if _,iOk := nInsM[sdkType];iOk {
                                //先解压SdkType.tar.gz文件
                                if unTarGzE := util.UnTarGz(unPackPath+sdkType+".tar.gz", unPackPath);unTarGzE == nil {
                                    //复制json启动文件
@@ -179,12 +221,15 @@
                                        }
                                    }
                                    if util.DirExists(unPackPath+sdkType+"/models") {
                                        if _,cE := util.CopyDirByCmd(unPackPath+sdkType+"/models", "/opt/vasystem/bin/zconf"); cE != nil {
                                            if _,cE := util.CopyDirByCmd(unPackPath+sdkType+"/models", "./zconf"); cE != nil {
                                            return false, cE
                                        }
                                    }
                                } else {
                                    logger.Debug("unTarGzE sdkType.tar.gz err:", unTarGzE)
                                    }
                                } else {
                                    logger.Debug("upper version has been installed")
                                }
                            }

@@ -208,6 +253,49 @@
    }

    return true, nil
}

func shouldVersionBeUpgrade(curVersion,dstVersion string) bool {
    if curVersion == "" {
        return true
    }
    if dstVersion == "" {
        return false
    }
    curIVArr := strings.Split(curVersion, ".")
    dstIVArr := strings.Split(dstVersion, ".")
    if len(curIVArr) !=3 || len(dstIVArr) != 3{
        return false
    }
    cH,cEH := strconv.Atoi(curIVArr[0])
    cM,cEM := strconv.Atoi(curIVArr[1])
    cL,cEL := strconv.Atoi(curIVArr[2])

    dH,dEH := strconv.Atoi(curIVArr[0])
    dM,dEM := strconv.Atoi(curIVArr[1])
    dL,dEL := strconv.Atoi(curIVArr[2])
    if cEH !=nil || cEM != nil || cEL != nil || dEH != nil ||dEM !=nil || dEL !=nil {
        return false
    }
    if cH > dH {
        return false
    } else if cH == dH {
        if cM > dM {
            return false
        } else if cM == dM {
            if cL > dL {
                return false
            } else if cL == dL {
                return false
            } else {
                return true
            }
        } else {
            return true
        }
    } else {
        return true
    }
}


@@ -243,6 +331,15 @@
    Enable             bool         `gorm:"column:enable;default:1" json:"enable"`
    DelFlag         int         `gorm:"column:del_flag;default:0" json:"del_flag"`
    Env             string         `gorm:"column:env" json:"env"` //运行环境及结果说明，json格式，包含so_file_path,runtime,param,depends(cuda版本，opencv版本，tensorflow版本等)
    IconBlob         string         `gorm:"column:iconBlob;type:text;" json:"iconBlob"` //图片
    Version         string         `gorm:"column:version;type:varchar(50);default:'1.0';" json:"version"` //版本号
}

type SdkInsOrUpgrade struct {
    Sdk
    RemoteVersion     string          `json:"remoteVersion"` //商城仓库版本号
    Installed         bool          `json:"installed"` //是否已安装
    IsUpgrade         bool          `json:"isUpgrade"` //是否需要升级
}

type SdkArgEntity struct {
@@ -273,6 +370,8 @@
    Args    []SdkRegisterArgVo `json:"args"` //算法参数
    Icon    string `json:"icon"`       //算法图标
    Url     string `json:"url"`                       //算法下载地址
    IconBlob         string                 `json:"iconBlob"` //图片
    Version         string                 `json:"version"` //版本号
}

type SdkRegisterArgVo struct {