package service import ( "bytes" "crypto/md5" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/http" "os" "path" "path/filepath" "strconv" "strings" "time" "vamicro/appcenter-service/models" "vamicro/config" "vamicro/extend/util" "basic.com/valib/bhomeclient.git" "basic.com/valib/bhomedbapi.git" "basic.com/valib/licence.git" "basic.com/valib/logger.git" "github.com/mitchellh/mapstructure" ) // 从服务器上下载文件到临时目录,校验之后如果完整将其拷贝到目标目录下 func DownSo(url string) (bool, error) { resp, err := http.Get(url) if err != nil { logger.Error("获取文件失败") return false, err } // 从resp中读出zip文件解压缩,解出face.so,face.txt,然后把解压出的so用MD5编码出一个temp.txt文件,与解压出的so.txt文件比对, body, err := ioutil.ReadAll(resp.Body) if err != nil { logger.Error("读取resp.body失败") return false, err } CopyFile(body, "/opt/temp/temp.zip") util.DeCompress("/opt/temp/temp.zip", "/opt/temp") fileName := GetFileNameFromUrl(url, false) md5str, err1 := File2md5("/opt/temp/" + fileName + ".so") if err1 != nil { logger.Error(err1) return false, err1 } md5str_origin, err2 := ioutil.ReadFile("/opt/temp/" + fileName + ".txt") if err2 != nil { logger.Error("读取解压后的md5文件失败") return false, err2 } flag := CompareMd5([]byte(md5str), md5str_origin) if flag { logger.Info("两次MD5编码一致!") } else { logger.Debug("两次MD5编码不一致,请重新下载") return false, nil } // 从url中截取soName soName := GetFileNameFromUrl(url, true) f, err := os.Create("/opt/workspace/ruleprocess/algorithm/" + soName) if err != nil { logger.Error("在项目目录下创建so文件失败") return false, err } data, _ := ioutil.ReadFile("/opt/temp/" + soName) _, err4 := f.Write(data) if err4 != nil { logger.Error("复制文件出错") return false, err4 } return true, nil } func CopyFile(byte []byte, dst string) (w int64, err error) { dstFile, err := os.Create(dst) if err != nil { logger.Debug(err.Error()) return } defer dstFile.Close() return io.Copy(dstFile, bytes.NewReader(byte)) } // 指定目录的文件生成相应的MD5码文件 func File2md5(filename string) (string, error) { // 文件生成MD5加密加密文件 file, err := os.Open(filename) if err != nil { logger.Debug("os Open error") return "", err } md5 := md5.New() _, err = io.Copy(md5, file) if err != nil { logger.Debug("io copy error") return "", err } md5Str := hex.EncodeToString(md5.Sum(nil)) return md5Str, nil } // 从url中截取出文件名,参数是是否带后缀 func GetFileNameFromUrl(url string, withSuffix bool) string { fileName := strings.Split(url, "/")[len(strings.Split(url, "/"))-1] if withSuffix { return fileName } else { withoutSuffix := strings.Split(fileName, ".")[0] return withoutSuffix } } // 比较两个MD5编码是否一致 func CompareMd5(value1 []byte, value2 []byte) bool { num := bytes.Compare(value1, value2) if num == 0 { return true } else { return false } } //installedFlag 如果是true,表示只查本地的,不需要比对查看未安装的算法 func GetSdkList(sdkName string, installedFlag, userId string) []SdkInsOrUpgrade { //logger.Debug("installedFlag:", installedFlag) t := time.Now() var api bhomedbapi.UserApi useIconTyp := 1 if userId != "" { found, userInfo := api.GetUserProfile(userId) logger.Debugf("GetSdkList GetUserProfile userInfo=%v, t=%v", userInfo, time.Since(t)) if found { ub, e := json.Marshal(userInfo) if e == nil { type UIconDef struct { UseIconType int `json:"useIconType"` } var ucd UIconDef if e = json.Unmarshal(ub, &ucd); e == nil { useIconTyp = ucd.UseIconType logger.Debug("useIconTyp already 2", useIconTyp) } } } } var sdkE models.Sdk t = time.Now() sdkAll, _ := sdkE.FindAll(sdkName) //本地已安装所有算法,带顺序 logger.Debugf("GetSdkList FindAll sdkName=%v, len(sdkAll)=%v, t=%v", sdkName, len(sdkAll), time.Since(t)) if sdkAll == nil { sdkAll = make([]models.Sdk, 0) } installedSdks := make([]SdkInsOrUpgrade, len(sdkAll)) iv := make(map[string]SdkVB) localSdkIdArr := make([]string, 0) localSdkM := make(map[string]SdkInsOrUpgrade) t = time.Now() for idx, ls := range sdkAll { siou := SdkInsOrUpgrade{ Installed: true, } var sArg models.SdkArgEntity args, e1 := sArg.FindBySdk(ls.Id) if e1 == nil { siou.Args = args } else { siou.Args = make([]models.SdkArgEntity, 0) } siou.Sdk = ls siou.ProtoRuleSo = ls.RuleSo if useIconTyp == 2 && siou.IconBlob2 != "" { siou.IconBlob = siou.IconBlob2 siou.IconBlob2 = "" } localSdkM[siou.Id] = siou iv[siou.Id] = SdkVB{ Version: ls.Version, BaseVersion: ls.BaseVersion, } installedSdks[idx] = siou localSdkIdArr = append(localSdkIdArr, siou.Id) } logger.Debugf("GetSdkList FindBySdk len(sdkAll)=%v, t=%v", len(sdkAll), time.Since(t)) nInsSdks := make([]SdkInsOrUpgrade, 0) installedFlagBool, _ := strconv.ParseBool(installedFlag) if !installedFlagBool { t = time.Now() // ***这个地方需要有效率问题 upSdks, insSdks := findAllMySdk(localSdkIdArr, iv, sdkName) logger.Debugf("GetSdkList findAllMySdk len(localSdkIdArr)=%v, len(iv)=%v, sdkName=%v, t=%v", len(localSdkIdArr), len(iv), sdkName, time.Since(t)) t = time.Now() for _, sdk := range upSdks { if v, ok := localSdkM[sdk.Id]; ok { //本地已安装 ls := v ls.SdkName = sdk.SdkName //以商城上的产品名称为准 ls.Installed = true if sdk.CanUpOrIns { ls.ProgressMsg = "" ls.IsUpgrade = shouldVersionBeUpgrade(ls.Version, sdk.Version) ls.RemoteVersion = sdk.Version //远端的版本号 if ip, insB := insIngMap.Load(sdk.Id); insB { iPgs := ip.(*InsProgress) ls.ProgressMsg = iPgs.Status + strconv.Itoa(iPgs.Progress) + "%" } } if useIconTyp == 2 && ls.IconBlob2 != "" { ls.IconBlob = ls.IconBlob2 ls.IconBlob2 = "" } localSdkM[sdk.Id] = ls for index, es := range installedSdks { if es.Id == sdk.Id { installedSdks[index] = ls break } } } } logger.Debugf("GetSdkList shouldVersionBeUpgrade len(upSdks)=%v, t=%v", len(upSdks), time.Since(t)) t = time.Now() for _, sdk := range insSdks { //本地未安装,但可以安装列表 if sdk.CanUpOrIns { bIns := SdkInsOrUpgrade{ RemoteVersion: sdk.Version, IsUpgrade: false, Installed: false, ProgressMsg: "", ProtoRuleSo: sdk.RuleSo, } if ip, insB := insIngMap.Load(sdk.Id); insB { iPgs := ip.(*InsProgress) bIns.ProgressMsg = iPgs.Status + strconv.Itoa(iPgs.Progress) + "%" } if useIconTyp == 2 && sdk.Sdk.IconBlob2 != "" { sdk.Sdk.IconBlob = sdk.Sdk.IconBlob2 sdk.Sdk.IconBlob2 = "" } bIns.Sdk = sdk.Sdk nInsSdks = append(nInsSdks, bIns) } } logger.Debugf("GetSdkList Load len(insSdks)=%v, t=%v", len(insSdks), time.Since(t)) } // 如果参数为空, 返回全部 if installedFlag == "" { return append(installedSdks, nInsSdks...) } // 返回所有未安装 if !installedFlagBool { return nInsSdks } // 返回所有已安装 return installedSdks } func queryDatabase(sdkName string) ([]map[string]interface{}, map[string]bool) { var api bhomedbapi.SdkApi ids := make(map[string]bool) sdks := []map[string]interface{}{} // 查询已经安装的算法 data := api.FindAll(sdkName) for _, sdk := range data { ids[sdk.Id] = true sdks = append(sdks, map[string]interface{}{ "id": sdk.Id, "ipc_id": sdk.IpcId, "sdk_type": sdk.SdkType, "sdk_name": sdk.SdkName, "icon": sdk.Icon, "enable": sdk.Enable, "installed": true, }) } return sdks, ids } type shopSdks struct { Sdks []SdkWithArg `json:"sdks"` CanInsList []SdkWithArg `json:"canInsList"` } type SdkVB struct { Version string `json:"version"` BaseVersion string `json:"baseVersion"` } const ( token = "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjQ3NDUwMjU5MjMsInVzZXIiOiJ7XCJpZFwiOlwiZTZjY2QzNmQtNGYxNi00NmZjLTg4ZDUtMDczNjU4NjZkMjA1XCIsXCJwZXJtaXNzaW9uc1wiOltcInByb2R1Y3RNYW5nZTpwdWJsaXNoXCIsXCJjb2RlTWFuZ2U6dmlld1wiLFwiZGV2aWNlTWFuYWdlOmFkZFwiLFwiYWRtaW5NYW5hZ2VcIixcIm9yZGVyTWFuZ2VcIixcImRldmljZU1hbmFnZTp2aWV3XCIsXCJwcm9kdWN0TWFuZ2U6YWRkXCIsXCJhZG1pbk1hbmFnZTp2aWV3XCIsXCJjb2RlTWFuZ2U6YWRkXCIsXCJwcm9kdWN0TWFuZ2U6b2ZmU2FsZVwiLFwib3JkZXJNYW5nZTpjYW5jZWxcIixcInByb2R1Y3RDZW50ZXI6ZG93bmxvYWRcIixcInByb2R1Y3RDZW50ZXI6YnV5XCIsXCJwcm9kdWN0TWFuZ2U6dmlld1wiLFwiYXBpXCIsXCJob21lXCIsXCJvcmRlck1hbmdlOnBheVwiLFwiYWRtaW5NYW5hZ2U6YWRkXCIsXCJvcmRlck1hbmdlOmRvd25sb2FkXCIsXCJwcm9kdWN0Q2VudGVyXCIsXCJkZXZpY2VNYW5hZ2U6dW5iaW5kXCIsXCJvcmRlck1hbmdlOnZpZXdcIixcImFkbWluTWFuYWdlOmVkaXRcIixcImRldmljZU1hbmFnZVwiLFwidmlwTWFuYWdlOmFkZFwiLFwidmlwTWFuYWdlOnZpZXdcIixcInByb2R1Y3RDZW50ZXI6dmlld1wiLFwidmlwTWFuYWdlOmVkaXRcIixcInZpcE1hbmFnZVwiLFwicHJvZHVjdE1hbmdlOmVkaXRcIixcImNvZGVNYW5nZVwiLFwicHJvZHVjdE1hbmdlXCJdLFwidXNlcm5hbWVcIjpcImJhc2ljXCJ9In0.vwjAFkWuEyadRLvIOGK8LFE3MjpY3SQ7j6AlTXnQDG8" ) func findAllMySdk(localSdkIdArr []string, iv map[string]SdkVB, inputTxt string) ([]SdkWithArg, []SdkWithArg) { t := time.Now() url := "http://" + util.GetShopUrl() + "/data/api-s/sdk/findAllMySdk" machineCode := licence.GetMachineCode() defer func() { logger.Debugf("findAllMySdk url=%v, costTime=%v", url, time.Since(t)) }() //获取平台类型 paramBody := map[string]interface{}{ "serverId": config.Server.AnalyServerId, "machineCode": machineCode, "localSdkIdArr": localSdkIdArr, "iv": iv, "inputTxt": inputTxt, } header := map[string]string{ "Authorization": token, } respBody, err := util.DoPostRequest(url, util.CONTENT_TYPE_JSON, paramBody, nil, header, time.Second*20) if err != nil { logger.Debug("DoPostRequest err:", err) return nil, nil } var res bhomedbapi.Result if err = json.Unmarshal(respBody, &res); err != nil { logger.Debug("unmarshal err:", err) return nil, nil } bytes, _ := json.Marshal(res.Data) var ss shopSdks if err := json.Unmarshal(bytes, &ss); err != nil { logger.Debug("unmarshal err:", err) return nil, nil } return ss.Sdks, ss.CanInsList } func GetLocalSdks() []map[string]interface{} { var algos = []map[string]interface{}{} AlgorithmFiles := "/opt/vasystem/bin/algorithm/*.json" files, err := filepath.Glob(AlgorithmFiles) if err != nil { fmt.Println("Cannot access algorithm json files: No such file or directory") return algos } for _, filename := range files { algo := make(map[string]interface{}) f, err := ioutil.ReadFile(filename) if err != nil { return algos } if err := json.Unmarshal(f, &algo); err != nil { return algos } algos = append(algos, map[string]interface{}{ "id": algo["sdkId"], "ipc_id": algo["ipcId"], "sdk_type": algo["sdkType"], "sdk_name": algo["sdkName"], "icon": algo["icon"], "args": algo["sdkArgs"], }) } return algos } func FindLocalSdkSoById(id string) map[string]interface{} { localAlgos := GetLocalSdks() for _, sdk := range localAlgos { if sdk["id"].(string) == id { return sdk } } return nil } type downOrUpResp struct { Url string `json:"url"` Md5 string `json:"md5"` Size uint64 `json:"size"` } //下载或者升级算法 func DownloadOrUpgrade(id string, h *bhomeclient.WrapperHandler) (bool, error) { if _, ok := insIngMap.Load(id); ok { insIngMap.Delete(id) } PreDownloading = true //防止清理程序把升级包清理了 rb := false ip := &InsProgress{ Status: InsStatus_Downloading, Progress: 0, } insIngMap.Store(id, ip) //放到正在安装列表中 defer func() { if !rb { insIngMap.Delete(id) } PreDownloading = false }() ver := "" baseVer := "" var apE models.App r1, _ := apE.SelectById(id) if r1 > 0 { ver = apE.Version } else { var sdkE models.Sdk if r2, _ := sdkE.SelectById(id); r2 > 0 { ver = sdkE.Version baseVer = sdkE.BaseVersion } } url := "http://" + util.GetShopUrl() + "/data/api-s/sdk/downloadOrUpgrade" machineCode := licence.GetMachineCode() if machineCode == "" { logger.Debug("获取机器码失败") return rb, errors.New("获取机器码失败") } paramBody := map[string]interface{}{ "modId": id, "machineCode": machineCode, "serverId": config.Server.AnalyServerId, "version": ver, "baseVersion": baseVer, } header := map[string]string{ "Authorization": token, } respBody, err := util.DoPostRequest(url, util.CONTENT_TYPE_JSON, paramBody, nil, header, time.Second*60) if err != nil { logger.Debug("DoPostRequest err:", err) return rb, err } var res bhomedbapi.Result if err = json.Unmarshal(respBody, &res); err != nil { logger.Debug("unmarshal err:", err) return rb, err } if !res.Success { logger.Debug("res.Data:", res.Data) return rb, errors.New("请求商城失败") } logger.Debug("res.Data:", res.Data) var resp downOrUpResp if err := mapstructure.Decode(res.Data.(map[string]interface{}), &resp); err != nil { logger.Debug("mapstructure.Decode err:", err) return rb, err } logger.Debug("resp:", resp) if resp.Url == "" || resp.Md5 == "" { return rb, errors.New("获取下载安装包失败") } ip.Size = resp.Size //异步安装 go startInstall(resp, id, ip, h) rb = true return rb, nil } //下载安装(升级)包,验证md5后安装 func startInstall(dor downOrUpResp, id string, iProgress *InsProgress, h *bhomeclient.WrapperHandler) (bool, error) { defer func() { iProgress.IsDone = true time.Sleep(10 * time.Second) insIngMap.Delete(id) }() //1.第一步先下载安装包 configPatchPath := "" if config.Server.PatchPath != "" { configPatchPath = config.Server.PatchPath } else { configPatchPath = "/opt/vasystem/patch" } if !util.DirExists(configPatchPath) { os.Mkdir(configPatchPath, 0777) } filenameWithSuffix := path.Base(dor.Url) ext := path.Ext(filenameWithSuffix) downUrl := "http://" + util.GetShopUrl() + "/files/" + dor.Url iProgress.Status = InsStatus_Downloading //如果检查到预下载文件则不重复下载 gzFilePath := configPatchPath + "/" + dor.Md5 + ext _, err := os.Stat(gzFilePath) if nil != err { if os.IsNotExist(err) { err := DownloadFile(gzFilePath, downUrl, iProgress) if err != nil { logger.Error("DownloadFile err:", err, " gzFilePath:", gzFilePath, " downUrl:", downUrl) iProgress.Status = InsStatus_Err + " 原因:" + err.Error() return false, err } } } iProgress.Status = InsStatus_PackageChecking //2.下载完成开始安装 if unPackB, unPackErr := unPackPatchPackage(dor.Md5, ext); !unPackB { //解压失败 iProgress.Status = InsStatus_UnPackErr + " 原因:" + unPackErr.Error() return false, unPackErr } //3.校验md5 fileMd5, err := util.FileMd5(gzFilePath) if err != nil { logger.Debug("FileMd5 err:", err) iProgress.Status = InsStatus_CheckErr + " 原因:" + err.Error() return false, err } if fileMd5 != dor.Md5 { logger.Debug("安装包md5校验失败") iProgress.Status = InsStatus_CheckErr + " 原因:md5校验失败" return false, errors.New("安装包md5校验失败") } else { logger.Debug("安装包md5校验成功") } iProgress.Status = InsStatus_Installing _, err = installPack(dor.Md5, ext, h) time.Sleep(2 * time.Second) if err == nil { iProgress.Status = InsStatus_Done return true, nil } else { iProgress.Status = InsStatus_InstallErr + " 原因:" + err.Error() go func() { time.Sleep(3 * time.Second) iProgress.Status = "" iProgress.Progress = 0 insIngMap.Delete(id) //安装失败也表示安装结束了 需要清除信息 }() return false, err } }