pubsub/task.git

			@@ -1,225 +1,300 @@
			package sdk

			import (
			"context"
			"encoding/json"
			"fmt"
			"github.com/long/test/httpclient"
			"github.com/long/test/tasktag"
			"github.com/long/test/util"
			"time"
			"taskpubsub/tasktag"
			"taskpubsub/util"

			"nanomsg.org/go-mangos"
			"nanomsg.org/go-mangos/protocol/pair"
			"nanomsg.org/go-mangos/transport/ipc"
			"nanomsg.org/go-mangos/transport/tcp"
			"github.com/gogo/protobuf/proto"

			"basic.com/pubsub/protomsg.git"
			"basic.com/valib/deliver.git"
			"basic.com/valib/logger.git"
			)

			var SocketManage = make(map[string]SocketContext)
			const (
			postPush = "_1"
			postPull = "_2"
			)

			type SocketContext struct {
			Sock mangos.Socket
			Context context.Context
			Cancel context.CancelFunc
			var SocketManage = make(map[string]util.SocketContext)
			var SdkMap = make(map[string]chan protomsg.SdkMessage)

			var shm bool = false

			var innerRecvTopic = []string{
			"facedetect-sdk-no-track", //to sdk-no-track 以图搜图
			}

			func Init() {
			var innerSendTopic = []string{
			"facedetect-sdk-no-track", //to sdk-no-track 以图搜图
			"virtual-faceextract-sdk-pull", //to web 以图搜图
			}

			sdklist := SdkAll() //获取所有sdk
			fmt.Println("sdk list have: ", sdklist)
			func initInnerTopic() {
			// for _, sendTopic := range innerSendTopic {
			// createSdkSendServerAndListen(sendTopic)
			// }

			SdkCreateTopic(sdklist) // 创建主题
			// for _, recvTopic := range innerRecvTopic {
			// createSdkRecvServerAndListen(recvTopic)
			// }
			}

			for _, sdkid := range sdklist { // 创建sdk server
			sdkid, socket, err := NewSdkListen(sdkid, "tcp", "192.168.1.124", 0)
			if err != nil {
			continue
			}
			// 接受管道数据 ==》发送给对应的进程
			go send(sdkid, socket, SdkMap[sdkid])
			func Init(useShm bool) {

			//从对应进程接受数据 == 》重新送回到管道
			go Recv(socket)
			shm = useShm

			logger.Info("============= init sdk info =====================")
			for _, sdkid := range util.Sdklist { // 创建sdk server
			createSdkTopicAndServer(sdkid)
			logger.Info()
			}

			// 手动输入的主题
			initInnerTopic()

			// es
			SdkMap["es"] = make(chan protomsg.SdkMessage)
			logger.Info("create es channel: ")
			go DealEsTopic()

			go autoUpdateSdk(util.Sdkflag)
			}

			func createSdkTopicAndServer(sdkid string) {
			createSdkSendServerAndListen(sdkid)
			createSdkRecvServerAndListen(sdkid)
			}

			func deleteSdkTopicAndServer(id string) {
			if _, isExist := SdkMap[id]; isExist { //存在
			close(SdkMap[id])
			delete(SdkMap, id)
			logger.Info("删除主题 sdk: ", id)
			}

			if _, isExist := SocketManage[id]; isExist { //存在
			SocketManage[id].Cancel()
			delete(SocketManage, id)
			logger.Info("删除server sdk: ", id)
			}
			}

			func createSdkSendServerAndListen(id string) {
			if _, isExist := SdkMap[id]; !isExist { //不存在
			SdkMap[id] = make(chan protomsg.SdkMessage)
			logger.Info("create", id)
			}

			url := "ipc:///tmp/" + id + postPush + ".ipc"
			m := deliver.PushPull
			if shm {
			m = deliver.Shm
			url = id + postPush
			}

			logger.Info("SDK URL: ", url)

			socket, err := util.NewSocketListen(int(m), url, shm)
			if err != nil {
			delete(SdkMap, id)
			logger.Error(id, "create socket error!")
			return
			}
			SocketManage[id] = socket

			go Send(id, socket, SdkMap[id])
			}

			func createSdkRecvServerAndListen(id string) {
			if _, isExist := SdkMap[id]; !isExist { //不存在
			SdkMap[id] = make(chan protomsg.SdkMessage)
			logger.Info("create", id)
			}

			url := "ipc:///tmp/" + id + postPull + ".ipc"
			m := deliver.PushPull
			if shm {
			m = deliver.Shm
			url = id + postPull

			}

			go es(SdkMap["es"])
			socket, err := util.NewSocketListen(int(m), url, shm)

			if err != nil {
			delete(SdkMap, id)
			logger.Error(id, "create socket error!")
			return
			}
			SocketManage[id] = socket

			go Recv(socket)
			}

			//单独处理 es 主题的情况
			func es(sdkmsgchan chan SdkMessage) {
			for data := range sdkmsgchan {
			fmt.Println("this data is finish all sdk! ", data)
			func DealEsTopic() {
			for {
			select {
			case <-SdkMap["es"]:
			//logger.Info("es finanl sdk!")
			}
			}

			}

			//动态处理
			func AutoDelSdk(Newsdklist []string) {
			var oldSdk []string
			for key, _ := range SdkMap {
			oldSdk = append(oldSdk, key)
			}
			sdkChanDel := util.Difference(oldSdk, Newsdklist)
			func autoUpdateSdk(sdkflag chan bool) {

			for key, op := range sdkChanDel {
			if op == "add" {
			SdkMap[key] = make(chan SdkMessage)
			fmt.Println("创建主题 sdk: ", key)
			} else {
			close(SdkMap[key])
			delete(SdkMap, key)
			fmt.Println("删除主题 sdk: ", key)
			for _ = range sdkflag {
			logger.Info("test autodelsdk")
			var oldSdkList []string
			for key := range SdkMap {
			oldSdkList = append(oldSdkList, key)
			}

			newSdkList := util.Sdklist

			// 手动添加的全部加上
			for _, sendTopic := range innerSendTopic {
			newSdkList = append(newSdkList, sendTopic)
			}
			for _, recvTopic := range innerRecvTopic {
			newSdkList = append(newSdkList, recvTopic)
			}

			sdkListUpdate := util.Difference(oldSdkList, newSdkList)
			logger.Info(sdkListUpdate)

			for key, op := range sdkListUpdate {
			if op == "add" {
			createSdkTopicAndServer(key)
			} else {
			deleteSdkTopicAndServer(key)
			}
			}
			}
			}

			//主题
			var SdkMap = make(map[string]chan SdkMessage)

			// 发送给算法进程的结构
			type SdkMessage struct {
			Cid string
			Tasklab tasktag.TaskLabel
			Data []byte
			}

			//sdk数据加工器
			func SdkData(cid string, taskid string, data []byte) (sdkmsg SdkMessage) {
			func ToSdkMsg(cid string, caddr string, taskid string, data []byte) protomsg.SdkMessage {
			var sdkmsg = protomsg.SdkMessage{}
			sdkmsg.Cid = cid
			sdkmsg.Tasklab = tasktag.TaskMapLab[taskid]
			sdkmsg.Data = data
			return
			sdkmsg.Caddr = caddr
			if val, ok := tasktag.TaskLabelMap.Load(taskid); !ok {
			sdkmsg.Tasklab = nil
			return sdkmsg
			} else {
			sdkmsg.Tasklab = val.(*protomsg.TaskLabel)
			sdkmsg.Data = data
			}
			return sdkmsg
			}

			//sdk数据分发器
			func SdkSendTopic(sdkmsg SdkMessage) (sdksend string) {
			if sdkmsg.Tasklab.Index < len(sdkmsg.Tasklab.Sdkids) {
			sdksend = sdkmsg.Tasklab.Sdkids[sdkmsg.Tasklab.Index]
			} else {
			sdksend = "es"
			}
			fmt.Println()
			fmt.Println("分发的主题是： ", sdksend)
			fmt.Println()
			return
			}

			// 调用 http 借口获取摄像机信息
			func SdkAll() (sdklist []string) {
			sdklist = httpclient.GetSdk("http://127.0.0.1:8000/data/api-v/sdk/findskdid")
			return
			}

			// 创建主题
			func SdkCreateTopic(sdklist []string) (err error) {
			for _, sdkid := range sdklist {
			SdkMap[sdkid] = make(chan SdkMessage)
			fmt.Println("create sdk channel: ", sdkid)
			}

			SdkMap["es"] = make(chan SdkMessage)
			fmt.Println("create es channel: ")

			return nil
			}

			var UrlPort = 9000

			// 创建 sdk server listen
			func NewSdkListen(sdkid string, protocol string, ip string, port int) (sid string, socket SocketContext, err error) {

			ctx, cancel := context.WithCancel(context.Background())

			var url string

			socket.Context = ctx
			socket.Cancel = cancel

			switch protocol {
			case "tcp":
			if port == 0 {
			port = UrlPort
			UrlPort++
			func GetSdkSendTopic(sdkmsg protomsg.SdkMessage) (sendTopic string) {
			if int(sdkmsg.Tasklab.Index) < len(sdkmsg.Tasklab.Sdkinfos) {
			sendTopic = sdkmsg.Tasklab.Sdkinfos[sdkmsg.Tasklab.Index].Ipcid
			if "Yolo" != sdkmsg.Tasklab.Sdkinfos[sdkmsg.Tasklab.Index].Sdktype &&
			"FaceDetect" != sdkmsg.Tasklab.Sdkinfos[sdkmsg.Tasklab.Index].Sdktype &&
			"FaceCompare" != sdkmsg.Tasklab.Sdkinfos[sdkmsg.Tasklab.Index].Sdktype {
			//if sdkmsg.Tasklab.Sdkinfos[sdkmsg.Tasklab.Index].Sdktype == "" {
			logger.Info("----------Sdktype:yitusoutu")
			logger.Info("分发的主题:", sendTopic, "!Sdktype：", sdkmsg.Tasklab.Sdkinfos[sdkmsg.Tasklab.Index].Sdktype)
			}
			url = fmt.Sprintf("%s://%s:%d", protocol, ip, port)

			case "ipc":
			url = fmt.Sprintf("%s://%s", sdkid)
			}
			fmt.Printf("sdkid= %s url=%s\n", sdkid, url)

			if socket.Sock, err = pair.NewSocket(); err != nil {
			fmt.Println(sdkid, "can't get new pair socket: ", err.Error())
			return sdkid, socket, err
			} else {
			sendTopic = "es"
			}

			socket.Sock.SetOption(mangos.OptionMaxRecvSize, 3210241024)
			socket.Sock.SetOption(mangos.OptionWriteQLen, 10)
			socket.Sock.SetOption(mangos.OptionReadQLen, 10)

			socket.Sock.AddTransport(tcp.NewTransport())
			socket.Sock.AddTransport(ipc.NewTransport())

			if err = socket.Sock.Listen(url); err != nil {
			fmt.Println("socket lisnte error ", sdkid)
			}
			SocketManage[sdkid] = socket
			return sdkid, socket, err
			return sendTopic
			}

			func Recv(socket SocketContext) {
			socket.Sock.SetOption(mangos.OptionRecvDeadline, 1*time.Second)
			func Recv(socket util.SocketContext) {

			tryCount := 0

			var repsdkmsg = protomsg.SdkMessage{}
			for {
			select {
			case <-socket.Context.Done():
			fmt.Println("socket close")
			logger.Info("socket close")
			return
			default:
			if msg, err := socket.Sock.Recv(); err != nil {
			//fmt.Printf("%s ", err)
			if socket.UseSHM {
			if tryCount > util.SHMMaxTryCount {
			socket.Sock.Close()
			socket, err = util.NewSocketListen(socket.Mode, socket.URL, socket.UseSHM)
			logger.Info("SDK RECV SHM TRY :", tryCount, " RESTART IT")

			tryCount = 0

			continue
			}
			tryCount++
			}
			continue
			} else {

			var repsdkmsg SdkMessage
			var reps interface{}

			err = json.Unmarshal(msg, &reps)
			err = proto.Unmarshal(msg, &repsdkmsg)
			if err != nil {
			logger.Error("unmarshal error: ", err)
			continue
			}

			switch v := reps.(type) {
			case map[string]interface{}:
			//调用计算函数，分发给下一个主题

			json.Unmarshal(msg, &repsdkmsg)
			nexttopic := SdkSendTopic(repsdkmsg)
			SdkMap[nexttopic] <- repsdkmsg
			case string:
			fmt.Println("this string is: ", v)
			repsdkmsg.Tasklab.Index++
			//调用计算函数，分发给下一个主题
			nexttopic := GetSdkSendTopic(repsdkmsg)
			if "facedetect-sdk-no-track" == nexttopic \|\| "virtual-faceextract-sdk-pull" == nexttopic {
			logger.Info("nexttopic:", nexttopic)
			}
			SdkMap[nexttopic] <- repsdkmsg
			}
			}
			}
			}

			func send(sdkid string, socket SocketContext, in chan SdkMessage) {
			var v SdkMessage
			var b []byte
			func Send(sdkid string, socket util.SocketContext, in chan protomsg.SdkMessage) {

			tryCount := 0

			for {
			select {
			case <-socket.Context.Done():
			fmt.Println("socket is close")
			case v = <-in:
			b, _ = json.Marshal(v)
			fmt.Printf("从管道sdkid=%s 接受数据 %d\n", sdkid, len(v.Data))
			if err := socket.Sock.Send(b); err != nil {
			fmt.Println("failed send")
			logger.Info("socket is close")
			return
			case v, ok := <-in:
			if ok {
			data, err := v.Marshal()
			if err != nil {
			logger.Error("proto marshal error ", err)
			continue
			}

			if err := socket.Sock.Send(data); err != nil {
			if "facedetect-sdk-no-track" == sdkid \|\| "virtual-faceextract-sdk-pull" == sdkid {
			logger.Error("failed send:sdkid=", sdkid)
			}

			if socket.UseSHM {
			if tryCount > util.SHMMaxTryCount {
			socket.Sock.Close()
			socket, err = util.NewSocketListen(socket.Mode, socket.URL, socket.UseSHM)
			logger.Info("SDK SEND SHM TRY :", util.SHMMaxTryCount, " RESTART IT")

			tryCount = 0

			continue
			}
			tryCount++
			}

			continue
			}
			if "facedetect-sdk-no-track" == sdkid \|\| "virtual-faceextract-sdk-pull" == sdkid {
			logger.Info(sdkid, " send success: ", len(data))
			}
			} else {
			logger.Debug(sdkid, " 主题关闭, 关闭send()")
			return
			}
			fmt.Printf("sdkid = %s ,send success:%d \n", sdkid, len(v.Data))
			}
			}
			}