#include "nng_wrap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace std; #include #include #include #include #include "nng/compat/nanomsg/nn.h" #include "nng/compat/nanomsg/reqrep.h" #include "nng/compat/nanomsg/pubsub.h" #include "nng/compat/nanomsg/survey.h" namespace nng_wrap { // common function static int client_socket(const string& url, const int protocol, int family=AF_SP){ int sock = nn_socket(family, protocol); if (sock < 0) return sock; int rc = nn_connect(sock, url.c_str()); if (rc < 0) { nn_close(sock); return rc; } return sock; } static void remove_exist(const string& url){ if (url.find("ipc://") == 0){ string address(url); address = address.substr(6); if (access(address.c_str(), F_OK) == 0){ remove(address.c_str()); } } } static int server_socket(const string& url, const int protocol, int family=AF_SP){ int sock = nn_socket(family, protocol); if (sock < 0) return sock; remove_exist(url); int rc = nn_bind(sock, url.c_str()); if (rc < 0) { nn_close(sock); return rc; } return sock; } static void set_socket_timeout(int sock, const int to_ms){ nn_setsockopt(sock, NN_SOL_SOCKET, NN_SNDTIMEO, &to_ms, sizeof(to_ms)); nn_setsockopt(sock, NN_SOL_SOCKET, NN_RCVTIMEO, &to_ms, sizeof(to_ms)); } static int send_and_recv(int sock, const void* in, const int in_len, void** out, int *out_len) { int rc = nn_send(sock, in, in_len, 0); if (rc != in_len) { nn_close(sock); return -1; } char *msg = NULL; rc = nn_recv(sock, &msg, NN_MSG, 0); if (rc < 0) { nn_close(sock); return rc; } nn_close(sock); copy_memory(out, out_len, msg, rc); nn_freemsg(msg); return 0; } ///////////////////////////////////////////////// // simple interface void free_nng(void* data, const int data_len){ if (data){ free(data); } } void copy_memory(void** dest, int *dest_len, const void* src, const int src_len){ char* tmp = (char*)malloc(src_len); memcpy(tmp, src, src_len); *dest = tmp; if(dest_len) *dest_len = src_len; } static string verbose_info{}; #ifndef PRNTVITAG // #define TAG do{ \ // if (verbose_info.length() > 8096) verbose_info.clear(); \ // verbose_info=string("function [")+__FUNCTION__+string("]");}while(0) /* #define PRNTVITAG(msg) do{ \ if (verbose_info.length() > 8096) verbose_info.clear(); \ verbose_info+=string("-> (") + msg + string(")"); \ }while(0) */ #define TAG #define PRNTVITAG(args) #endif void get_last_error(int* ec, void** emsg, int* emsg_len){ *emsg = NULL; *emsg_len = 0; *ec = nn_errno(); const char* msg = nn_strerror(*ec); string strMsg(msg); strMsg = strMsg + "{" + verbose_info + "}"; copy_memory(emsg, emsg_len, strMsg.data(), strMsg.size()); verbose_info.clear(); } /////////////////////////////////////////////////////// // simple request waiting reply static constexpr int timeout_req_rep = 5162; int simple_request(const std::string& url, const void* in, const int in_len, void** out, int *out_len, const int to_ms){ TAG; *out = NULL; *out_len = 0; // printf("------>> simple_request url %s\n", (char*)url.data); int sock = client_socket(url, NN_REQ); if (sock < 0) { PRNTVITAG("client_socket faild"); return false; } int tm = to_ms < timeout_req_rep ? timeout_req_rep : to_ms; set_socket_timeout(sock, tm); auto ret = send_and_recv(sock, in, in_len, out, out_len); // printf("------>> simple_request out data address %p len %d\n", out->data, out->data_len); if (ret < 0) { PRNTVITAG("send_and_recv faild"); return false; } return true; } /////////////////////////////////////////////////////////// // base class #define DISABLE_COPY_AND_ASSIGN(className) className(const className&)=delete; \ className(className&&)=delete; \ className& operator=(const className&)=delete; \ className& operator=(className&&)=delete class _nn{ public: DISABLE_COPY_AND_ASSIGN(_nn); _nn()=default; virtual ~_nn(){ if (socket_ > 0) nn_close(socket_); } int socket_{-1}; string url_{}; }; /////////////////////////////////////////////// // publish struct psmsg{ DISABLE_COPY_AND_ASSIGN(psmsg); psmsg()=delete; psmsg(const std::string& t, std::string&& m) :topic_(t),msg_(std::move(m)){} std::string topic_{}; std::string msg_{}; }; class _ps : public _nn{ public: DISABLE_COPY_AND_ASSIGN(_ps); _ps()=default; virtual ~_ps(){ t_quit_.store(true, memory_order_relaxed); if (t_.joinable()) t_.join(); } thread t_; atomic_bool t_quit_{false}; deque msg_{}; mutex mtx_msg_{}; condition_variable cv_msg_{}; }; static _ps pub_; static int connect_to_center(const string& topic){ if (pub_.socket_ > 0) return pub_.socket_; pub_.url_ = topic; TAG; int sock = client_socket(topic, NN_REQ); if (sock < 0){ PRNTVITAG("client_socket faild"); return -1; } set_socket_timeout(sock, timeout_req_rep); pub_.socket_ = sock; pub_.t_ = thread([]{ while (!pub_.t_quit_.load()) { psmsg *msg{NULL}; { unique_lock l{pub_.mtx_msg_}; pub_.cv_msg_.wait(l, []{ return !pub_.msg_.empty() || pub_.t_quit_.load(); }); if(pub_.t_quit_.load()) break; msg = &pub_.msg_.front(); if (msg->topic_.empty()) {pub_.msg_.pop_front(); continue;} } const auto &topic = msg->topic_; const auto topic_size = topic.size(); const auto &data = msg->msg_; const auto data_size = data.size(); char *sndmsg = (char*)malloc(topic_size + data_size); memcpy(sndmsg, topic.data(), topic_size); memcpy(sndmsg+topic_size, data.data(), data_size); int rc = nn_send(pub_.socket_, sndmsg, data_size+topic_size, 0); free(sndmsg); if (rc == (int)(data_size+topic_size)){ char* tmp{}; rc = nn_recv(pub_.socket_, &tmp, NN_MSG, 0); if (rc > 0){ nn_freemsg(tmp); printf("======>> publish topic %s data length %lu\n", topic.c_str(), data_size); lock_guard l{pub_.mtx_msg_}; pub_.msg_.pop_front(); continue; }else{ PRNTVITAG("publish req-rep thread nn_recv faild"); } }else{ PRNTVITAG("publish req-rep thread nn_send faild"); } } }); return sock; } int publish(const std::string& topic, const void* data, const int data_len){ if (!data && data_len == 0){ // printf("======>> publish start url %s\n", topic.c_str()); return connect_to_center(topic); } if (pub_.socket_ < 0){ connect_to_center(pub_.url_); } if(pub_.socket_ < 0) { PRNTVITAG("publish socket_ < 0"); return -1; } // printf("======>> publish topic %s\n", topic.c_str()); lock_guard l{pub_.mtx_msg_}; pub_.msg_.emplace_back(topic, string{(const char*)data, (const size_t)data_len}); pub_.cv_msg_.notify_one(); return pub_.msg_.size(); } /////////////////////////////////////////////// // subscribe class _ps_sub : public _ps{ public: DISABLE_COPY_AND_ASSIGN(_ps_sub); _ps_sub()=default; ~_ps_sub()=default; unordered_set topics_{}; mutex mtx_topics_{}; unordered_set failed_topics_{}; mutex mtx_failed_topics_{}; }; static _ps_sub sub_; int subscribe_center(const std::string& url){ if (sub_.socket_ > 0) return 0; sub_.url_ = url; int sock = client_socket(url, NN_SUB); if (sock < 0){ return -1; } // set_socket_timeout(sock, timeout_req_rep); sub_.socket_ = sock; sub_.t_ = thread([]{ while (!sub_.t_quit_.load()) { char* m; int m_len = nn_recv(sub_.socket_, &m, NN_MSG, NN_DONTWAIT); if (m_len > 0){ string topic{}, msg{}; { lock_guard l{sub_.mtx_topics_}; for(auto && i : sub_.topics_){ auto topic_len = i.size(); if (m_len <= (int)topic_len) continue; topic.assign(m, topic_len); if (topic == i){ msg.assign(m+topic_len, m_len-topic_len); break; } } } nn_freemsg(m); printf("======>> subscribe recv topic %s msg length %lu\n", topic.c_str(), msg.length()); if (!msg.empty()){ lock_guard l(sub_.mtx_msg_); sub_.msg_.emplace_back(topic, move(msg)); sub_.cv_msg_.notify_all(); } }else { { lock_guard l{sub_.mtx_failed_topics_}; if (!sub_.failed_topics_.empty()){ for(auto iter = sub_.failed_topics_.begin(); iter != sub_.failed_topics_.end();){ if (nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_UNSUBSCRIBE, iter->c_str(), iter->length()) >= 0){ iter = sub_.failed_topics_.erase(iter); }else{ iter++; } } } } this_thread::sleep_for(chrono::milliseconds{6}); // printf("======>> subscribe nn_recv failed %s\n", nn_strerror(nn_errno())); } } }); return 0; } int subscribe_topic(const std::string& topic){ if (sub_.socket_ < 0){ subscribe_center(sub_.url_); } if (sub_.socket_ < 0) return -1; auto ret = nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_SUBSCRIBE, topic.c_str(), topic.length()); // printf("set NN_SUB_SUBSCRIBE topic %s ret %d\n", topic.c_str(), ret); if (ret < 0){ lock_guard l{sub_.mtx_failed_topics_}; sub_.failed_topics_.insert(topic); } lock_guard l{sub_.mtx_topics_}; sub_.topics_.insert(topic); return 0; } int unsubscribe_topic(const std::string& topic){ lock_guard l(sub_.mtx_topics_); auto iter = sub_.topics_.find(topic); if (iter != sub_.topics_.end()){ nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_UNSUBSCRIBE, topic.c_str(), topic.length()); sub_.topics_.erase(iter); } return 0; } int subscribe_read(std::string* topic, std::string* msg, const int to_ms){ int tm = to_ms > 0 ? to_ms : 30; unique_lock l(sub_.mtx_msg_); auto status = sub_.cv_msg_.wait_for(l, chrono::milliseconds{tm}, []{ return !sub_.msg_.empty(); }); if (!status){ PRNTVITAG("subscribe_read timeout"); return -1; } const auto& tmp = sub_.msg_.front(); *topic = tmp.topic_; *msg = tmp.msg_; sub_.msg_.pop_front(); return 0; } /////////////////////////////////////////////////////////// // survey respondent for heartbeat class _sv : public _nn{ public: DISABLE_COPY_AND_ASSIGN(_sv); _sv()=default; ~_sv(){ t_quit_.store(true, memory_order_relaxed); if (t_.joinable()) t_.join(); } thread t_; atomic_bool t_quit_{false}; string fixed_msg_{}; }; static _sv survey_; int respond_survey(const std::string& url, std::string&& fixed_msg){ survey_.url_ = url; survey_.fixed_msg_ = move(fixed_msg); survey_.t_ = thread([]{ int& sock = survey_.socket_; const auto& msg = survey_.fixed_msg_; while (!survey_.t_quit_.load()) { if (sock < 0){ sock = client_socket(survey_.url_, NN_RESPONDENT); if (sock > 0){ set_socket_timeout(sock, 126); } } if (sock < 0) continue; char* tmp{}; int rc = nn_recv(sock, &tmp, NN_MSG, 0); if (rc > 0){ nn_freemsg(tmp); rc = nn_send(sock, msg.data(), msg.size(), 0); if (rc < 0){ PRNTVITAG("heartbeat survey failed"); PRNTVITAG(nn_strerror(nn_errno())); } } } }); return 0; } ////////////////////////////////////////////// // reply for request int request2(const std::string &ipc, const void* r, const int r_len, void** reply, int* reply_len, const int to_ms) { return simple_request(ipc, r, r_len, reply, reply_len, to_ms); } enum { INIT, RECV, WAIT, SEND }; struct work { int state{-1}; nng_aio *aio{}; nng_msg *msg{}; nng_ctx ctx; void(*cb_recv)(work*){}; }; class _rr : public _nn{ public: DISABLE_COPY_AND_ASSIGN(_rr); _rr()=default; ~_rr(){ } nng_socket sock_local_{0}; nng_socket sock_remote_{0}; int port_{-1}; unordered_map msg_{}; unordered_map works_{}; uint64_t work_index_{0}; mutex mtx_msg_{}; condition_variable cv_msg_{}; }; static _rr reply_; static void server_cb(void *arg) { struct work *work = (struct work*)arg; nng_msg * msg; int rv; uint32_t when{0}; switch (work->state) { case INIT: work->state = RECV; nng_ctx_recv(work->ctx, work->aio); break; case RECV: if ((rv = nng_aio_result(work->aio)) != 0) { break; } msg = nng_aio_get_msg(work->aio); work->msg = msg; work->state = WAIT; if (work->cb_recv) work->cb_recv(work); // nng_sleep_aio(when, work->aio); break; case WAIT: // We could add more data to the message here. nng_aio_set_msg(work->aio, work->msg); work->msg = NULL; work->state = SEND; nng_ctx_send(work->ctx, work->aio); break; case SEND: if ((rv = nng_aio_result(work->aio)) != 0) { nng_msg_free(work->msg); break; } work->state = RECV; nng_ctx_recv(work->ctx, work->aio); break; default: break; } } static void cb_recv_for_aio(work* w){ nng_msg *om = w->msg; string msg{(const char*)nng_msg_body(om), nng_msg_len(om)}; nng_msg_free(om); lock_guard l{reply_.mtx_msg_}; reply_.works_[reply_.work_index_] = w; reply_.msg_[reply_.work_index_] = msg; reply_.work_index_++; reply_.cv_msg_.notify_all(); } static struct work *alloc_work(nng_socket sock) { struct work *w; int rv; if ((w = (struct work*)nng_alloc(sizeof(*w))) == NULL) { return NULL;; } w->cb_recv = cb_recv_for_aio; if ((rv = nng_aio_alloc(&w->aio, server_cb, w)) != 0) { return NULL; } if ((rv = nng_ctx_open(&w->ctx, sock)) != 0) { return NULL; } w->state = INIT; return (w); } static constexpr int PARALLEL = 62; static struct work* works[PARALLEL]{}; static int create_server(nng_socket* sock, const string& url){ TAG; if (sock->id > 0) return 0; int rv = nng_rep0_open(sock); if (rv < 0){ PRNTVITAG("create_server nng_rep0_open faild"); PRNTVITAG(url); return rv; } for (int i = 0; i < PARALLEL; i++) { works[i] = alloc_work(*sock); } remove_exist(url); rv = nng_listen(*sock, url.c_str(), NULL, 0); if (rv < 0){ PRNTVITAG("create_server nng_listen failed"); PRNTVITAG(url); return rv; } for (int i = 0; i < PARALLEL; i++) { server_cb(works[i]); // this starts them going (INIT state) } return 0; } int start_reply(const std::string& url, const int port){ TAG; string ipc = "ipc:///tmp/" + url; if (url.find("ipc://") == 0){ ipc = url; } reply_.url_ = ipc; if(create_server(&reply_.sock_local_, ipc) != 0) return -1; if (port > 0){ reply_.port_ = port; ipc = "tcp://0.0.0.0:" + to_string(port); if(create_server(&reply_.sock_remote_, ipc) != 0) return -1; }else { reply_.sock_remote_.id = numeric_limits::max(); } return 0; } int read_request(void** src, std::string* msg, const int to_ms){ if (reply_.sock_local_.id == 0 || reply_.sock_remote_.id == 0) { if (start_reply(reply_.url_, reply_.port_) != 0) return -1; } int tm = to_ms > 0 ? to_ms : 30; uint64_t key{}; string tmpmsg; { unique_lock l(reply_.mtx_msg_); auto status = reply_.cv_msg_.wait_for(l, chrono::milliseconds{tm}, []{ return !reply_.msg_.empty(); }); if (!status){ PRNTVITAG("subscribe_read timeout"); return -1; } const auto& iter = reply_.msg_.begin(); key = iter->first; tmpmsg = iter->second; reply_.msg_.erase(iter); } *msg = move(tmpmsg); auto s = (uint64_t*)malloc(sizeof(uint64_t)); *s = key; *src = s; return 0; } int send_reply(const void* src, const void* msg, const int msg_len){ struct work* w{}; { auto key = *(static_cast(const_cast(src))); lock_guard l{reply_.mtx_msg_}; auto iter = reply_.works_.find(key); if (iter == reply_.works_.end()) return -1; w = iter->second; reply_.works_.erase(iter); } TAG; nng_msg_alloc(&w->msg, 0); nng_msg_append(w->msg, msg, msg_len); nng_sleep_aio(0, w->aio); return 0; } }