#include "nng_wrap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace std; #include #include #include #include #include "nng/compat/nanomsg/nn.h" #include "nng/compat/nanomsg/reqrep.h" #include "nng/compat/nanomsg/pubsub.h" #include "nng/compat/nanomsg/survey.h" namespace nng_wrap { // common function static int client_socket(const string& url, const int protocol, int family=AF_SP){ int sock = nn_socket(family, protocol); if (sock < 0) return sock; int rc = nn_connect(sock, url.c_str()); if (rc < 0) { nn_close(sock); return rc; } return sock; } static void remove_exist(const string& url){ if (url.find("ipc://") == 0){ string address(url); address = address.substr(6); if (access(address.c_str(), F_OK) == 0){ remove(address.c_str()); } } } // static int server_socket(const string& url, const int protocol, int family=AF_SP){ // int sock = nn_socket(family, protocol); // if (sock < 0) return sock; // remove_exist(url); // int rc = nn_bind(sock, url.c_str()); // if (rc < 0) { // nn_close(sock); // return rc; // } // return sock; // } static void set_socket_timeout(int sock, const int to_ms){ nn_setsockopt(sock, NN_SOL_SOCKET, NN_SNDTIMEO, &to_ms, sizeof(to_ms)); nn_setsockopt(sock, NN_SOL_SOCKET, NN_RCVTIMEO, &to_ms, sizeof(to_ms)); } static int send_and_recv(int sock, const void* in, const int in_len, void** out, int *out_len) { int rc = nn_send(sock, in, in_len, 0); if (rc != in_len) { nn_close(sock); return -1; } char *msg = NULL; rc = nn_recv(sock, &msg, NN_MSG, 0); if (rc < 0) { nn_close(sock); return rc; } nn_close(sock); copy_memory(out, out_len, msg, rc); nn_freemsg(msg); return 0; } ///////////////////////////////////////////////// // simple interface void free_nng(void* data, const int data_len){ if (data){ free(data); } } void copy_memory(void** dest, int *dest_len, const void* src, const int src_len){ char* tmp = (char*)malloc(src_len); memcpy(tmp, src, src_len); *dest = tmp; if(dest_len) *dest_len = src_len; } static thread_local string verbose_info{}; #ifndef PRNTVITAG #define TAG do{ verbose_info.clear(); \ verbose_info=string("function [")+__FUNCTION__+string("]"); \ }while(0) #define PRNTVITAG(msg) do{ \ verbose_info+=string("-> (") + msg + string(")"); \ }while(0) // #define TAG // #define PRNTVITAG(args) #endif void get_last_error(int* ec, void** emsg, int* emsg_len){ *emsg = NULL; *emsg_len = 0; *ec = nn_errno(); const char* msg = nn_strerror(*ec); string strMsg(msg); strMsg = strMsg + "{" + verbose_info + "}"; copy_memory(emsg, emsg_len, strMsg.data(), strMsg.size()); verbose_info.clear(); } /////////////////////////////////////////////////////// // simple request waiting reply static constexpr int timeout_req_rep = 5162; int simple_request(const std::string& url, const void* in, const int in_len, void** out, int *out_len, const int to_ms){ TAG; *out = NULL; *out_len = 0; // printf("------>> simple_request url %s\n", (char*)url.data); int sock = client_socket(url, NN_REQ); if (sock < 0) { PRNTVITAG("client_socket faild"); return false; } int tm = to_ms < timeout_req_rep ? timeout_req_rep : to_ms; set_socket_timeout(sock, tm); auto ret = send_and_recv(sock, in, in_len, out, out_len); // printf("------>> simple_request out data address %p len %d\n", out->data, out->data_len); if (ret < 0) { PRNTVITAG("send_and_recv faild"); return false; } return true; } /////////////////////////////////////////////////////////// // base class #define DISABLE_COPY_AND_ASSIGN(className) \ className(const className&)=delete; \ className(className&&)=delete; \ className& operator=(const className&)=delete; \ className& operator=(className&&)=delete class _nn{ public: DISABLE_COPY_AND_ASSIGN(_nn); _nn()=default; virtual ~_nn(){ if (socket_ > 0) nn_close(socket_); } int socket_{-1}; string url_{}; }; /////////////////////////////////////////////// // publish class _ps : public _nn{ public: struct psmsg{ DISABLE_COPY_AND_ASSIGN(psmsg); psmsg()=delete; psmsg(const std::string& t, std::string&& m) :topic_(t),data_(std::move(m)){} std::string topic_{}; std::string data_{}; }; public: DISABLE_COPY_AND_ASSIGN(_ps); _ps()=default; virtual ~_ps(){ t_quit_.store(true, memory_order_relaxed); if (t_.joinable()) t_.join(); } thread t_; atomic_bool t_quit_{false}; deque msg_{}; mutex mtx_msg_{}; condition_variable cv_msg_{}; }; static _ps pub_; static int connect_to_center(const string& topic){ if (pub_.socket_ > 0) return pub_.socket_; pub_.url_ = topic; TAG; int sock = client_socket(topic, NN_REQ); if (sock < 0){ PRNTVITAG("client_socket faild"); return -1; } set_socket_timeout(sock, timeout_req_rep); pub_.socket_ = sock; pub_.t_ = thread([]{ while (!pub_.t_quit_.load()) { _ps::psmsg *msg{NULL}; { unique_lock l{pub_.mtx_msg_}; pub_.cv_msg_.wait(l, []{ return !pub_.msg_.empty() || pub_.t_quit_.load(); }); if(pub_.t_quit_.load()) break; msg = &pub_.msg_.front(); if (msg->topic_.empty()) {pub_.msg_.pop_front(); continue;} } string sndmsg(msg->topic_ + msg->data_); int rc = nn_send(pub_.socket_, sndmsg.data(), sndmsg.size(), 0); if (rc == (int)sndmsg.size()){ char* tmp{}; rc = nn_recv(pub_.socket_, &tmp, NN_MSG, 0); if (rc > 0){ nn_freemsg(tmp); printf("======>> publish topic %s data length %lu\n", msg->topic_.c_str(), msg->data_.size()); lock_guard l{pub_.mtx_msg_}; pub_.msg_.pop_front(); continue; }else{ PRNTVITAG("publish req-rep thread nn_recv faild"); } }else{ PRNTVITAG("publish req-rep thread nn_send faild"); } } }); return sock; } int publish(const std::string& topic, const void* data, const int data_len){ if (!data && data_len == 0){ // printf("======>> publish start url %s\n", topic.c_str()); return connect_to_center(topic); } if (pub_.socket_ < 0){ connect_to_center(pub_.url_); } if(pub_.socket_ < 0) { PRNTVITAG("publish socket_ < 0"); return -1; } // printf("======>> publish topic %s\n", topic.c_str()); lock_guard l{pub_.mtx_msg_}; pub_.msg_.emplace_back(topic, string{(const char*)data, (const size_t)data_len}); pub_.cv_msg_.notify_one(); return pub_.msg_.size(); } /////////////////////////////////////////////// // subscribe class _ps_sub : public _ps{ public: DISABLE_COPY_AND_ASSIGN(_ps_sub); _ps_sub()=default; ~_ps_sub()=default; unordered_set topics_{}; mutex mtx_topics_{}; unordered_set failed_topics_{}; mutex mtx_failed_topics_{}; }; static _ps_sub sub_; int subscribe_center(const std::string& url){ if (sub_.socket_ > 0) return 0; sub_.url_ = url; TAG; int sock = client_socket(url, NN_SUB); if (sock < 0){ PRNTVITAG("client_socket faild\n"); return -1; } // set_socket_timeout(sock, timeout_req_rep); sub_.socket_ = sock; sub_.t_ = thread([]{ while (!sub_.t_quit_.load()) { char* m; int m_len = nn_recv(sub_.socket_, &m, NN_MSG, NN_DONTWAIT); if (m_len > 0){ string tmp_msg{m, (size_t)m_len}; nn_freemsg(m); string topic{}, msg{}; { lock_guard l{sub_.mtx_topics_}; for(auto && i : sub_.topics_){ if (tmp_msg.size() < i.size()) continue; topic = tmp_msg.substr(0, i.size()); if (topic == i){ msg = tmp_msg.substr(i.size()); break; } } } printf("======>> subscribe recv topic %s msg length %lu\n", topic.c_str(), msg.length()); if (!msg.empty()){ lock_guard l(sub_.mtx_msg_); sub_.msg_.emplace_back(topic, move(msg)); sub_.cv_msg_.notify_all(); } }else { { lock_guard l{sub_.mtx_failed_topics_}; if (!sub_.failed_topics_.empty()){ for(auto iter = sub_.failed_topics_.begin(); iter != sub_.failed_topics_.end();){ if (nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_UNSUBSCRIBE, iter->c_str(), iter->length()) >= 0){ iter = sub_.failed_topics_.erase(iter); }else{ iter++; } } } } this_thread::sleep_for(chrono::milliseconds{6}); // printf("======>> subscribe nn_recv failed %s\n", nn_strerror(nn_errno())); } } }); return 0; } int subscribe_topic(const std::string& topic){ TAG; if (sub_.socket_ < 0){ subscribe_center(sub_.url_); } if (sub_.socket_ < 0) { PRNTVITAG("socket_ < 0"); return -1; } auto ret = nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_SUBSCRIBE, topic.c_str(), topic.length()); // printf("set NN_SUB_SUBSCRIBE topic %s ret %d\n", topic.c_str(), ret); if (ret < 0){ PRNTVITAG("nn_setsockopt failed"); lock_guard l{sub_.mtx_failed_topics_}; sub_.failed_topics_.insert(topic); } lock_guard l{sub_.mtx_topics_}; sub_.topics_.insert(topic); return 0; } int unsubscribe_topic(const std::string& topic){ lock_guard l(sub_.mtx_topics_); auto iter = sub_.topics_.find(topic); if (iter != sub_.topics_.end()){ nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_UNSUBSCRIBE, topic.c_str(), topic.length()); sub_.topics_.erase(iter); } return 0; } int subscribe_read(std::string* topic, std::string* msg, const int to_ms){ TAG; int tm = to_ms > 0 ? to_ms : 30; unique_lock l(sub_.mtx_msg_); auto status = sub_.cv_msg_.wait_for(l, chrono::milliseconds{tm}, []{ return !sub_.msg_.empty(); }); if (!status){ PRNTVITAG("subscribe_read timeout"); return -1; } const auto& tmp = sub_.msg_.front(); *topic = tmp.topic_; *msg = tmp.data_; sub_.msg_.pop_front(); return 0; } /////////////////////////////////////////////////////////// // survey respondent for heartbeat class _sv : public _nn{ public: DISABLE_COPY_AND_ASSIGN(_sv); _sv()=default; ~_sv(){ t_quit_.store(true, memory_order_relaxed); if (t_.joinable()) t_.join(); } thread t_; atomic_bool t_quit_{false}; string fixed_msg_{}; }; static _sv survey_; int respond_survey(const std::string& url, std::string&& fixed_msg){ survey_.url_ = url; survey_.fixed_msg_ = move(fixed_msg); survey_.t_ = thread([]{ TAG; int& sock = survey_.socket_; const auto& msg = survey_.fixed_msg_; while (!survey_.t_quit_.load()) { if (sock < 0){ sock = client_socket(survey_.url_, NN_RESPONDENT); if (sock > 0) set_socket_timeout(sock, 126); } if (sock < 0) continue; char* tmp{}; int rc = nn_recv(sock, &tmp, NN_MSG, 0); if (rc > 0){ nn_freemsg(tmp); rc = nn_send(sock, msg.data(), msg.size(), 0); if (rc < 0){ PRNTVITAG("heartbeat survey failed"); PRNTVITAG(nn_strerror(nn_errno())); } } } }); return 0; } ////////////////////////////////////////////// // reply for request enum { INIT, RECV, WAIT, SEND }; struct work { int state{-1}; nng_aio *aio{}; nng_msg *msg{}; nng_ctx ctx; void(*cb_recv)(work*){}; }; class _rr : public _nn{ public: DISABLE_COPY_AND_ASSIGN(_rr); _rr()=default; ~_rr(){ if(sock_local_.id > 0) nng_close(sock_local_); if(sock_remote_.id > 0) nng_close(sock_remote_); t_quit_.store(true, memory_order_relaxed); if (t_unblock_&&t_unblock_->joinable()) t_unblock_->join(); } const string unblocking_msg_{"~!@#$%^&*()-=<>=-()*&^%$#@!~"}; unique_ptr t_unblock_{nullptr}; atomic_bool t_quit_{false}; nng_socket sock_local_{0}; nng_socket sock_remote_{0}; int port_{-1}; unordered_map msg_{}; class worker{ worker& in_op(const worker& w){if(&w!=this){w_=w.w_;life_=w.life_;}return *this;}; public: worker()=default; ~worker()=default; worker(struct work* w):w_(w),life_(0){} worker(const worker& w):w_(w.w_),life_(w.life_){} worker(worker&& w):w_(w.w_),life_(w.life_){} worker& operator=(const worker& w){return in_op(w);} worker& operator=(worker&& w){return in_op(w);} operator struct work*() const{return w_;} operator int&() {return life_;} struct work* w_{}; int life_{}; }; unordered_map works_{}; uint64_t work_index_{0}; mutex mtx_msg_{}; condition_variable cv_msg_{}; }; static _rr reply_; int request2(const std::string &ipc, const void* r, const int r_len, void** reply, int* reply_len, const int to_ms) { const auto suc = simple_request(ipc, r, r_len, reply, reply_len, to_ms); if (suc){ const size_t sl = reply_.unblocking_msg_.size(); const size_t rl = *reply_len; if (sl != rl) return true; const auto& s = reply_.unblocking_msg_; auto r = (const char*)(*reply); if (s.compare(0, sl, r, rl) == 0){ free(*reply); *reply = NULL; *reply_len = 0; return false; } } return suc; } static void server_cb(void *arg) { if (!arg) return; struct work *work = (struct work*)arg; nng_msg * msg; int rv; // uint32_t when{0}; switch (work->state) { case INIT: work->state = RECV; nng_ctx_recv(work->ctx, work->aio); break; case RECV: if ((rv = nng_aio_result(work->aio)) != 0) { break; } msg = nng_aio_get_msg(work->aio); work->msg = msg; work->state = WAIT; if (work->cb_recv) work->cb_recv(work); // nng_sleep_aio(when, work->aio); break; case WAIT: // We could add more data to the message here. nng_aio_set_msg(work->aio, work->msg); work->msg = NULL; work->state = SEND; nng_ctx_send(work->ctx, work->aio); break; case SEND: if ((rv = nng_aio_result(work->aio)) != 0) { nng_msg_free(work->msg); } work->state = RECV; nng_ctx_recv(work->ctx, work->aio); break; default: break; } } static void cb_recv_for_aio(work* w){ nng_msg *om = w->msg; if (!om) return; string msg{(const char*)nng_msg_body(om), nng_msg_len(om)}; nng_msg_free(om); lock_guard l{reply_.mtx_msg_}; reply_.works_.emplace(reply_.work_index_, w); reply_.msg_.emplace(reply_.work_index_, move(msg)); // reply_.works_.insert({reply_.work_index_, w}); // reply_.msg_.insert({reply_.work_index_, msg}); // reply_.works_[reply_.work_index_] = w; // reply_.msg_[reply_.work_index_] = msg; reply_.work_index_++; reply_.cv_msg_.notify_all(); } static struct work *alloc_work(nng_socket sock) { struct work *w; int rv; if ((w = (struct work*)nng_alloc(sizeof(*w))) == NULL) { return NULL;; } w->cb_recv = cb_recv_for_aio; if ((rv = nng_aio_alloc(&w->aio, server_cb, w)) != 0) { return NULL; } if ((rv = nng_ctx_open(&w->ctx, sock)) != 0) { return NULL; } w->state = INIT; return (w); } static constexpr int PARALLEL = 62; static struct work* works_local[PARALLEL]{}; static struct work* works_remote[PARALLEL]{}; static int create_server(nng_socket* sock, const string& url, work** works){ TAG; if (sock->id > 0) return 0; int rv = nng_rep0_open(sock); if (rv < 0){ PRNTVITAG("create_server nng_rep0_open faild"); PRNTVITAG(url); return rv; } for (int i = 0; i < PARALLEL; i++) { works[i] = alloc_work(*sock); } remove_exist(url); rv = nng_listen(*sock, url.c_str(), NULL, 0); if (rv < 0){ PRNTVITAG("create_server nng_listen failed"); PRNTVITAG(url); return rv; } for (int i = 0; i < PARALLEL; i++) { server_cb(works[i]); // this starts them going (INIT state) } return 0; } static void aio_unblock(work* w, const void* msg, const int msg_len){ nng_msg_alloc(&w->msg, 0); nng_msg_append(w->msg, msg, msg_len); nng_sleep_aio(0, w->aio); } int start_reply(const std::string& url, const int port){ string ipc = "ipc:///tmp/" + url; if (url.find("ipc://") == 0){ ipc = url; } reply_.url_ = ipc; if(create_server(&reply_.sock_local_, ipc, works_local) != 0) return -1; if (port > 0){ reply_.port_ = port; ipc = "tcp://0.0.0.0:" + to_string(port); if(create_server(&reply_.sock_remote_, ipc, works_remote) != 0) return -1; }else { reply_.sock_remote_.id = numeric_limits::max(); } if (!reply_.t_unblock_){ reply_.t_unblock_.reset(new thread([]{ constexpr int idle = 10; const auto data = reply_.unblocking_msg_.data(); const auto data_size = reply_.unblocking_msg_.size(); while (!reply_.t_quit_.load()) { this_thread::sleep_for(chrono::milliseconds{10}); vector tmp{}; { lock_guard l{reply_.mtx_msg_}; for(auto iter = reply_.works_.begin(); iter != reply_.works_.end();){ if ((iter->second+=idle) > timeout_req_rep){ tmp.push_back(iter->second.w_); iter = reply_.works_.erase(iter); }else { ++iter; } } } for(auto && w : tmp){ aio_unblock(w, data, data_size); } } })); } return 0; } int read_request(void** src, std::string* msg, const int to_ms){ if (reply_.sock_local_.id == 0 || reply_.sock_remote_.id == 0) if (start_reply(reply_.url_, reply_.port_) != 0) return -1; int tm = to_ms > 0 ? to_ms : 30; uint64_t key{}; { unique_lock l(reply_.mtx_msg_); auto status = reply_.cv_msg_.wait_for(l, chrono::milliseconds{tm}, []{ return !reply_.msg_.empty(); }); if (!status){ PRNTVITAG("subscribe_read timeout"); return -1; } auto iter = reply_.msg_.begin(); key = iter->first; *msg = move(iter->second); reply_.msg_.erase(iter); } *src = malloc(sizeof(uint64_t)); *(uint64_t*)(*src) = key; return 0; } int send_reply(const void* src, const void* msg, const int msg_len){ struct work* w{}; { auto key = *(static_cast(const_cast(src))); lock_guard l{reply_.mtx_msg_}; auto iter = reply_.works_.find(key); if (iter == reply_.works_.end()) return -1; w = iter->second; reply_.works_.erase(iter); } aio_unblock(w, msg, msg_len); return 0; } }