#include "nng_wrap.h"
|
|
#include <string.h>
|
|
#include <random>
|
#include <vector>
|
#include <unordered_map>
|
#include <unordered_set>
|
#include <thread>
|
#include <atomic>
|
#include <deque>
|
#include <mutex>
|
#include <condition_variable>
|
#include <functional>
|
#include <chrono>
|
#include <future>
|
using namespace std;
|
|
#include <unistd.h>
|
|
#include <nng/nng.h>
|
#include <nng/protocol/reqrep0/rep.h>
|
#include <nng/supplemental/util/platform.h>
|
|
#include "nng/compat/nanomsg/nn.h"
|
#include "nng/compat/nanomsg/reqrep.h"
|
#include "nng/compat/nanomsg/pubsub.h"
|
#include "nng/compat/nanomsg/survey.h"
|
|
namespace nng_wrap {
|
|
// common function
|
static int client_socket(const string& url, const int protocol, int family=AF_SP){
|
int sock = nn_socket(family, protocol);
|
if (sock < 0) return sock;
|
int rc = nn_connect(sock, url.c_str());
|
if (rc < 0) {
|
nn_close(sock);
|
return rc;
|
}
|
return sock;
|
}
|
|
static void remove_exist(const string& url){
|
if (url.find("ipc://") == 0){
|
string address(url);
|
address = address.substr(6);
|
if (access(address.c_str(), F_OK) == 0){
|
remove(address.c_str());
|
}
|
}
|
}
|
|
static int server_socket(const string& url, const int protocol, int family=AF_SP){
|
int sock = nn_socket(family, protocol);
|
if (sock < 0) return sock;
|
remove_exist(url);
|
int rc = nn_bind(sock, url.c_str());
|
if (rc < 0) {
|
nn_close(sock);
|
return rc;
|
}
|
return sock;
|
}
|
|
static void set_socket_timeout(int sock, const int to_ms){
|
nn_setsockopt(sock, NN_SOL_SOCKET, NN_SNDTIMEO, &to_ms, sizeof(to_ms));
|
nn_setsockopt(sock, NN_SOL_SOCKET, NN_RCVTIMEO, &to_ms, sizeof(to_ms));
|
}
|
|
static int send_and_recv(int sock,
|
const void* in, const int in_len,
|
void** out, int *out_len)
|
{
|
int rc = nn_send(sock, in, in_len, 0);
|
if (rc != in_len) {
|
nn_close(sock);
|
return -1;
|
}
|
char *msg = NULL;
|
rc = nn_recv(sock, &msg, NN_MSG, 0);
|
if (rc < 0) {
|
nn_close(sock);
|
return rc;
|
}
|
nn_close(sock);
|
copy_memory(out, out_len, msg, rc);
|
nn_freemsg(msg);
|
return 0;
|
}
|
|
/////////////////////////////////////////////////
|
// simple interface
|
|
void free_nng(void* data, const int data_len){
|
if (data){
|
free(data);
|
}
|
}
|
|
void copy_memory(void** dest, int *dest_len, const void* src, const int src_len){
|
char* tmp = (char*)malloc(src_len);
|
memcpy(tmp, src, src_len);
|
*dest = tmp;
|
if(dest_len)
|
*dest_len = src_len;
|
}
|
|
static string verbose_info{};
|
#ifndef PRNTVITAG
|
// #define TAG do{ \
|
// if (verbose_info.length() > 8096) verbose_info.clear(); \
|
// verbose_info=string("function [")+__FUNCTION__+string("]");}while(0)
|
/* #define PRNTVITAG(msg) do{ \
|
if (verbose_info.length() > 8096) verbose_info.clear(); \
|
verbose_info+=string("-> (") + msg + string(")"); \
|
}while(0) */
|
#define TAG
|
#define PRNTVITAG(args)
|
#endif
|
void get_last_error(int* ec, void** emsg, int* emsg_len){
|
*emsg = NULL;
|
*emsg_len = 0;
|
|
*ec = nn_errno();
|
const char* msg = nn_strerror(*ec);
|
string strMsg(msg);
|
strMsg = strMsg + "{" + verbose_info + "}";
|
copy_memory(emsg, emsg_len, strMsg.data(), strMsg.size());
|
verbose_info.clear();
|
}
|
|
///////////////////////////////////////////////////////
|
// simple request waiting reply
|
|
static constexpr int timeout_req_rep = 5162;
|
int simple_request(const std::string& url,
|
const void* in, const int in_len,
|
void** out, int *out_len, const int to_ms){
|
|
TAG;
|
*out = NULL;
|
*out_len = 0;
|
|
// printf("------>> simple_request url %s\n", (char*)url.data);
|
int sock = client_socket(url, NN_REQ);
|
if (sock < 0) {
|
PRNTVITAG("client_socket faild");
|
return false;
|
}
|
|
int tm = to_ms < timeout_req_rep ? timeout_req_rep : to_ms;
|
set_socket_timeout(sock, tm);
|
|
auto ret = send_and_recv(sock, in, in_len, out, out_len);
|
// printf("------>> simple_request out data address %p len %d\n", out->data, out->data_len);
|
if (ret < 0) {
|
PRNTVITAG("send_and_recv faild");
|
return false;
|
}
|
return true;
|
}
|
|
///////////////////////////////////////////////////////////
|
// base class
|
#define DISABLE_COPY_AND_ASSIGN(className) className(const className&)=delete; \
|
className(className&&)=delete; \
|
className& operator=(const className&)=delete; \
|
className& operator=(className&&)=delete
|
class _nn{
|
public:
|
DISABLE_COPY_AND_ASSIGN(_nn);
|
_nn()=default;
|
virtual ~_nn(){ if (socket_ > 0) nn_close(socket_); }
|
int socket_{-1};
|
string url_{};
|
};
|
|
///////////////////////////////////////////////
|
// publish
|
struct psmsg{
|
DISABLE_COPY_AND_ASSIGN(psmsg);
|
psmsg()=delete;
|
psmsg(const std::string& t, std::string&& m)
|
:topic_(t),msg_(std::move(m)){}
|
std::string topic_{};
|
std::string msg_{};
|
};
|
class _ps : public _nn{
|
public:
|
DISABLE_COPY_AND_ASSIGN(_ps);
|
_ps()=default;
|
virtual ~_ps(){
|
t_quit_.store(true, memory_order_relaxed);
|
if (t_.joinable()) t_.join();
|
}
|
|
thread t_;
|
atomic_bool t_quit_{false};
|
deque<psmsg> msg_{};
|
mutex mtx_msg_{};
|
condition_variable cv_msg_{};
|
};
|
|
static _ps pub_;
|
static int connect_to_center(const string& topic){
|
if (pub_.socket_ > 0) return pub_.socket_;
|
pub_.url_ = topic;
|
|
TAG;
|
int sock = client_socket(topic, NN_REQ);
|
if (sock < 0){
|
PRNTVITAG("client_socket faild");
|
return -1;
|
}
|
set_socket_timeout(sock, timeout_req_rep);
|
pub_.socket_ = sock;
|
pub_.t_ = thread([]{
|
while (!pub_.t_quit_.load()) {
|
psmsg *msg{NULL};
|
{
|
unique_lock<mutex> l{pub_.mtx_msg_};
|
pub_.cv_msg_.wait(l, []{
|
return !pub_.msg_.empty() || pub_.t_quit_.load();
|
});
|
if(pub_.t_quit_.load()) break;
|
msg = &pub_.msg_.front();
|
if (msg->topic_.empty()) {pub_.msg_.pop_front(); continue;}
|
}
|
const auto &topic = msg->topic_;
|
const auto topic_size = topic.size();
|
const auto &data = msg->msg_;
|
const auto data_size = data.size();
|
|
char *sndmsg = (char*)malloc(topic_size + data_size);
|
memcpy(sndmsg, topic.data(), topic_size);
|
memcpy(sndmsg+topic_size, data.data(), data_size);
|
int rc = nn_send(pub_.socket_, sndmsg, data_size+topic_size, 0);
|
free(sndmsg);
|
if (rc == (int)(data_size+topic_size)){
|
char* tmp{};
|
rc = nn_recv(pub_.socket_, &tmp, NN_MSG, 0);
|
if (rc > 0){
|
nn_freemsg(tmp);
|
printf("======>> publish topic %s data length %lu\n", topic.c_str(), data_size);
|
lock_guard<mutex> l{pub_.mtx_msg_};
|
pub_.msg_.pop_front();
|
continue;
|
}else{
|
PRNTVITAG("publish req-rep thread nn_recv faild");
|
}
|
}else{
|
PRNTVITAG("publish req-rep thread nn_send faild");
|
}
|
|
}
|
});
|
return sock;
|
}
|
|
int publish(const std::string& topic, const void* data, const int data_len){
|
if (!data && data_len == 0){
|
// printf("======>> publish start url %s\n", topic.c_str());
|
return connect_to_center(topic);
|
}
|
if (pub_.socket_ < 0){
|
connect_to_center(pub_.url_);
|
}
|
if(pub_.socket_ < 0) {
|
PRNTVITAG("publish socket_ < 0");
|
return -1;
|
}
|
|
// printf("======>> publish topic %s\n", topic.c_str());
|
lock_guard<mutex> l{pub_.mtx_msg_};
|
pub_.msg_.emplace_back(topic, string{(const char*)data, (const size_t)data_len});
|
pub_.cv_msg_.notify_one();
|
return pub_.msg_.size();
|
}
|
|
///////////////////////////////////////////////
|
// subscribe
|
class _ps_sub : public _ps{
|
public:
|
DISABLE_COPY_AND_ASSIGN(_ps_sub);
|
_ps_sub()=default;
|
~_ps_sub()=default;
|
|
unordered_set<string> topics_{};
|
mutex mtx_topics_{};
|
unordered_set<string> failed_topics_{};
|
mutex mtx_failed_topics_{};
|
};
|
|
static _ps_sub sub_;
|
int subscribe_center(const std::string& url){
|
if (sub_.socket_ > 0) return 0;
|
sub_.url_ = url;
|
|
int sock = client_socket(url, NN_SUB);
|
if (sock < 0){
|
return -1;
|
}
|
// set_socket_timeout(sock, timeout_req_rep);
|
sub_.socket_ = sock;
|
sub_.t_ = thread([]{
|
while (!sub_.t_quit_.load()) {
|
char* m;
|
int m_len = nn_recv(sub_.socket_, &m, NN_MSG, NN_DONTWAIT);
|
if (m_len > 0){
|
string topic{}, msg{};
|
{
|
lock_guard<mutex> l{sub_.mtx_topics_};
|
for(auto && i : sub_.topics_){
|
auto topic_len = i.size();
|
if (m_len <= (int)topic_len) continue;
|
topic.assign(m, topic_len);
|
if (topic == i){
|
msg.assign(m+topic_len, m_len-topic_len);
|
break;
|
}
|
}
|
}
|
nn_freemsg(m);
|
printf("======>> subscribe recv topic %s msg length %lu\n", topic.c_str(), msg.length());
|
if (!msg.empty()){
|
lock_guard<mutex> l(sub_.mtx_msg_);
|
sub_.msg_.emplace_back(topic, move(msg));
|
sub_.cv_msg_.notify_all();
|
}
|
|
}else {
|
{
|
lock_guard<mutex> l{sub_.mtx_failed_topics_};
|
if (!sub_.failed_topics_.empty()){
|
for(auto iter = sub_.failed_topics_.begin(); iter != sub_.failed_topics_.end();){
|
if (nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_UNSUBSCRIBE, iter->c_str(), iter->length()) >= 0){
|
iter = sub_.failed_topics_.erase(iter);
|
}else{
|
iter++;
|
}
|
}
|
}
|
}
|
this_thread::sleep_for(chrono::milliseconds{6});
|
// printf("======>> subscribe nn_recv failed %s\n", nn_strerror(nn_errno()));
|
}
|
}
|
});
|
return 0;
|
}
|
|
int subscribe_topic(const std::string& topic){
|
if (sub_.socket_ < 0){
|
subscribe_center(sub_.url_);
|
}
|
if (sub_.socket_ < 0) return -1;
|
|
auto ret = nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_SUBSCRIBE, topic.c_str(), topic.length());
|
// printf("set NN_SUB_SUBSCRIBE topic %s ret %d\n", topic.c_str(), ret);
|
if (ret < 0){
|
lock_guard<mutex> l{sub_.mtx_failed_topics_};
|
sub_.failed_topics_.insert(topic);
|
}
|
lock_guard<mutex> l{sub_.mtx_topics_};
|
sub_.topics_.insert(topic);
|
|
return 0;
|
}
|
|
int unsubscribe_topic(const std::string& topic){
|
lock_guard<mutex> l(sub_.mtx_topics_);
|
auto iter = sub_.topics_.find(topic);
|
if (iter != sub_.topics_.end()){
|
nn_setsockopt(sub_.socket_, NN_SUB, NN_SUB_UNSUBSCRIBE, topic.c_str(), topic.length());
|
sub_.topics_.erase(iter);
|
}
|
|
return 0;
|
}
|
|
int subscribe_read(std::string* topic, std::string* msg, const int to_ms){
|
|
int tm = to_ms > 0 ? to_ms : 30;
|
|
unique_lock<mutex> l(sub_.mtx_msg_);
|
auto status = sub_.cv_msg_.wait_for(l, chrono::milliseconds{tm}, []{
|
return !sub_.msg_.empty();
|
});
|
if (!status){
|
PRNTVITAG("subscribe_read timeout");
|
return -1;
|
}
|
const auto& tmp = sub_.msg_.front();
|
*topic = tmp.topic_;
|
*msg = tmp.msg_;
|
sub_.msg_.pop_front();
|
|
return 0;
|
}
|
|
///////////////////////////////////////////////////////////
|
// survey respondent for heartbeat
|
|
class _sv : public _nn{
|
public:
|
DISABLE_COPY_AND_ASSIGN(_sv);
|
_sv()=default;
|
~_sv(){
|
t_quit_.store(true, memory_order_relaxed);
|
if (t_.joinable()) t_.join();
|
}
|
|
thread t_;
|
atomic_bool t_quit_{false};
|
string fixed_msg_{};
|
};
|
|
static _sv survey_;
|
int respond_survey(const std::string& url, std::string&& fixed_msg){
|
survey_.url_ = url;
|
survey_.fixed_msg_ = move(fixed_msg);
|
survey_.t_ = thread([]{
|
int& sock = survey_.socket_;
|
const auto& msg = survey_.fixed_msg_;
|
while (!survey_.t_quit_.load()) {
|
if (sock < 0){
|
sock = client_socket(survey_.url_, NN_RESPONDENT);
|
if (sock > 0){
|
set_socket_timeout(sock, 126);
|
}
|
}
|
if (sock < 0) continue;
|
|
char* tmp{};
|
int rc = nn_recv(sock, &tmp, NN_MSG, 0);
|
if (rc > 0){
|
nn_freemsg(tmp);
|
rc = nn_send(sock, msg.data(), msg.size(), 0);
|
if (rc < 0){
|
PRNTVITAG("heartbeat survey failed");
|
PRNTVITAG(nn_strerror(nn_errno()));
|
}
|
}
|
}
|
});
|
|
return 0;
|
}
|
|
//////////////////////////////////////////////
|
// reply for request
|
|
int request2(const std::string &ipc, const void* r, const int r_len,
|
void** reply, int* reply_len, const int to_ms)
|
{
|
return simple_request(ipc, r, r_len, reply, reply_len, to_ms);
|
}
|
|
enum { INIT, RECV, WAIT, SEND };
|
struct work {
|
int state{-1};
|
nng_aio *aio{};
|
nng_msg *msg{};
|
nng_ctx ctx;
|
void(*cb_recv)(work*){};
|
};
|
|
class _rr : public _nn{
|
public:
|
DISABLE_COPY_AND_ASSIGN(_rr);
|
_rr()=default;
|
~_rr(){
|
|
}
|
|
nng_socket sock_local_{0};
|
nng_socket sock_remote_{0};
|
int port_{-1};
|
|
unordered_map<uint64_t, string> msg_{};
|
unordered_map<uint64_t, struct work*> works_{};
|
uint64_t work_index_{0};
|
mutex mtx_msg_{};
|
condition_variable cv_msg_{};
|
|
};
|
|
static _rr reply_;
|
|
static void
|
server_cb(void *arg)
|
{
|
struct work *work = (struct work*)arg;
|
nng_msg * msg;
|
int rv;
|
uint32_t when{0};
|
|
switch (work->state) {
|
case INIT:
|
work->state = RECV;
|
nng_ctx_recv(work->ctx, work->aio);
|
break;
|
case RECV:
|
if ((rv = nng_aio_result(work->aio)) != 0) {
|
break;
|
}
|
msg = nng_aio_get_msg(work->aio);
|
|
work->msg = msg;
|
work->state = WAIT;
|
if (work->cb_recv) work->cb_recv(work);
|
// nng_sleep_aio(when, work->aio);
|
break;
|
case WAIT:
|
// We could add more data to the message here.
|
nng_aio_set_msg(work->aio, work->msg);
|
work->msg = NULL;
|
work->state = SEND;
|
nng_ctx_send(work->ctx, work->aio);
|
break;
|
case SEND:
|
if ((rv = nng_aio_result(work->aio)) != 0) {
|
nng_msg_free(work->msg);
|
break;
|
}
|
work->state = RECV;
|
nng_ctx_recv(work->ctx, work->aio);
|
break;
|
default:
|
break;
|
}
|
}
|
|
static void cb_recv_for_aio(work* w){
|
nng_msg *om = w->msg;
|
string msg{(const char*)nng_msg_body(om), nng_msg_len(om)};
|
nng_msg_free(om);
|
|
lock_guard<mutex> l{reply_.mtx_msg_};
|
reply_.works_[reply_.work_index_] = w;
|
reply_.msg_[reply_.work_index_] = msg;
|
reply_.work_index_++;
|
reply_.cv_msg_.notify_all();
|
}
|
|
static struct work *alloc_work(nng_socket sock)
|
{
|
struct work *w;
|
int rv;
|
|
if ((w = (struct work*)nng_alloc(sizeof(*w))) == NULL) {
|
return NULL;;
|
}
|
w->cb_recv = cb_recv_for_aio;
|
|
if ((rv = nng_aio_alloc(&w->aio, server_cb, w)) != 0) {
|
return NULL;
|
}
|
if ((rv = nng_ctx_open(&w->ctx, sock)) != 0) {
|
return NULL;
|
}
|
w->state = INIT;
|
return (w);
|
}
|
|
static constexpr int PARALLEL = 62;
|
static struct work* works[PARALLEL]{};
|
|
static int create_server(nng_socket* sock, const string& url){
|
TAG;
|
if (sock->id > 0) return 0;
|
|
int rv = nng_rep0_open(sock);
|
if (rv < 0){
|
PRNTVITAG("create_server nng_rep0_open faild");
|
PRNTVITAG(url);
|
return rv;
|
}
|
for (int i = 0; i < PARALLEL; i++) {
|
works[i] = alloc_work(*sock);
|
}
|
|
remove_exist(url);
|
rv = nng_listen(*sock, url.c_str(), NULL, 0);
|
if (rv < 0){
|
PRNTVITAG("create_server nng_listen failed");
|
PRNTVITAG(url);
|
return rv;
|
}
|
|
for (int i = 0; i < PARALLEL; i++) {
|
server_cb(works[i]); // this starts them going (INIT state)
|
}
|
|
return 0;
|
}
|
|
int start_reply(const std::string& url, const int port){
|
TAG;
|
|
string ipc = "ipc:///tmp/" + url;
|
if (url.find("ipc://") == 0){
|
ipc = url;
|
}
|
reply_.url_ = ipc;
|
if(create_server(&reply_.sock_local_, ipc) != 0) return -1;
|
|
if (port > 0){
|
reply_.port_ = port;
|
ipc = "tcp://0.0.0.0:" + to_string(port);
|
if(create_server(&reply_.sock_remote_, ipc) != 0) return -1;
|
}else {
|
reply_.sock_remote_.id = numeric_limits<int32_t>::max();
|
}
|
|
return 0;
|
}
|
|
int read_request(void** src, std::string* msg, const int to_ms){
|
|
if (reply_.sock_local_.id == 0 || reply_.sock_remote_.id == 0) {
|
if (start_reply(reply_.url_, reply_.port_) != 0)
|
return -1;
|
}
|
|
int tm = to_ms > 0 ? to_ms : 30;
|
|
uint64_t key{};
|
string tmpmsg;
|
{
|
unique_lock<mutex> l(reply_.mtx_msg_);
|
auto status = reply_.cv_msg_.wait_for(l, chrono::milliseconds{tm}, []{
|
return !reply_.msg_.empty();
|
});
|
if (!status){
|
PRNTVITAG("subscribe_read timeout");
|
return -1;
|
}
|
const auto& iter = reply_.msg_.begin();
|
key = iter->first;
|
tmpmsg = iter->second;
|
reply_.msg_.erase(iter);
|
}
|
*msg = move(tmpmsg);
|
|
auto s = (uint64_t*)malloc(sizeof(uint64_t));
|
*s = key;
|
*src = s;
|
|
return 0;
|
}
|
|
int send_reply(const void* src, const void* msg, const int msg_len){
|
struct work* w{};
|
{
|
auto key = *(static_cast<uint64_t*>(const_cast<void*>(src)));
|
|
lock_guard<mutex> l{reply_.mtx_msg_};
|
auto iter = reply_.works_.find(key);
|
if (iter == reply_.works_.end()) return -1;
|
w = iter->second;
|
reply_.works_.erase(iter);
|
}
|
|
TAG;
|
|
nng_msg_alloc(&w->msg, 0);
|
nng_msg_append(w->msg, msg, msg_len);
|
|
nng_sleep_aio(0, w->aio);
|
|
return 0;
|
}
|
|
|
}
|
