From abc89da58e0a321fc4d27d0ca70620cd14141d3a Mon Sep 17 00:00:00 2001
From: wangzhengquan <wangzhengquan85@126.com>
Date: 星期二, 01 十二月 2020 15:34:35 +0800
Subject: [PATCH] thread local, thread pool , thread safe
---
src/socket/net_mod_socket.c | 261 +++++++++++++++++---------------
src/socket/net_mod_socket.h | 17 -
lib/libusgcommon.so.bk | 0
src/socket/net_conn_pool.h | 37 ++++
src/socket/net_conn_pool.c | 109 +++++++++++++
5 files changed, 289 insertions(+), 135 deletions(-)
diff --git a/lib/libusgcommon.so b/lib/libusgcommon.so.bk
similarity index 100%
rename from lib/libusgcommon.so
rename to lib/libusgcommon.so.bk
Binary files differ
diff --git a/src/socket/net_conn_pool.c b/src/socket/net_conn_pool.c
new file mode 100644
index 0000000..77e91d3
--- /dev/null
+++ b/src/socket/net_conn_pool.c
@@ -0,0 +1,109 @@
+#include "net_conn_pool.h"
+#include "socket_io.h"
+#include "logger_factory.h"
+
+NetConnPool::NetConnPool() {
+ int i;
+ maxi = -1;
+ nready = 0 ;
+ for (i = 0; i < OPEN_MAX; i++) {
+ conns[i].fd = -1;
+ conns[i].events = 0;
+ }
+}
+
+NetConnPool::~NetConnPool() {
+ int connfd;
+ for (auto map_iter = connectionMap.begin(); map_iter != connectionMap.end(); map_iter++) {
+ connfd = map_iter->second;
+ Close(connfd);
+ }
+}
+
+
+int NetConnPool::getConn(const char *host, int port) {
+ std::map<std::string, int>::iterator mapIter;
+ int connfd;
+ int i;
+ char mapKey[ADDRSTRLEN];
+ char portstr[NI_MAXSERV];
+
+ sprintf(mapKey, "%s:%d", host, port);
+ mapIter = connectionMap.find(mapKey);
+ if( mapIter != connectionMap.end()) {
+ connfd = mapIter->second;
+// printf("hit: %s\n", mapKey);
+ } else {
+// printf("mis: %s\n", mapKey);
+ sprintf(portstr, "%d", port);
+// printf("open before: %s\n", mapKey);
+ connfd = open_clientfd(host, portstr);
+// printf("open after: %s\n", mapKey);
+ if(connfd < 0) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::connect %s:%d ", host, port);
+ return -1;
+ }
+ connectionMap.insert({mapKey, connfd});
+ }
+
+
+ for (i = 0; i < OPEN_MAX; i++) { /* Find an available slot */
+ if (conns[i].fd < 0)
+ {
+ /* Add connected descriptor to the req_resp_pool */
+ conns[i].fd = connfd;
+
+ conns[i].events = POLLIN;
+ /* Add the descriptor to descriptor set */
+ break;
+ }
+ }
+
+ if (i > maxi)
+ maxi = i;
+
+ if (i == OPEN_MAX) {
+ /* Couldn't find an empty slot */
+ LoggerFactory::getLogger()->error(errno, "add_client error: Too many clients");
+ return -1;
+ }
+
+
+ return connfd;
+}
+
+void NetConnPool::putConn(int connfd) {
+ int i;
+ for (i = 0; i <= maxi; i++) {
+ if(conns[i].fd == connfd) {
+ conns[i].fd = -1;
+ }
+ }
+}
+
+void NetConnPool::closeConn(int connfd) {
+ int i;
+ std::map<std::string, int>::iterator map_iter;
+ if(close(connfd) != 0) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::close_connect close");
+ }
+
+
+ for (i = 0; i <= maxi; i++) {
+ if(conns[i].fd == connfd) {
+ conns[i].fd = -1;
+ }
+ }
+
+ for ( map_iter = connectionMap.begin(); map_iter != connectionMap.end(); ) {
+ if(connfd == map_iter->second) {
+// std::cout << "map_iter->first==" << map_iter->first << std::endl;
+ map_iter = connectionMap.erase(map_iter);
+ } else {
+ ++map_iter;
+ }
+ }
+
+ // LoggerFactory::getLogger()->debug( "closed %d\n", connfd);
+
+}
\ No newline at end of file
diff --git a/src/socket/net_conn_pool.h b/src/socket/net_conn_pool.h
new file mode 100644
index 0000000..f8562c9
--- /dev/null
+++ b/src/socket/net_conn_pool.h
@@ -0,0 +1,37 @@
+#ifndef __NET_CONN_POOL_H__
+#define __NET_CONN_POOL_H__
+
+#include "usg_common.h"
+#include <poll.h>
+
+#define OPEN_MAX 1024
+
+#define ADDRSTRLEN (NI_MAXHOST + NI_MAXSERV + 10)
+
+class NetConnPool{ /* Represents a pool of connected descriptors */
+public:
+ int nready; /* Number of ready descriptors from select */
+ int maxi; /* Highwater index into client array */
+ struct pollfd conns[OPEN_MAX];
+ std::map<std::string, int> connectionMap;
+
+ NetConnPool() ;
+ ~NetConnPool() ;
+ /**
+ * 鑾峰彇杩炴帴
+ */
+ int getConn(const char *host, int port);
+ /**
+ * 鏀惧洖杩炴帴
+ */
+ void putConn(int connfd);
+ /**
+ * 鍏抽棴杩炴帴
+ */
+ void closeConn(int connfd);
+
+
+
+};
+
+#endif
\ No newline at end of file
diff --git a/src/socket/net_mod_socket.c b/src/socket/net_mod_socket.c
index 5e0b0b1..a3d4543 100644
--- a/src/socket/net_mod_socket.c
+++ b/src/socket/net_mod_socket.c
@@ -1,15 +1,21 @@
#include "net_mod_socket.h"
#include "socket_io.h"
#include "net_mod_socket_io.h"
+#include "net_conn_pool.h"
#include <sys/types.h> /* See NOTES */
#include <sys/socket.h>
+#include <pthread.h>
+static Logger *logger = LoggerFactory::getLogger();
+
+static pthread_once_t once = PTHREAD_ONCE_INIT;
+static pthread_key_t poolKey;
NetModSocket::NetModSocket()
{
- if (Signal(SIGPIPE, SIG_IGN) == SIG_ERR) err_msg(errno, "signal");
+ if (Signal(SIGPIPE, SIG_IGN) == SIG_ERR) err_msg(errno, "signal");
}
@@ -49,64 +55,26 @@
}
-
-int NetModSocket::connect( pool &mpool, net_node_t *node) {
-
- int connfd;
- int i;
- char portstr[32];
-
- // printf("mis: %s\n", mapKey);
- sprintf(portstr, "%d", node->port);
-// printf("open before: %s\n", mapKey);
- connfd = open_clientfd(node->host, portstr);
-// printf("open after: %s\n", mapKey);
- if(connfd < 0) {
- LoggerFactory::getLogger()->error(errno, "NetModSocket::connect %s:%d ", node->host, node->port);
- exit(1);
- return -1;
- }
-
- for (i = 0; i < OPEN_MAX; i++) { /* Find an available slot */
- if (mpool.conns[i].fd < 0)
- {
- /* Add connected descriptor to the mpool */
- mpool.conns[i].fd = connfd;
-
- mpool.conns[i].events = POLLIN;
- /* Add the descriptor to descriptor set */
- break;
- }
- }
-
- if (i > mpool.maxi)
- mpool.maxi = i;
-
- if (i == OPEN_MAX) {
- /* Couldn't find an empty slot */
- LoggerFactory::getLogger()->error(errno, "add_client error: Too many clients");
- return -1;
- }
-
- return connfd;
+/* Free thread-specific data buffer */
+void NetModSocket::_destructor_(void *_pool)
+{
+ NetConnPool *mpool = (NetConnPool *)_pool;
+ delete mpool;
}
+ /* One-time key creation function */
+void NetModSocket::_createKey_(void)
+{
+ int ret;
-void NetModSocket::close_connect(pool &mpool, int connfd) {
- int i;
- if(close(connfd) != 0) {
- LoggerFactory::getLogger()->error(errno, "NetModSocket::close_connect close");
+ /* Allocate a unique thread-specific data key and save the address
+ of the destructor for thread-specific data buffers */
+
+ ret = pthread_key_create(&poolKey, _destructor_);
+ if (ret != 0) {
+ logger->error(ret, "pthread_key_create");
+ exit(1);
}
-
-
- for (i = 0; i <= mpool.maxi; i++) {
- if(mpool.conns[i].fd == connfd) {
- mpool.conns[i].fd = -1;
- }
- }
-
- // LoggerFactory::getLogger()->debug( "NetModSocket::close_connect %d\n", connfd);
-
}
int NetModSocket::_sendandrecv_(net_node_t *node_arr, int arrlen, void *send_buf, int send_size,
@@ -116,7 +84,7 @@
net_node_t *node;
void *recv_buf;
- pool mpool;
+
net_mod_request_head_t request_head = {};
@@ -124,8 +92,39 @@
net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t));
+
+ int ret;
+ NetConnPool *mpool;
+
+ /* Make first caller allocate key for thread-specific data */
+
+ ret = pthread_once(&once, _createKey_);
+ if (ret != 0) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::_sendandrecv_ pthread_once");
+ exit(1);
+ }
+
+ mpool = (NetConnPool *)pthread_getspecific(poolKey);
+ if (mpool == NULL)
+ {
+ /* If first call from this thread, allocate
+ buffer for thread, and save its location */
+ mpool = new NetConnPool();
+ if (mpool == NULL) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::_sendandrecv_ malloc");
+ exit(1);
+ }
+
+
+
+ ret = pthread_setspecific(poolKey, mpool);
+ if (ret != 0) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::_sendandrecv_ pthread_setspecific");
+ exit(1);
+ }
+ }
- init_conn_pool(mpool);
+
for (i = 0; i< arrlen; i++) {
@@ -142,7 +141,7 @@
continue;
}
- if( (connfd = connect(mpool, node)) < 0 ) {
+ if( (connfd = mpool->getConn(node->host, node->port)) < 0 ) {
continue;
}
@@ -157,73 +156,76 @@
// printf("write_request %s:%d\n", request_head.host, request_head.port);
if(write_request(connfd, request_head, send_buf, send_size, NULL, 0) != 0) {
LoggerFactory::getLogger()->error("write_request failture %s:%d\n", node->host, node->port);
- close_connect(mpool, connfd);
+ mpool->closeConn( connfd);
} else {
n_req++;
}
}
-// printf(" mpool.maxi = %d\n", mpool.maxi);
+// printf(" mpool->maxi = %d\n", mpool->maxi);
// printf(" n_req = %d\n", n_req);
while(n_resp < n_req)
{
/* Wait for listening/connected descriptor(s) to become ready */
- if( (mpool.nready = poll(mpool.conns, mpool.maxi + 1, msec) ) <= 0) {
+ if( (mpool->nready = poll(mpool->conns, mpool->maxi + 1, msec) ) <= 0) {
// wirite_set 鍜� read_set 鍦ㄦ寚瀹氭椂闂村唴閮芥病鍑嗗濂�
break;
}
-// printf("mpool.nready =%d\n", mpool.nready);
- for (i = 0; (i <= mpool.maxi) && (mpool.nready > 0); i++) {
- if ( (connfd = mpool.conns[i].fd) > 0 ) {
+// printf("mpool->nready =%d\n", mpool->nready);
+ for (i = 0; (i <= mpool->maxi) && (mpool->nready > 0); i++) {
+ if ( (connfd = mpool->conns[i].fd) > 0 ) {
/* If the descriptor is ready, echo a text line from it */
- if (mpool.conns[i].revents & POLLIN )
+ if (mpool->conns[i].revents & POLLIN )
{
- mpool.nready--;
+ mpool->nready--;
// printf("POLLIN %d\n", connfd);
if( (n = read_response(connfd, ret_arr+n_recv_suc)) == 0) {
n_recv_suc++;
+ // 鎴愬姛鏀跺埌杩斿洖娑堟伅锛屾竻绌鸿鍏ヤ綅
+ mpool->conns[i].fd = -1;
}
- // else if(n == -1) {
- // // 缃戠粶閿欒
- // } else {
- // // 瀵规柟key鏄叧闂殑
- // }
+ else if(n == -1) {
+ // 缃戠粶閿欒
+ mpool->closeConn( connfd);
+ // mpool->conns[i].fd = -1;
+ } else {
+ // 瀵规柟key鏄叧闂殑
+ mpool->conns[i].fd = -1;
+ }
- mpool.conns[i].fd = -1;
- close_connect(mpool, connfd);
n_resp++;
// printf("read response %d\n", n);
}
- if (mpool.conns[i].revents & POLLOUT ) {
+ if (mpool->conns[i].revents & POLLOUT ) {
// printf("poll POLLOUT %d\n", connfd);
}
- if (mpool.conns[i].revents & (POLLRDHUP | POLLHUP | POLLERR) )
+ if (mpool->conns[i].revents & (POLLRDHUP | POLLHUP | POLLERR) )
{
// printf("poll POLLERR %d\n", connfd);
- mpool.nready--;
- close_connect(mpool, connfd);
- mpool.conns[i].fd = -1;
+ mpool->nready--;
+ mpool->closeConn( connfd);
+ // mpool->conns[i].fd = -1;
}
}
}
}
//瓒呮椂鍚庯紝鍏抽棴瓒呮椂杩炴帴
- for (i = 0; i <= mpool.maxi; i++) {
- if ( (connfd = mpool.conns[i].fd) > 0 ) {
+ for (i = 0; i <= mpool->maxi; i++) {
+ if ( (connfd = mpool->conns[i].fd) > 0 ) {
// 鍏抽棴骞舵竻闄ゅ啓鍏ユ垨璇诲彇澶辫触鐨勮繛鎺�
- close_connect(mpool, connfd);
- mpool.conns[i].fd = -1;
+ mpool->closeConn( connfd);
+ // mpool->conns[i].fd = -1;
}
}
- mpool.maxi = -1;
+ mpool->maxi = -1;
*recv_arr = ret_arr;
if(recv_arr_size != NULL) {
@@ -257,10 +259,33 @@
net_mod_recv_msg_t recv_msg;
char portstr[32];
int n_req = 0, n_pub_suc = 0, n_resp = 0;
- int ret;
- pool mpool;
- init_conn_pool(mpool);
+ int ret;
+ NetConnPool *mpool;
+
+ /* Make first caller allocate key for thread-specific data */
+ ret = pthread_once(&once, _createKey_);
+ if (ret != 0) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::_sendandrecv_ pthread_once");
+ exit(1);
+ }
+
+ mpool = (NetConnPool *)pthread_getspecific(poolKey);
+ if (mpool == NULL)
+ {
+ /* If first call from this thread, allocte buffer for thread, and save its location */
+ mpool = new NetConnPool();
+ if (mpool == NULL) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::_sendandrecv_ malloc");
+ exit(1);
+ }
+
+ ret = pthread_setspecific(poolKey, mpool);
+ if (ret != 0) {
+ LoggerFactory::getLogger()->error(errno, "NetModSocket::_sendandrecv_ pthread_setspecific");
+ exit(1);
+ }
+ }
for (i = 0; i < arrlen; i++) {
@@ -274,7 +299,7 @@
} else {
sprintf(portstr, "%d", node->port);
- if( (connfd = connect(mpool, node)) < 0 ) {
+ if( (connfd = mpool->getConn(node->host, node->port)) < 0 ) {
continue;
}
request_head.mod = BUS;
@@ -285,7 +310,7 @@
if(write_request(connfd, request_head, content, content_size, topic, request_head.topic_length) != 0) {
LoggerFactory::getLogger()->error(" NetModSocket::_pub_ write_request failture %s:%d\n", node->host, node->port);
- close_connect(mpool, connfd);
+ mpool->closeConn( connfd);
} else {
n_req++;
}
@@ -296,61 +321,60 @@
while(n_resp < n_req)
{
/* Wait for listening/connected descriptor(s) to become ready */
- if( (mpool.nready = poll(mpool.conns, mpool.maxi + 1, timeout) ) <= 0) {
+ if( (mpool->nready = poll(mpool->conns, mpool->maxi + 1, timeout) ) <= 0) {
// wirite_set 鍜� read_set 鍦ㄦ寚瀹氭椂闂村唴閮芥病鍑嗗濂�
break;
}
-// printf("mpool.nready =%d\n", mpool.nready);
- for (i = 0; (i <= mpool.maxi) && (mpool.nready > 0); i++) {
- if ( (connfd = mpool.conns[i].fd) > 0 ) {
- if (mpool.conns[i].revents & POLLIN )
+// printf("mpool->nready =%d\n", mpool->nready);
+ for (i = 0; (i <= mpool->maxi) && (mpool->nready > 0); i++) {
+ if ( (connfd = mpool->conns[i].fd) > 0 ) {
+ if (mpool->conns[i].revents & POLLIN )
{
- mpool.nready--;
+ mpool->nready--;
// printf("POLLIN %d\n", connfd);
if( (ret = read_response(connfd, &recv_msg)) == 0) {
// 鎴愬姛鏀跺埌杩斿洖娑堟伅锛屾竻绌鸿鍏ヤ綅
- mpool.conns[i].fd = -1;
+ mpool->conns[i].fd = -1;
n_pub_suc++;
}
- // else if(ret == -1) {
- // // 缃戠粶杩炴帴閿欒
- // } else {
- // // 瀵规柟鐨刱ey鏄叧闂殑
- // }
- mpool.conns[i].fd = -1;
- close_connect(mpool, connfd);
+ else if(ret == -1) {
+ // 缃戠粶杩炴帴閿欒
+ mpool->closeConn( connfd);
+ } else {
+ // 瀵规柟鐨刱ey鏄叧闂殑
+ mpool->conns[i].fd = -1;
+ }
n_resp++;
// printf("read response %d\n", n);
-
}
- if (mpool.conns[i].revents & POLLOUT ) {
+ if (mpool->conns[i].revents & POLLOUT ) {
// printf("poll POLLOUT %d\n", connfd);
}
- if (mpool.conns[i].revents & (POLLRDHUP | POLLHUP | POLLERR) )
+ if (mpool->conns[i].revents & (POLLRDHUP | POLLHUP | POLLERR) )
{
// printf("poll POLLERR %d\n", connfd);
- mpool.nready--;
+ mpool->nready--;
- mpool.conns[i].fd = -1;
- close_connect(mpool, connfd);
+ mpool->conns[i].fd = -1;
+ mpool->closeConn( connfd);
}
}
}
}
//瓒呮椂鍚庯紝鍏抽棴瓒呮椂杩炴帴
- for (i = 0; i <= mpool.maxi; i++) {
- if ( (connfd = mpool.conns[i].fd) > 0 ) {
+ for (i = 0; i <= mpool->maxi; i++) {
+ if ( (connfd = mpool->conns[i].fd) > 0 ) {
// 鍏抽棴骞舵竻闄ゅ啓鍏ユ垨璇诲彇澶辫触鐨勮繛鎺�
- close_connect(mpool, connfd);
- mpool.conns[i].fd = -1;
+ mpool->closeConn( connfd);
+ // mpool->conns[i].fd = -1;
}
}
- mpool.maxi = -1;
+ mpool->maxi = -1;
return n_pub_suc;
}
@@ -706,17 +730,6 @@
-void NetModSocket::init_conn_pool(pool& mpool)
-{
- /* Initially, there are no connected descriptors */
- int i;
- mpool.maxi = -1; //line:conc:echoservers:beginempty
- for (i = 0; i < OPEN_MAX; i++) {
- mpool.conns[i].fd = -1;
- mpool.conns[i].events = 0;
- }
-
-}
/**
uint32_t mod;
diff --git a/src/socket/net_mod_socket.h b/src/socket/net_mod_socket.h
index 5e7b7b7..367dec8 100644
--- a/src/socket/net_mod_socket.h
+++ b/src/socket/net_mod_socket.h
@@ -5,12 +5,13 @@
#include "socket_io.h"
#include <poll.h>
-#define OPEN_MAX 1024
+
#define GET(p) (*(uint32_t *)(p))
#define PUT(p, val) (*(uint32_t *)(p) = (val))
#define GET_INT32(p) (*(int32_t *)(p))
#define PUT_INT32(p, val) (*(int32_t *)(p) = (val))
+
class NetModServerSocket;
@@ -60,13 +61,7 @@
};
class NetModSocket {
- struct pool{ /* Represents a pool of connected descriptors */ //line:conc:echoservers:beginpool
-
- int nready; /* Number of ready descriptors from select */
- int maxi; /* Highwater index into client array */
- struct pollfd conns[OPEN_MAX];
- // std::map<std::string, int> connectionMap;
- } ;
+
friend class NetModServerSocket;
private:
@@ -82,9 +77,9 @@
static void * encode_response_head(net_mod_response_head_t & response);
static net_mod_response_head_t decode_response_head(void *_headbs);
- void init_conn_pool( pool& mpool);
- int connect(pool& mpool, net_node_t* node);
- void close_connect(pool& mpool , int connfd);
+ static void _destructor_(void *_pool);
+ static void _createKey_(void);
+
int read_response(int clientfd, net_mod_recv_msg_t *recv_msg);
int write_request(int clientfd, net_mod_request_head_t &request_head, void *send_buf, int send_size, void *topic_buf, int topic_size);
--
Gitblit v1.8.0