#include "net_mod_socket.h"
|
#include "socket_io.h"
|
#include "net_mod_socket_io.h"
|
#include <sys/types.h> /* See NOTES */
|
#include <sys/socket.h>
|
|
|
|
|
NetModSocket::NetModSocket()
|
{
|
init_req_rep_req_resp_pool();
|
|
if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) err_msg(errno, "signal");
|
}
|
|
|
NetModSocket::~NetModSocket() {
|
int clientfd;
|
for (auto map_iter = req_resp_pool.connectionMap.begin(); map_iter != req_resp_pool.connectionMap.end(); map_iter++) {
|
clientfd = map_iter->second;
|
Close(clientfd);
|
|
}
|
|
}
|
|
|
void NetModSocket::init_req_rep_req_resp_pool()
|
{
|
/* Initially, there are no connected descriptors */
|
int i;
|
req_resp_pool.maxi = -1; //line:conc:echoservers:beginempty
|
for (i = 0; i < OPEN_MAX; i++) {
|
req_resp_pool.conns[i].fd = -1;
|
req_resp_pool.conns[i].events = 0;
|
}
|
|
}
|
|
int NetModSocket::connect( net_node_t *node) {
|
std::map<std::string, int>::iterator mapIter;
|
int connfd;
|
int i;
|
char mapKey[256];
|
char portstr[32];
|
|
sprintf(mapKey, "%s:%d", node->host, node->port);
|
mapIter = req_resp_pool.connectionMap.find(mapKey);
|
if( mapIter != req_resp_pool.connectionMap.end()) {
|
connfd = mapIter->second;
|
// printf("hit: %s\n", mapKey);
|
} else {
|
// printf("mis: %s\n", mapKey);
|
sprintf(portstr, "%d", node->port);
|
// printf("open before: %s\n", mapKey);
|
connfd = open_clientfd(node->host, portstr);
|
// printf("open after: %s\n", mapKey);
|
if(connfd < 0) {
|
LoggerFactory::getLogger()->error(errno, "connect %s:%d ", node->host, node->port);
|
return -1;
|
}
|
req_resp_pool.connectionMap.insert({mapKey, connfd});
|
}
|
|
|
for (i = 0; i < OPEN_MAX; i++) { /* Find an available slot */
|
if (req_resp_pool.conns[i].fd < 0)
|
{
|
/* Add connected descriptor to the req_resp_pool */
|
req_resp_pool.conns[i].fd = connfd;
|
|
req_resp_pool.conns[i].events = POLLIN;
|
/* Add the descriptor to descriptor set */
|
break;
|
}
|
}
|
|
if (i > req_resp_pool.maxi)
|
req_resp_pool.maxi = i;
|
|
if (i == OPEN_MAX) {
|
/* Couldn't find an empty slot */
|
LoggerFactory::getLogger()->error(errno, "add_client error: Too many clients");
|
return -1;
|
}
|
|
|
return connfd;
|
}
|
|
void NetModSocket::close_connect(int connfd) {
|
int i;
|
char mapKey[256];
|
std::map<std::string, int>::iterator map_iter;
|
if(close(connfd) != 0) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::close_connect close");
|
}
|
|
|
for (i = 0; i <= req_resp_pool.maxi; i++) {
|
if(req_resp_pool.conns[i].fd == connfd) {
|
req_resp_pool.conns[i].fd = -1;
|
}
|
}
|
|
for ( map_iter = req_resp_pool.connectionMap.begin(); map_iter != req_resp_pool.connectionMap.end(); ) {
|
if(connfd == map_iter->second) {
|
// std::cout << "map_iter->first==" << map_iter->first << std::endl;
|
map_iter = req_resp_pool.connectionMap.erase(map_iter);
|
} else {
|
++map_iter;
|
}
|
}
|
|
LoggerFactory::getLogger()->debug( "closed %d\n", connfd);
|
|
}
|
|
|
int NetModSocket::sendandrecv(net_node_t *node_arr, int arrlen, void *send_buf, int send_size,
|
net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) {
|
|
int i, n, recv_size, connfd;
|
net_node_t *node;
|
void *recv_buf;
|
int timeout = 5 * 1000;
|
|
int n_req = 0, n_recv_suc = 0, n_resp;
|
|
net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t));
|
|
//init_req_rep_req_resp_pool();
|
|
for (i = 0; i< arrlen; i++) {
|
|
node = &node_arr[i];
|
if(node->host == NULL) {
|
// 本地发送
|
shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size);
|
strcpy( ret_arr[n_recv_suc].host,"localshm");
|
ret_arr[n_recv_suc].port = 0;
|
ret_arr[n_recv_suc].key = node->key;
|
ret_arr[n_recv_suc].content = recv_buf;
|
ret_arr[n_recv_suc].content_length = recv_size;
|
n_recv_suc++;
|
continue;
|
}
|
|
if( (connfd = connect(node)) < 0 ) {
|
continue;
|
}
|
|
// printf("write_request %s:%d\n", node->host, node->port);
|
if(write_request(connfd, node->key, send_buf, send_size) != 0) {
|
LoggerFactory::getLogger()->error("write_request failture %s:%d\n", node->host, node->port);
|
close_connect(connfd);
|
// req_resp_pool.conns[i].fd = -1;
|
} else {
|
n_req++;
|
}
|
|
}
|
|
// printf(" req_resp_pool.maxi = %d\n", req_resp_pool.maxi);
|
// printf(" n_req = %d\n", n_req);
|
|
// int tmp = 0;
|
while(n_resp < n_req)
|
{
|
// printf(" while %d\n", tmp++);
|
/* Wait for listening/connected descriptor(s) to become ready */
|
if( (req_resp_pool.nready = poll(req_resp_pool.conns, req_resp_pool.maxi + 1, timeout) ) <= 0) {
|
// wirite_set 和 read_set 在指定时间内都没准备好
|
break;
|
}
|
// printf("req_resp_pool.nready =%d\n", req_resp_pool.nready);
|
for (i = 0; (i <= req_resp_pool.maxi) && (req_resp_pool.nready > 0); i++) {
|
if ( (connfd = req_resp_pool.conns[i].fd) > 0 ) {
|
/* If the descriptor is ready, echo a text line from it */
|
if (req_resp_pool.conns[i].revents & POLLIN )
|
{
|
req_resp_pool.nready--;
|
// printf("POLLIN %d\n", connfd);
|
if( (n = read_response(connfd, ret_arr+n_recv_suc)) == 0) {
|
|
// 成功收到返回消息，清空读入位
|
req_resp_pool.conns[i].fd = -1;
|
n_recv_suc++;
|
|
} else if(n == -1) {
|
req_resp_pool.conns[i].fd = -1;
|
close_connect(connfd);
|
} else {
|
req_resp_pool.conns[i].fd = -1;
|
|
}
|
n_resp++;
|
// printf("read response %d\n", n);
|
|
}
|
|
if (req_resp_pool.conns[i].revents & POLLOUT ) {
|
// printf("poll POLLOUT %d\n", connfd);
|
}
|
|
if (req_resp_pool.conns[i].revents & (POLLRDHUP | POLLHUP | POLLERR) )
|
{
|
// printf("poll POLLERR %d\n", connfd);
|
req_resp_pool.nready--;
|
close_connect(connfd);
|
req_resp_pool.conns[i].fd = -1;
|
}
|
}
|
}
|
}
|
|
for (i = 0; i <= req_resp_pool.maxi; i++) {
|
if ( (connfd = req_resp_pool.conns[i].fd) > 0 ) {
|
// 关闭并清除写入或读取失败的连接
|
close_connect(connfd);
|
req_resp_pool.conns[i].fd = -1;
|
}
|
}
|
|
req_resp_pool.maxi = -1;
|
|
*recv_arr = ret_arr;
|
if(recv_arr_size != NULL) {
|
*recv_arr_size = n_recv_suc;
|
}
|
return n_recv_suc;
|
|
}
|
|
int NetModSocket::write_request(int clientfd, int key, void *send_buf, int send_size) {
|
net_mod_request_head_t request_head = {};
|
int buf_size;
|
char *buf;
|
int max_buf_size;
|
if((buf = (char *)malloc(MAXBUF)) == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::NetModSocket malloc");
|
exit(1);
|
} else {
|
max_buf_size = MAXBUF;
|
}
|
|
buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH;
|
if(max_buf_size < buf_size) {
|
|
if((buf = (char *)realloc(buf, buf_size)) == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request realloc buf");
|
exit(1);
|
} else {
|
max_buf_size = buf_size;
|
}
|
}
|
|
request_head.mod = REQ_REP;
|
request_head.key = key;
|
request_head.content_length = send_size;
|
request_head.topic_length = 0;
|
|
// optval = 1;
|
// setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval));
|
memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size);
|
|
|
if(rio_writen(clientfd, buf, buf_size) != buf_size ) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request rio_writen");
|
free(buf);
|
return -1;
|
}
|
free(buf);
|
return 0;
|
}
|
|
/**
|
* @return 0 成功, 1 对方没有对应的key, -1 网络错误
|
*
|
*/
|
int NetModSocket::read_response(int connfd, net_mod_recv_msg_t *recv_msg) {
|
int recv_size;
|
void *recv_buf;
|
char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH];
|
|
net_mod_response_head_t response_head;
|
if ( rio_readn(connfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::read_response rio_readnb response_head");
|
|
return -1;
|
}
|
|
response_head = NetModSocket::decode_response_head(response_head_bs);
|
if(response_head.code != 0) {
|
// 对方没有对应的key
|
return 1;
|
}
|
|
recv_buf = malloc(response_head.content_length);
|
if(recv_buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc recv_buf");
|
exit(1);
|
}
|
if ( (recv_size = rio_readn(connfd, recv_buf, response_head.content_length) ) != response_head.content_length) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::read_response rio_readnb recv_buf");
|
|
return -1;
|
}
|
|
strcpy( recv_msg->host, response_head.host);
|
recv_msg->port = response_head.port;
|
recv_msg->key = response_head.key;
|
recv_msg->content = recv_buf;
|
recv_msg->content_length = recv_size;
|
return 0;
|
}
|
|
int NetModSocket::sendandrecv_safe(net_node_t *node_arr, int arrlen, void *send_buf, int send_size,
|
net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) {
|
|
int i, clientfd;
|
net_node_t *node;
|
void *recv_buf;
|
int recv_size;
|
char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH];
|
net_mod_request_head_t request_head = {};
|
net_mod_response_head_t response_head;
|
|
|
char portstr[32];
|
char *buf = NULL;
|
int buf_size, max_buf_size;
|
|
if(buf == NULL) {
|
buf = (char *)malloc(MAXBUF);
|
max_buf_size = MAXBUF;
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc");
|
}
|
|
int nsuc = 0;
|
net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t));
|
|
for (i = 0; i< arrlen; i++) {
|
|
node = &node_arr[i];
|
if(node->host == NULL) {
|
// 本地发送
|
shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size);
|
goto LABEL_ARR_PUSH;
|
}
|
|
sprintf(portstr, "%d", node->port);
|
clientfd = open_clientfd(node->host, portstr);
|
if(clientfd < 0) {
|
continue;
|
}
|
|
buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH;
|
if(max_buf_size < buf_size) {
|
if((buf = (char *)realloc(buf, buf_size)) == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe realloc buf");
|
} else {
|
max_buf_size = buf_size;
|
}
|
|
}
|
|
|
request_head.mod = REQ_REP;
|
request_head.key = node->key;
|
request_head.content_length = send_size;
|
request_head.topic_length = 0;
|
|
// optval = 1;
|
// setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval));
|
memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size);
|
|
|
if(rio_writen(clientfd, buf, buf_size) != buf_size ) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe rio_writen buf");
|
|
close(clientfd);
|
continue;
|
}
|
// optval = 0;
|
// setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval));
|
|
if ( rio_readn(clientfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe rio_readnb response_head_bs");
|
|
close(clientfd);
|
continue;
|
}
|
|
response_head = NetModSocket::decode_response_head(response_head_bs);
|
if(response_head.code != 0) {
|
continue;
|
}
|
|
recv_buf = malloc(response_head.content_length);
|
if(recv_buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc");
|
exit(1);
|
}
|
if ( (recv_size = rio_readn(clientfd, recv_buf, response_head.content_length) ) != response_head.content_length) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe rio_readnb recv_buf");
|
|
close(clientfd);
|
continue;
|
}
|
|
LABEL_ARR_PUSH:
|
if(node->host != NULL) {
|
strcpy(ret_arr[nsuc].host, node->host);
|
} else {
|
strcpy(ret_arr[nsuc].host, "local");
|
}
|
|
ret_arr[nsuc].port = node->port;
|
ret_arr[nsuc].key = node->key;
|
ret_arr[nsuc].content = recv_buf;
|
ret_arr[nsuc].content_length = recv_size;
|
|
nsuc++;
|
}
|
|
*recv_arr = ret_arr;
|
if(recv_arr_size != NULL) {
|
*recv_arr_size = nsuc;
|
}
|
|
free(buf);
|
return nsuc;
|
|
}
|
|
// int pub(char *topic, int topic_size, void *content, int content_size, int port);
|
|
int NetModSocket::pub(net_node_t *node_arr, int arrlen, char *topic, int topic_size, void *content, int content_size) {
|
int i, clientfd;
|
net_node_t *node;
|
|
char *buf;
|
int max_buf_size, buf_size;
|
|
net_mod_request_head_t request_head;
|
|
char portstr[32];
|
|
buf = (char *)malloc(MAXBUF);
|
max_buf_size = MAXBUF;
|
if(buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc");
|
exit(1);
|
}
|
int nsuc = 0;
|
for (i = 0; i< arrlen; i++) {
|
|
node = &node_arr[i];
|
if(node->host == NULL) {
|
// 本地发送
|
shmModSocket.pub(topic, topic_size, content, content_size, node->key);
|
|
} else {
|
sprintf(portstr, "%d", node->port);
|
clientfd = open_clientfd(node->host, portstr);
|
if(clientfd < 0) {
|
continue;
|
}
|
|
request_head.mod = BUS;
|
request_head.key = node->key;
|
request_head.content_length = content_size;
|
request_head.topic_length = strlen(topic) + 1;
|
|
buf_size = NET_MODE_REQUEST_HEAD_LENGTH + content_size + request_head.topic_length;
|
|
if(max_buf_size < buf_size) {
|
if( ( buf = (char *)realloc(buf, buf_size)) == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::pub realloc buf ");
|
} else {
|
max_buf_size = buf_size;
|
}
|
|
}
|
|
memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, content, content_size);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH + content_size, topic, request_head.topic_length);
|
|
if(rio_writen(clientfd, buf, buf_size) != buf_size ) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::pub rio_writen conent ");
|
close(clientfd);
|
continue;
|
}
|
}
|
|
nsuc++;
|
}
|
|
free(buf);
|
return nsuc;
|
}
|
|
void NetModSocket::free_recv_msg_arr(net_mod_recv_msg_t * arr, size_t size) {
|
|
for(int i =0; i< size; i++) {
|
free(arr[i].content);
|
}
|
free(arr);
|
}
|
|
/**
|
uint32_t mod;
|
char host[NI_MAXHOST];
|
uint32_t port;
|
uint32_t key;
|
uint32_t content_length;
|
uint32_t topic_length;
|
*/
|
|
void * NetModSocket::encode_request_head(net_mod_request_head_t & request) {
|
void * headbs = malloc(NET_MODE_REQUEST_HEAD_LENGTH);
|
char *tmp_ptr = (char *)headbs;
|
PUT(tmp_ptr, htonl(request.mod));
|
|
tmp_ptr += 4;
|
memcpy(tmp_ptr, request.host, NI_MAXHOST);
|
|
tmp_ptr += NI_MAXHOST;
|
PUT(tmp_ptr, htonl(request.port));
|
|
tmp_ptr += 4;
|
PUT(tmp_ptr, htonl(request.key));
|
|
tmp_ptr += 4;
|
PUT(tmp_ptr, htonl(request.content_length));
|
|
tmp_ptr += 4;
|
PUT(tmp_ptr, htonl(request.topic_length));
|
|
|
return headbs;
|
}
|
|
net_mod_request_head_t NetModSocket::decode_request_head(void *headbs) {
|
char *tmp_ptr = (char *)headbs;
|
net_mod_request_head_t head;
|
|
head.mod = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += NI_MAXHOST;
|
memcpy(head.host, tmp_ptr, NI_MAXHOST);
|
|
tmp_ptr += 4;
|
head.port = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += 4;
|
head.key = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += 4;
|
head.content_length = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += 4;
|
head.topic_length = ntohl(GET(tmp_ptr));
|
|
return head;
|
}
|
|
/**
|
char host[NI_MAXHOST];
|
uint32_t port;
|
uint32_t key;
|
uint32_t content_length;
|
uint32_t code;
|
*/
|
void * NetModSocket::encode_response_head(net_mod_response_head_t & response) {
|
void * headbs = malloc(NET_MODE_RESPONSE_HEAD_LENGTH);
|
char *tmp_ptr = (char *)headbs;
|
|
memcpy(tmp_ptr, response.host, NI_MAXHOST);
|
|
tmp_ptr += NI_MAXHOST;
|
PUT(tmp_ptr, htonl(response.port));
|
|
tmp_ptr += 4;
|
PUT(tmp_ptr , htonl(response.key));
|
|
tmp_ptr += 4;
|
PUT(tmp_ptr, htonl(response.content_length));
|
|
tmp_ptr += 4;
|
PUT(tmp_ptr, htonl(response.code));
|
|
return headbs;
|
}
|
|
net_mod_response_head_t NetModSocket::decode_response_head(void *headbs) {
|
char *tmp_ptr = (char *)headbs;
|
net_mod_response_head_t head;
|
|
memcpy(head.host, tmp_ptr, NI_MAXHOST);
|
|
tmp_ptr += NI_MAXHOST;
|
head.port = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += 4;
|
head.key = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += 4;
|
head.content_length = ntohl(GET(tmp_ptr));
|
|
tmp_ptr += 4;
|
head.code = ntohl(GET(tmp_ptr));
|
|
return head;
|
}
|
