#include "net_mod_socket.h"
|
#include "socket_io.h"
|
#include "net_mod_socket_io.h"
|
#include <sys/types.h> /* See NOTES */
|
#include <sys/socket.h>
|
|
|
|
|
NetModSocket::NetModSocket()
|
{
|
init_req_rep_req_resp_pool();
|
|
|
}
|
|
|
NetModSocket::~NetModSocket() {
|
int clientfd;
|
for (auto map_iter = req_resp_pool.connectionMap.begin(); map_iter != req_resp_pool.connectionMap.end(); map_iter++) {
|
clientfd = map_iter->second;
|
Close(clientfd);
|
|
}
|
|
}
|
|
|
void NetModSocket::init_req_rep_req_resp_pool()
|
{
|
/* Initially, there are no connected descriptors */
|
int i;
|
req_resp_pool.maxi = -1; //line:conc:echoservers:beginempty
|
for (i = 0; i < FD_SETSIZE; i++)
|
req_resp_pool.connfd[i] = -1; //line:conc:echoservers:endempty
|
|
/* Initially, listenfd is only member of select read set */
|
FD_ZERO(&req_resp_pool.read_set);
|
FD_ZERO(&req_resp_pool.write_set);
|
FD_ZERO(&req_resp_pool.except_set);
|
}
|
|
int NetModSocket::connect( net_node_t *node) {
|
std::map<std::string, int>::iterator mapIter;
|
int connfd;
|
int i;
|
char mapKey[256];
|
char portstr[32];
|
|
sprintf(mapKey, "%s:%d", node->host, node->port);
|
if( ( mapIter = req_resp_pool.connectionMap.find(mapKey)) != req_resp_pool.connectionMap.end()) {
|
connfd = mapIter->second;
|
|
} else {
|
|
sprintf(portstr, "%d", node->port);
|
connfd = open_clientfd(node->host, portstr);
|
if(connfd < 0) {
|
return -1;
|
}
|
req_resp_pool.connectionMap.insert({mapKey, connfd});
|
}
|
|
|
for (i = 0; i < FD_SETSIZE; i++) { /* Find an available slot */
|
if (req_resp_pool.connfd[i] < 0)
|
{
|
/* Add connected descriptor to the req_resp_pool */
|
req_resp_pool.connfd[i] = connfd;
|
req_resp_pool.connfdNodeMap.insert({connfd, node});
|
// Rio_readinitb(&req_resp_pool.clientrio[i], connfd); //line:conc:echoservers:endaddclient
|
|
/* Add the descriptor to descriptor set */
|
FD_SET(connfd, &req_resp_pool.read_set); //line:conc:echoservers:addconnfd
|
FD_SET(connfd, &req_resp_pool.write_set);
|
FD_SET(connfd, &req_resp_pool.except_set);
|
/* Update max descriptor and req_resp_pool highwater mark */
|
if (connfd > req_resp_pool.maxfd)
|
req_resp_pool.maxfd = connfd;
|
if (i > req_resp_pool.maxi)
|
req_resp_pool.maxi = i;
|
break;
|
}
|
}
|
|
if (i == FD_SETSIZE) {
|
/* Couldn't find an empty slot */
|
LoggerFactory::getLogger()->error(errno, "add_client error: Too many clients");
|
return -1;
|
|
}
|
|
return connfd;
|
}
|
|
void NetModSocket::close_connect(int connfd) {
|
|
net_node_t *node = req_resp_pool.connfdNodeMap.find(connfd)->second;
|
|
// std::map<std::string, int>::iterator mapIter;
|
if(close(connfd) != 0) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::close_connect");
|
}
|
FD_CLR(connfd, &req_resp_pool.read_set);
|
FD_CLR(connfd, &req_resp_pool.write_set);
|
FD_CLR(connfd, &req_resp_pool.except_set);
|
|
char mapKey[256];
|
// char portstr[32];
|
sprintf(mapKey, "%s:%d", node->host, node->port);
|
req_resp_pool.connectionMap.erase(mapKey);
|
// LoggerFactory::getLogger()->debug("close_connect");
|
}
|
|
|
int NetModSocket::sendandrecv(net_node_t *node_arr, int arrlen, void *send_buf, int send_size,
|
net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) {
|
|
int i, n, recv_size, connfd;
|
net_node_t *node;
|
void *recv_buf;
|
struct timeval timeout = {5, 0};
|
|
int n_conn_suc = 0, n_recv_suc = 0;
|
|
net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t));
|
|
//init_req_rep_req_resp_pool();
|
|
for (i = 0; i< arrlen; i++) {
|
|
node = &node_arr[i];
|
if(node->host == NULL) {
|
// 本地发送
|
shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size);
|
strcpy( ret_arr[n_recv_suc].host,"localshm");
|
ret_arr[n_recv_suc].port = 0;
|
ret_arr[n_recv_suc].key = node->key;
|
ret_arr[n_recv_suc].content = recv_buf;
|
ret_arr[n_recv_suc].content_length = recv_size;
|
n_recv_suc++;
|
continue;
|
}
|
|
if( (connfd = connect(node)) < 0 ) {
|
continue;
|
}
|
|
n_conn_suc++;
|
}
|
|
// printf("n_conn_suc =%d\n", n_conn_suc);
|
|
while(n_recv_suc < n_conn_suc)
|
{
|
/* Wait for listening/connected descriptor(s) to become ready */
|
req_resp_pool.ready_read_set = req_resp_pool.read_set;
|
req_resp_pool.ready_write_set = req_resp_pool.write_set;
|
req_resp_pool.ready_except_set = req_resp_pool.except_set;
|
if( (req_resp_pool.nready = select(req_resp_pool.maxfd + 1,
|
&req_resp_pool.ready_read_set, &req_resp_pool.ready_write_set,
|
&req_resp_pool.ready_except_set, &timeout)) <= 0) {
|
// wirite_set 和 read_set 在指定时间内都没准备好
|
break;
|
}
|
// printf("req_resp_pool.nready =%d\n", req_resp_pool.nready);
|
for (i = 0; (i <= req_resp_pool.maxi) && (req_resp_pool.nready > 0); i++) {
|
if ( (connfd = req_resp_pool.connfd[i]) > 0 ) {
|
/* If the descriptor is ready, echo a text line from it */
|
node = req_resp_pool.connfdNodeMap.find(connfd)->second;
|
if ( FD_ISSET(connfd, &req_resp_pool.ready_read_set))
|
{
|
req_resp_pool.nready--;
|
if( (n = read_response(connfd, ret_arr+n_recv_suc)) == 0) {
|
|
// 成功收到返回消息，清空读入位
|
FD_CLR(connfd, &req_resp_pool.read_set);
|
req_resp_pool.connfd[i] = -1;
|
n_recv_suc++;
|
} else if(n == -1) {
|
close_connect(connfd);
|
}
|
|
}
|
|
if (FD_ISSET(connfd, &req_resp_pool.ready_write_set))
|
{
|
req_resp_pool.nready--;
|
// printf("write %d\n", connfd);
|
if(write_request(connfd, node->key, send_buf, send_size) != 0) {
|
close_connect(connfd);
|
} else{
|
// 一次写入完成后清空写入位
|
FD_CLR(connfd, &req_resp_pool.write_set);
|
}
|
|
}
|
if (FD_ISSET(connfd, &req_resp_pool.ready_except_set))
|
{
|
req_resp_pool.nready--;
|
close_connect(connfd);
|
}
|
}
|
}
|
}
|
|
FD_ZERO(&req_resp_pool.except_set);
|
for (i = 0; i <= req_resp_pool.maxi; i++) {
|
if ( (connfd = req_resp_pool.connfd[i]) > 0 ) {
|
// 关闭并清除写入或读取失败的连接
|
close_connect(connfd);
|
}
|
}
|
req_resp_pool.maxi = -1;
|
|
*recv_arr = ret_arr;
|
if(recv_arr_size != NULL) {
|
*recv_arr_size = n_recv_suc;
|
}
|
return n_recv_suc;
|
|
}
|
|
int NetModSocket::write_request(int clientfd, int key, void *send_buf, int send_size) {
|
net_mod_request_head_t request_head = {};
|
int buf_size;
|
char *buf;
|
int max_buf_size;
|
buf = (char *)malloc(MAXBUF);
|
if(buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::NetModSocket malloc");
|
exit(1);
|
} else {
|
max_buf_size = MAXBUF;
|
}
|
|
buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH;
|
if(max_buf_size < buf_size) {
|
buf = (char *)realloc(buf, buf_size);
|
max_buf_size = buf_size;
|
if(buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request realloc");
|
exit(1);
|
}
|
}
|
|
request_head.mod = REQ_REP;
|
request_head.key = key;
|
request_head.content_length = send_size;
|
request_head.topic_length = 0;
|
|
// optval = 1;
|
// setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval));
|
memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size);
|
|
|
if(rio_writen(clientfd, buf, buf_size) != buf_size ) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request rio_writen");
|
free(buf);
|
return -1;
|
}
|
free(buf);
|
return 0;
|
}
|
|
/**
|
* @return 0 成功, 1 对方没有对应的key, -1 网络错误
|
*
|
*/
|
int NetModSocket::read_response(int connfd, net_mod_recv_msg_t *recv_msg) {
|
int recv_size;
|
void *recv_buf;
|
char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH];
|
|
net_mod_response_head_t response_head;
|
net_node_t *node = req_resp_pool.connfdNodeMap.find(connfd)->second;
|
if ( rio_readn(connfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb");
|
|
return -1;
|
}
|
|
response_head = NetModSocket::decode_response_head(response_head_bs);
|
if(response_head.code != 0) {
|
// 对方没有对应的key
|
return 1;
|
}
|
|
recv_buf = malloc(response_head.content_length);
|
if(recv_buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc recv_buf");
|
exit(1);
|
}
|
if ( (recv_size = rio_readn(connfd, recv_buf, response_head.content_length) ) != response_head.content_length) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb");
|
|
return -1;
|
}
|
|
|
strcpy( recv_msg->host, node->host);
|
recv_msg->port = node->port;
|
recv_msg->key = node->key;
|
recv_msg->content = recv_buf;
|
recv_msg->content_length = recv_size;
|
return 0;
|
}
|
|
int NetModSocket::sendandrecv_safe(net_node_t *node_arr, int arrlen, void *send_buf, int send_size,
|
net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) {
|
|
int i, clientfd;
|
net_node_t *node;
|
void *recv_buf;
|
int recv_size;
|
char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH];
|
net_mod_request_head_t request_head = {};
|
net_mod_response_head_t response_head;
|
|
|
char portstr[32];
|
char *buf = NULL;
|
int buf_size, max_buf_size;
|
|
if(buf == NULL) {
|
buf = (char *)malloc(MAXBUF);
|
max_buf_size = MAXBUF;
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc");
|
}
|
|
int nsuc = 0;
|
net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t));
|
|
for (i = 0; i< arrlen; i++) {
|
|
node = &node_arr[i];
|
if(node->host == NULL) {
|
// 本地发送
|
shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size);
|
goto LABEL_ARR_PUSH;
|
}
|
|
sprintf(portstr, "%d", node->port);
|
clientfd = open_clientfd(node->host, portstr);
|
if(clientfd < 0) {
|
continue;
|
}
|
|
buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH;
|
if(max_buf_size < buf_size) {
|
buf = (char *)realloc(buf, buf_size);
|
max_buf_size = buf_size;
|
}
|
|
|
request_head.mod = REQ_REP;
|
request_head.key = node->key;
|
request_head.content_length = send_size;
|
request_head.topic_length = 0;
|
|
// optval = 1;
|
// setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval));
|
memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size);
|
|
|
if(rio_writen(clientfd, buf, buf_size) != buf_size ) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send conent rio_writen");
|
|
close(clientfd);
|
continue;
|
}
|
// optval = 0;
|
// setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval));
|
|
if ( rio_readn(clientfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb");
|
|
close(clientfd);
|
continue;
|
}
|
|
response_head = NetModSocket::decode_response_head(response_head_bs);
|
if(response_head.code != 0) {
|
continue;
|
}
|
|
recv_buf = malloc(response_head.content_length);
|
if(recv_buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc");
|
exit(1);
|
}
|
if ( (recv_size = rio_readn(clientfd, recv_buf, response_head.content_length) ) != response_head.content_length) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb");
|
|
close(clientfd);
|
continue;
|
}
|
|
LABEL_ARR_PUSH:
|
if(node->host != NULL) {
|
strcpy(ret_arr[nsuc].host, node->host);
|
} else {
|
strcpy(ret_arr[nsuc].host, "local");
|
}
|
|
ret_arr[nsuc].port = node->port;
|
ret_arr[nsuc].key = node->key;
|
ret_arr[nsuc].content = recv_buf;
|
ret_arr[nsuc].content_length = recv_size;
|
|
nsuc++;
|
}
|
|
*recv_arr = ret_arr;
|
if(recv_arr_size != NULL) {
|
*recv_arr_size = nsuc;
|
}
|
|
free(buf);
|
return nsuc;
|
|
}
|
|
// int pub(char *topic, int topic_size, void *content, int content_size, int port);
|
|
int NetModSocket::pub(net_node_t *node_arr, int arrlen, char *topic, int topic_size, void *content, int content_size) {
|
int i, clientfd;
|
net_node_t *node;
|
|
char *buf;
|
int max_buf_size, buf_size;
|
|
net_mod_request_head_t request_head;
|
|
char portstr[32];
|
|
buf = (char *)malloc(MAXBUF);
|
max_buf_size = MAXBUF;
|
if(buf == NULL) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc");
|
exit(1);
|
}
|
int nsuc = 0;
|
for (i = 0; i< arrlen; i++) {
|
|
node = &node_arr[i];
|
if(node->host == NULL) {
|
// 本地发送
|
shmModSocket.pub(topic, topic_size, content, content_size, node->key);
|
|
} else {
|
sprintf(portstr, "%d", node->port);
|
clientfd = open_clientfd(node->host, portstr);
|
if(clientfd < 0) {
|
continue;
|
}
|
|
request_head.mod = BUS;
|
request_head.key = node->key;
|
request_head.content_length = content_size;
|
request_head.topic_length = strlen(topic) + 1;
|
|
buf_size = NET_MODE_REQUEST_HEAD_LENGTH + content_size + request_head.topic_length;
|
|
if(max_buf_size < buf_size) {
|
buf = (char *)realloc(buf, buf_size);
|
max_buf_size = buf_size;
|
}
|
|
memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, content, content_size);
|
memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH + content_size, topic, request_head.topic_length);
|
|
if(rio_writen(clientfd, buf, buf_size) != buf_size ) {
|
LoggerFactory::getLogger()->error(errno, "NetModSocket::pub rio_writen conent ");
|
close(clientfd);
|
continue;
|
}
|
}
|
|
nsuc++;
|
}
|
|
free(buf);
|
return nsuc;
|
}
|
|
void NetModSocket::free_recv_msg_arr(net_mod_recv_msg_t * arr, size_t size) {
|
|
for(int i =0; i< size; i++) {
|
free(arr[i].content);
|
}
|
free(arr);
|
}
|
|
// ssize_t recv(void *buf, size_t len) {
|
|
// return rio_readlineb(&rio, buf, MAXLINE);
|
|
// }
|
|
void * NetModSocket::encode_request_head(net_mod_request_head_t & request) {
|
char * head = (char *)malloc(NET_MODE_REQUEST_HEAD_LENGTH);
|
PUT(head, htonl(request.mod));
|
PUT(head + 4, htonl(request.key));
|
PUT(head + 8, htonl(request.content_length));
|
PUT(head + 12, htonl(request.topic_length));
|
return head;
|
}
|
|
net_mod_request_head_t NetModSocket::decode_request_head(void *_headbs) {
|
char *headbs = (char *)_headbs;
|
net_mod_request_head_t head;
|
head.mod = ntohl(GET(headbs));
|
head.key = ntohl(GET(headbs + 4));
|
head.content_length = ntohl(GET(headbs + 8));
|
head.topic_length = ntohl(GET(headbs + 12));
|
return head;
|
}
|
|
void * NetModSocket::encode_response_head(net_mod_response_head_t & response) {
|
char * head = (char *)malloc(NET_MODE_RESPONSE_HEAD_LENGTH);
|
PUT(head, htonl(response.code));
|
PUT(head + 4, htonl(response.content_length));
|
return head;
|
}
|
|
net_mod_response_head_t NetModSocket::decode_response_head(void *_headbs) {
|
char *headbs = (char *)_headbs;
|
net_mod_response_head_t head;
|
head.code = ntohl(GET(headbs));
|
head.content_length = ntohl(GET(headbs + 4));
|
return head;
|
}
|
