#include "net_mod_socket.h" #include "socket_io.h" #include "net_mod_socket_io.h" #include /* See NOTES */ #include NetModSocket::NetModSocket() { init_req_rep_req_resp_pool(); } NetModSocket::~NetModSocket() { int clientfd; for (auto map_iter = req_resp_pool.connectionMap.begin(); map_iter != req_resp_pool.connectionMap.end(); map_iter++) { clientfd = map_iter->second; Close(clientfd); } } void NetModSocket::init_req_rep_req_resp_pool() { /* Initially, there are no connected descriptors */ int i; req_resp_pool.maxi = -1; //line:conc:echoservers:beginempty for (i = 0; i < FD_SETSIZE; i++) req_resp_pool.connfd[i] = -1; //line:conc:echoservers:endempty /* Initially, listenfd is only member of select read set */ FD_ZERO(&req_resp_pool.read_set); FD_ZERO(&req_resp_pool.write_set); FD_ZERO(&req_resp_pool.except_set); } int NetModSocket::connect( net_node_t *node) { std::map::iterator mapIter; int connfd; int i; char mapKey[256]; char portstr[32]; sprintf(mapKey, "%s:%d", node->host, node->port); if( ( mapIter = req_resp_pool.connectionMap.find(mapKey)) != req_resp_pool.connectionMap.end()) { connfd = mapIter->second; } else { sprintf(portstr, "%d", node->port); connfd = open_clientfd(node->host, portstr); if(connfd < 0) { return -1; } req_resp_pool.connectionMap.insert({mapKey, connfd}); } for (i = 0; i < FD_SETSIZE; i++) { /* Find an available slot */ if (req_resp_pool.connfd[i] < 0) { /* Add connected descriptor to the req_resp_pool */ req_resp_pool.connfd[i] = connfd; req_resp_pool.connfdNodeMap.insert({connfd, node}); // Rio_readinitb(&req_resp_pool.clientrio[i], connfd); //line:conc:echoservers:endaddclient /* Add the descriptor to descriptor set */ FD_SET(connfd, &req_resp_pool.read_set); //line:conc:echoservers:addconnfd FD_SET(connfd, &req_resp_pool.write_set); FD_SET(connfd, &req_resp_pool.except_set); /* Update max descriptor and req_resp_pool highwater mark */ if (connfd > req_resp_pool.maxfd) req_resp_pool.maxfd = connfd; if (i > req_resp_pool.maxi) req_resp_pool.maxi = i; break; } } if (i == FD_SETSIZE) { /* Couldn't find an empty slot */ LoggerFactory::getLogger()->error(errno, "add_client error: Too many clients"); return -1; } return connfd; } void NetModSocket::close_connect(int connfd) { net_node_t *node = req_resp_pool.connfdNodeMap.find(connfd)->second; // std::map::iterator mapIter; if(close(connfd) != 0) { LoggerFactory::getLogger()->error(errno, "NetModSocket::close_connect"); } FD_CLR(connfd, &req_resp_pool.read_set); FD_CLR(connfd, &req_resp_pool.write_set); FD_CLR(connfd, &req_resp_pool.except_set); char mapKey[256]; // char portstr[32]; sprintf(mapKey, "%s:%d", node->host, node->port); req_resp_pool.connectionMap.erase(mapKey); // LoggerFactory::getLogger()->debug("close_connect"); } int NetModSocket::sendandrecv(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) { int i, n, recv_size, connfd; net_node_t *node; void *recv_buf; struct timeval timeout = {5, 0}; int n_conn_suc = 0, n_recv_suc = 0; net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t)); //init_req_rep_req_resp_pool(); for (i = 0; i< arrlen; i++) { node = &node_arr[i]; if(node->host == NULL) { // 本地发送 shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size); strcpy( ret_arr[n_recv_suc].host,"localshm"); ret_arr[n_recv_suc].port = 0; ret_arr[n_recv_suc].key = node->key; ret_arr[n_recv_suc].content = recv_buf; ret_arr[n_recv_suc].content_length = recv_size; n_recv_suc++; continue; } if( (connfd = connect(node)) < 0 ) { continue; } n_conn_suc++; } // printf("n_conn_suc =%d\n", n_conn_suc); while(n_recv_suc < n_conn_suc) { /* Wait for listening/connected descriptor(s) to become ready */ req_resp_pool.ready_read_set = req_resp_pool.read_set; req_resp_pool.ready_write_set = req_resp_pool.write_set; req_resp_pool.ready_except_set = req_resp_pool.except_set; if( (req_resp_pool.nready = select(req_resp_pool.maxfd + 1, &req_resp_pool.ready_read_set, &req_resp_pool.ready_write_set, &req_resp_pool.ready_except_set, &timeout)) <= 0) { // wirite_set 和 read_set 在指定时间内都没准备好 break; } // printf("req_resp_pool.nready =%d\n", req_resp_pool.nready); for (i = 0; (i <= req_resp_pool.maxi) && (req_resp_pool.nready > 0); i++) { if ( (connfd = req_resp_pool.connfd[i]) > 0 ) { /* If the descriptor is ready, echo a text line from it */ node = req_resp_pool.connfdNodeMap.find(connfd)->second; if ( FD_ISSET(connfd, &req_resp_pool.ready_read_set)) { req_resp_pool.nready--; if( (n = read_response(connfd, ret_arr+n_recv_suc)) == 0) { // 成功收到返回消息,清空读入位 FD_CLR(connfd, &req_resp_pool.read_set); req_resp_pool.connfd[i] = -1; n_recv_suc++; } else if(n == -1) { close_connect(connfd); } } if (FD_ISSET(connfd, &req_resp_pool.ready_write_set)) { req_resp_pool.nready--; // printf("write %d\n", connfd); if(write_request(connfd, node->key, send_buf, send_size) != 0) { close_connect(connfd); } else{ // 一次写入完成后清空写入位 FD_CLR(connfd, &req_resp_pool.write_set); } } if (FD_ISSET(connfd, &req_resp_pool.ready_except_set)) { req_resp_pool.nready--; close_connect(connfd); } } } } FD_ZERO(&req_resp_pool.except_set); for (i = 0; i <= req_resp_pool.maxi; i++) { if ( (connfd = req_resp_pool.connfd[i]) > 0 ) { // 关闭并清除写入或读取失败的连接 close_connect(connfd); } } req_resp_pool.maxi = -1; *recv_arr = ret_arr; if(recv_arr_size != NULL) { *recv_arr_size = n_recv_suc; } return n_recv_suc; } int NetModSocket::write_request(int clientfd, int key, void *send_buf, int send_size) { net_mod_request_head_t request_head = {}; int buf_size; char *buf; int max_buf_size; buf = (char *)malloc(MAXBUF); if(buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::NetModSocket malloc"); exit(1); } else { max_buf_size = MAXBUF; } buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH; if(max_buf_size < buf_size) { buf = (char *)realloc(buf, buf_size); max_buf_size = buf_size; if(buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request realloc"); exit(1); } } request_head.mod = REQ_REP; request_head.key = key; request_head.content_length = send_size; request_head.topic_length = 0; // optval = 1; // setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval)); memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size); if(rio_writen(clientfd, buf, buf_size) != buf_size ) { LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request rio_writen"); free(buf); return -1; } free(buf); return 0; } /** * @return 0 成功, 1 对方没有对应的key, -1 网络错误 * */ int NetModSocket::read_response(int connfd, net_mod_recv_msg_t *recv_msg) { int recv_size; void *recv_buf; char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH]; net_mod_response_head_t response_head; net_node_t *node = req_resp_pool.connfdNodeMap.find(connfd)->second; if ( rio_readn(connfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb"); return -1; } response_head = NetModSocket::decode_response_head(response_head_bs); if(response_head.code != 0) { // 对方没有对应的key return 1; } recv_buf = malloc(response_head.content_length); if(recv_buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc recv_buf"); exit(1); } if ( (recv_size = rio_readn(connfd, recv_buf, response_head.content_length) ) != response_head.content_length) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb"); return -1; } strcpy( recv_msg->host, node->host); recv_msg->port = node->port; recv_msg->key = node->key; recv_msg->content = recv_buf; recv_msg->content_length = recv_size; return 0; } int NetModSocket::sendandrecv_safe(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) { int i, clientfd; net_node_t *node; void *recv_buf; int recv_size; char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH]; net_mod_request_head_t request_head = {}; net_mod_response_head_t response_head; char portstr[32]; char *buf = NULL; int buf_size, max_buf_size; if(buf == NULL) { buf = (char *)malloc(MAXBUF); max_buf_size = MAXBUF; LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc"); } int nsuc = 0; net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t)); for (i = 0; i< arrlen; i++) { node = &node_arr[i]; if(node->host == NULL) { // 本地发送 shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size); goto LABEL_ARR_PUSH; } sprintf(portstr, "%d", node->port); clientfd = open_clientfd(node->host, portstr); if(clientfd < 0) { continue; } buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH; if(max_buf_size < buf_size) { buf = (char *)realloc(buf, buf_size); max_buf_size = buf_size; } request_head.mod = REQ_REP; request_head.key = node->key; request_head.content_length = send_size; request_head.topic_length = 0; // optval = 1; // setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval)); memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size); if(rio_writen(clientfd, buf, buf_size) != buf_size ) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send conent rio_writen"); close(clientfd); continue; } // optval = 0; // setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval)); if ( rio_readn(clientfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb"); close(clientfd); continue; } response_head = NetModSocket::decode_response_head(response_head_bs); if(response_head.code != 0) { continue; } recv_buf = malloc(response_head.content_length); if(recv_buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc"); exit(1); } if ( (recv_size = rio_readn(clientfd, recv_buf, response_head.content_length) ) != response_head.content_length) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send rio_readnb"); close(clientfd); continue; } LABEL_ARR_PUSH: if(node->host != NULL) { strcpy(ret_arr[nsuc].host, node->host); } else { strcpy(ret_arr[nsuc].host, "local"); } ret_arr[nsuc].port = node->port; ret_arr[nsuc].key = node->key; ret_arr[nsuc].content = recv_buf; ret_arr[nsuc].content_length = recv_size; nsuc++; } *recv_arr = ret_arr; if(recv_arr_size != NULL) { *recv_arr_size = nsuc; } free(buf); return nsuc; } // int pub(char *topic, int topic_size, void *content, int content_size, int port); int NetModSocket::pub(net_node_t *node_arr, int arrlen, char *topic, int topic_size, void *content, int content_size) { int i, clientfd; net_node_t *node; char *buf; int max_buf_size, buf_size; net_mod_request_head_t request_head; char portstr[32]; buf = (char *)malloc(MAXBUF); max_buf_size = MAXBUF; if(buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc"); exit(1); } int nsuc = 0; for (i = 0; i< arrlen; i++) { node = &node_arr[i]; if(node->host == NULL) { // 本地发送 shmModSocket.pub(topic, topic_size, content, content_size, node->key); } else { sprintf(portstr, "%d", node->port); clientfd = open_clientfd(node->host, portstr); if(clientfd < 0) { continue; } request_head.mod = BUS; request_head.key = node->key; request_head.content_length = content_size; request_head.topic_length = strlen(topic) + 1; buf_size = NET_MODE_REQUEST_HEAD_LENGTH + content_size + request_head.topic_length; if(max_buf_size < buf_size) { buf = (char *)realloc(buf, buf_size); max_buf_size = buf_size; } memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, content, content_size); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH + content_size, topic, request_head.topic_length); if(rio_writen(clientfd, buf, buf_size) != buf_size ) { LoggerFactory::getLogger()->error(errno, "NetModSocket::pub rio_writen conent "); close(clientfd); continue; } } nsuc++; } free(buf); return nsuc; } void NetModSocket::free_recv_msg_arr(net_mod_recv_msg_t * arr, size_t size) { for(int i =0; i< size; i++) { free(arr[i].content); } free(arr); } // ssize_t recv(void *buf, size_t len) { // return rio_readlineb(&rio, buf, MAXLINE); // } void * NetModSocket::encode_request_head(net_mod_request_head_t & request) { char * head = (char *)malloc(NET_MODE_REQUEST_HEAD_LENGTH); PUT(head, htonl(request.mod)); PUT(head + 4, htonl(request.key)); PUT(head + 8, htonl(request.content_length)); PUT(head + 12, htonl(request.topic_length)); return head; } net_mod_request_head_t NetModSocket::decode_request_head(void *_headbs) { char *headbs = (char *)_headbs; net_mod_request_head_t head; head.mod = ntohl(GET(headbs)); head.key = ntohl(GET(headbs + 4)); head.content_length = ntohl(GET(headbs + 8)); head.topic_length = ntohl(GET(headbs + 12)); return head; } void * NetModSocket::encode_response_head(net_mod_response_head_t & response) { char * head = (char *)malloc(NET_MODE_RESPONSE_HEAD_LENGTH); PUT(head, htonl(response.code)); PUT(head + 4, htonl(response.content_length)); return head; } net_mod_response_head_t NetModSocket::decode_response_head(void *_headbs) { char *headbs = (char *)_headbs; net_mod_response_head_t head; head.code = ntohl(GET(headbs)); head.content_length = ntohl(GET(headbs + 4)); return head; }