#include "net_mod_socket.h" #include "socket_io.h" #include "net_mod_socket_io.h" #include /* See NOTES */ #include NetModSocket::NetModSocket() { init_req_rep_req_resp_pool(); if (Signal(SIGPIPE, SIG_IGN) == SIG_ERR) err_msg(errno, "signal"); } NetModSocket::~NetModSocket() { int clientfd; for (auto map_iter = req_resp_pool.connectionMap.begin(); map_iter != req_resp_pool.connectionMap.end(); map_iter++) { clientfd = map_iter->second; Close(clientfd); } } /** * 绑定端口到socket, 如果不绑定则系统自动分配一个 * @return 0 成功, 其他值 失败的错误码 */ int NetModSocket::bind(int key) { return shmModSocket.bind(key); } /** * 强制绑定端口到socket, 适用于程序非正常关闭的情况下,重启程序绑定原来还没释放的key * @return 0 成功, 其他值 失败的错误码 */ int NetModSocket::force_bind( int key) { return shmModSocket.force_bind(key); } int NetModSocket::sendandrecv(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) { _sendandrecv_(node_arr, arrlen, send_buf,send_size, recv_arr, recv_arr_size, -1); } int NetModSocket::sendandrecv_timeout(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size, int timeout) { _sendandrecv_(node_arr, arrlen, send_buf,send_size, recv_arr, recv_arr_size, timeout); } int NetModSocket::sendandrecv_nowait(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) { _sendandrecv_(node_arr, arrlen, send_buf,send_size, recv_arr, recv_arr_size, 0); } int NetModSocket::_sendandrecv_(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size, int timeout = 5 * 1000) { int i, n, recv_size, connfd; net_node_t *node; void *recv_buf; net_mod_request_head_t request_head = {}; int n_req = 0, n_recv_suc = 0, n_resp; net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t)); //init_req_rep_req_resp_pool(); for (i = 0; i< arrlen; i++) { node = &node_arr[i]; if(node->host == NULL) { // 本地发送 shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size); strcpy( ret_arr[n_recv_suc].host,"localshm"); ret_arr[n_recv_suc].port = 0; ret_arr[n_recv_suc].key = node->key; ret_arr[n_recv_suc].content = recv_buf; ret_arr[n_recv_suc].content_length = recv_size; n_recv_suc++; continue; } if( (connfd = connect(node)) < 0 ) { continue; } request_head.mod = REQ_REP; memcpy(request_head.host, node->host, sizeof(request_head.host)); request_head.port = node->port; request_head.key = node->key; request_head.content_length = send_size; // printf("write_request %s:%d\n", request_head.host, request_head.port); if(write_request(connfd, request_head, send_buf, send_size) != 0) { LoggerFactory::getLogger()->error("write_request failture %s:%d\n", node->host, node->port); close_connect(connfd); // req_resp_pool.conns[i].fd = -1; } else { n_req++; } } // printf(" req_resp_pool.maxi = %d\n", req_resp_pool.maxi); // printf(" n_req = %d\n", n_req); // int tmp = 0; while(n_resp < n_req) { // printf(" while %d\n", tmp++); /* Wait for listening/connected descriptor(s) to become ready */ if( (req_resp_pool.nready = poll(req_resp_pool.conns, req_resp_pool.maxi + 1, timeout) ) <= 0) { // wirite_set 和 read_set 在指定时间内都没准备好 break; } // printf("req_resp_pool.nready =%d\n", req_resp_pool.nready); for (i = 0; (i <= req_resp_pool.maxi) && (req_resp_pool.nready > 0); i++) { if ( (connfd = req_resp_pool.conns[i].fd) > 0 ) { /* If the descriptor is ready, echo a text line from it */ if (req_resp_pool.conns[i].revents & POLLIN ) { req_resp_pool.nready--; // printf("POLLIN %d\n", connfd); if( (n = read_response(connfd, ret_arr+n_recv_suc)) == 0) { // 成功收到返回消息,清空读入位 req_resp_pool.conns[i].fd = -1; n_recv_suc++; } else if(n == -1) { req_resp_pool.conns[i].fd = -1; close_connect(connfd); } else { req_resp_pool.conns[i].fd = -1; } n_resp++; // printf("read response %d\n", n); } if (req_resp_pool.conns[i].revents & POLLOUT ) { // printf("poll POLLOUT %d\n", connfd); } if (req_resp_pool.conns[i].revents & (POLLRDHUP | POLLHUP | POLLERR) ) { // printf("poll POLLERR %d\n", connfd); req_resp_pool.nready--; close_connect(connfd); req_resp_pool.conns[i].fd = -1; } } } } for (i = 0; i <= req_resp_pool.maxi; i++) { if ( (connfd = req_resp_pool.conns[i].fd) > 0 ) { // 关闭并清除写入或读取失败的连接 close_connect(connfd); req_resp_pool.conns[i].fd = -1; } } req_resp_pool.maxi = -1; *recv_arr = ret_arr; if(recv_arr_size != NULL) { *recv_arr_size = n_recv_suc; } return n_recv_suc; } int NetModSocket::sendandrecv_safe(net_node_t *node_arr, int arrlen, void *send_buf, int send_size, net_mod_recv_msg_t ** recv_arr, int *recv_arr_size) { int i, clientfd; net_node_t *node; void *recv_buf; int recv_size; char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH]; net_mod_request_head_t request_head = {}; net_mod_response_head_t response_head; char portstr[32]; char *buf = NULL; int buf_size, max_buf_size; if(buf == NULL) { buf = (char *)malloc(MAXBUF); max_buf_size = MAXBUF; LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc"); } int nsuc = 0; net_mod_recv_msg_t *ret_arr = (net_mod_recv_msg_t *)calloc(arrlen, sizeof(net_mod_recv_msg_t)); for (i = 0; i< arrlen; i++) { node = &node_arr[i]; if(node->host == NULL) { // 本地发送 shmModSocket.sendandrecv(send_buf, send_size, node->key, &recv_buf, &recv_size); goto LABEL_ARR_PUSH; } sprintf(portstr, "%d", node->port); clientfd = open_clientfd(node->host, portstr); if(clientfd < 0) { continue; } buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH; if(max_buf_size < buf_size) { if((buf = (char *)realloc(buf, buf_size)) == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe realloc buf"); } else { max_buf_size = buf_size; } } request_head.mod = REQ_REP; request_head.key = node->key; request_head.content_length = send_size; request_head.topic_length = 0; // optval = 1; // setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval)); memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size); if(rio_writen(clientfd, buf, buf_size) != buf_size ) { LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe rio_writen buf"); close(clientfd); continue; } // optval = 0; // setsockopt(clientfd, IPPROTO_TCP, TCP_CORK, &optval, sizeof(optval)); if ( rio_readn(clientfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) { LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe rio_readnb response_head_bs"); close(clientfd); continue; } response_head = NetModSocket::decode_response_head(response_head_bs); if(response_head.code != 0) { continue; } recv_buf = malloc(response_head.content_length); if(recv_buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc"); exit(1); } if ( (recv_size = rio_readn(clientfd, recv_buf, response_head.content_length) ) != response_head.content_length) { LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv_safe rio_readnb recv_buf"); close(clientfd); continue; } LABEL_ARR_PUSH: if(node->host != NULL) { strcpy(ret_arr[nsuc].host, node->host); } else { strcpy(ret_arr[nsuc].host, "local"); } ret_arr[nsuc].port = node->port; ret_arr[nsuc].key = node->key; ret_arr[nsuc].content = recv_buf; ret_arr[nsuc].content_length = recv_size; nsuc++; } *recv_arr = ret_arr; if(recv_arr_size != NULL) { *recv_arr_size = nsuc; } free(buf); return nsuc; } // int pub(char *topic, int topic_size, void *content, int content_size, int port); int NetModSocket::pub(net_node_t *node_arr, int arrlen, char *topic, int topic_size, void *content, int content_size) { int i, clientfd; net_node_t *node; char *buf; int max_buf_size, buf_size; net_mod_request_head_t request_head; char portstr[32]; int nsuc = 0; if((buf = (char *)malloc(MAXBUF)) == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::sendandrecv malloc"); exit(1); } else { max_buf_size = MAXBUF; } for (i = 0; i< arrlen; i++) { node = &node_arr[i]; if(node->host == NULL) { // 本地发送 if(shmModSocket.pub(topic, topic_size, content, content_size, node->key) == 0 ) { nsuc++; } } else { sprintf(portstr, "%d", node->port); clientfd = open_clientfd(node->host, portstr); if(clientfd < 0) { continue; } request_head.mod = BUS; request_head.key = node->key; request_head.content_length = content_size; request_head.topic_length = strlen(topic) + 1; buf_size = NET_MODE_REQUEST_HEAD_LENGTH + content_size + request_head.topic_length; if(max_buf_size < buf_size) { if( ( buf = (char *)realloc(buf, buf_size)) == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::pub realloc buf "); } else { max_buf_size = buf_size; } } memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, content, content_size); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH + content_size, topic, request_head.topic_length); if(rio_writen(clientfd, buf, buf_size) != buf_size ) { LoggerFactory::getLogger()->error(errno, "NetModSocket::pub rio_writen conent "); } else { nsuc++; } close(clientfd); } } free(buf); return nsuc; } /** * 发送信息 * @key 发送给谁 * @return 0 成功, 其他值 失败的错误码 */ int NetModSocket::sendto(const void *buf, const int size, const int key){ return shmModSocket.sendto(buf, size, key); } // 发送信息超时返回。 @sec 秒 , @nsec 纳秒 int NetModSocket::sendto_timeout(const void *buf, const int size, const int key, int sec, int nsec){ struct timespec timeout = {sec, nsec}; return shmModSocket.sendto_timeout(buf, size, key, &timeout); } // 发送信息立刻返回。 int NetModSocket::sendto_nowait(const void *buf, const int size, const int key){ return shmModSocket.sendto_nowait(buf, size, key); } /** * 接收信息 * @key 从谁哪里收到的信息 * @return 0 成功, 其他值 失败的错误码 */ int NetModSocket::recvfrom(void **buf, int *size, int *key) { return shmModSocket.recvfrom(buf, size, key); } // 接受信息超时返回。 @sec 秒 , @nsec 纳秒 int NetModSocket::recvfrom_timeout(void **buf, int *size, int *key, int sec, int nsec){ struct timespec timeout = {sec, nsec}; return shmModSocket.recvfrom_timeout(buf, size, key, &timeout); } int NetModSocket::recvfrom_nowait(void **buf, int *size, int *key){ return shmModSocket.recvfrom_nowait(buf, size, key); } /** * 发送请求信息并等待接收应答 * @key 发送给谁 * @return 0 成功, 其他值 失败的错误码 */ int NetModSocket::sendandrecv( const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size){ return shmModSocket.sendandrecv(send_buf, send_size, key, recv_buf, recv_size); } // 超时返回。 @sec 秒 , @nsec 纳秒 int NetModSocket::sendandrecv_timeout( const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size, int sec, int nsec){ struct timespec timeout = {sec, nsec}; return shmModSocket.sendandrecv_timeout(send_buf, send_size, key, recv_buf, recv_size, &timeout); } int NetModSocket::sendandrecv_nowait( const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size) { return shmModSocket.sendandrecv_nowait(send_buf, send_size, key, recv_buf, recv_size); } /** * 启动bus * * @return 0 成功, 其他值 失败的错误码 */ int NetModSocket::start_bus() { return shmModSocket.start_bus(); } /** * 订阅指定主题 * @topic 主题 * @size 主题长度 * @key 总线端口 */ int NetModSocket::sub( void *topic, int size, int key){ return shmModSocket.sub((char *)topic, size, key); } // 超时返回。 @sec 秒 , @nsec 纳秒 int NetModSocket::sub_timeout( void *topic, int size, int key, int sec, int nsec){ struct timespec timeout = {sec, nsec}; return shmModSocket.sub_timeout((char *)topic, size, key, &timeout); } int NetModSocket::sub_nowait( void *topic, int size, int key){ return shmModSocket.sub_nowait((char *)topic, size, key); } /** * 取消订阅指定主题 * @topic 主题 * @size 主题长度 * @key 总线端口 */ int NetModSocket::desub( void *topic, int size, int key){ return shmModSocket.desub((char *)topic, size, key); } // 超时返回。 @sec 秒 , @nsec 纳秒 int NetModSocket::desub_timeout( void *topic, int size, int key, int sec, int nsec){ struct timespec timeout = {sec, nsec}; return shmModSocket.desub_timeout((char *)topic, size, key, &timeout); } int NetModSocket::desub_nowait( void *topic, int size, int key){ return shmModSocket.desub_nowait((char *)topic, size, key); } /** * 发布主题 * @topic 主题 * @content 主题内容 * @key 总线端口 */ int NetModSocket::pub( char *topic, int topic_size, void *content, int content_size, int key){ return shmModSocket.pub(topic, topic_size, content, content_size, key); } // 超时返回。 @sec 秒 , @nsec 纳秒 int NetModSocket::pub_timeout( char *topic, int topic_size, void *content, int content_size, int key, int sec, int nsec){ struct timespec timeout = {sec, nsec}; return shmModSocket.pub_timeout(topic, topic_size, content, content_size, key, &timeout); } int NetModSocket::pub_nowait( char *topic, int topic_size, void *content, int content_size, int key){ return shmModSocket.pub_nowait(topic, topic_size, content, content_size, key); } /** * 获取soket端口号 */ int NetModSocket::get_key() { return shmModSocket.get_key(); } void NetModSocket::free_recv_msg_arr(net_mod_recv_msg_t * arr, size_t size) { for(int i =0; i< size; i++) { free(arr[i].content); } free(arr); } //====================================================================================== int NetModSocket::write_request(int clientfd, net_mod_request_head_t &request_head, void *send_buf, int send_size) { int buf_size; char *buf; int max_buf_size; if((buf = (char *)malloc(MAXBUF)) == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::NetModSocket malloc"); exit(1); } else { max_buf_size = MAXBUF; } buf_size = send_size + NET_MODE_REQUEST_HEAD_LENGTH; if(max_buf_size < buf_size) { if((buf = (char *)realloc(buf, buf_size)) == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request realloc buf"); exit(1); } else { max_buf_size = buf_size; } } memcpy(buf, NetModSocket::encode_request_head(request_head), NET_MODE_REQUEST_HEAD_LENGTH); memcpy(buf + NET_MODE_REQUEST_HEAD_LENGTH, send_buf, send_size); if(rio_writen(clientfd, buf, buf_size) != buf_size ) { LoggerFactory::getLogger()->error(errno, "NetModSocket::write_request rio_writen"); free(buf); return -1; } free(buf); return 0; } /** * @return 0 成功, 1 对方没有对应的key, -1 网络错误 * */ int NetModSocket::read_response(int connfd, net_mod_recv_msg_t *recv_msg) { int recv_size; void *recv_buf; char response_head_bs[NET_MODE_RESPONSE_HEAD_LENGTH]; net_mod_response_head_t response_head; if ( rio_readn(connfd, response_head_bs, NET_MODE_RESPONSE_HEAD_LENGTH) != NET_MODE_RESPONSE_HEAD_LENGTH) { LoggerFactory::getLogger()->error(errno, "NetModSocket::read_response rio_readnb response_head"); return -1; } response_head = NetModSocket::decode_response_head(response_head_bs); // printf(">>>> read_response %s\n", response_head.host); if(response_head.code != 0) { // 对方没有对应的key return 1; } recv_buf = malloc(response_head.content_length); if(recv_buf == NULL) { LoggerFactory::getLogger()->error(errno, "NetModSocket::send malloc recv_buf"); exit(1); } if ( (recv_size = rio_readn(connfd, recv_buf, response_head.content_length) ) != response_head.content_length) { LoggerFactory::getLogger()->error(errno, "NetModSocket::read_response rio_readnb recv_buf"); return -1; } strcpy( recv_msg->host, response_head.host); recv_msg->port = response_head.port; recv_msg->key = response_head.key; recv_msg->content = recv_buf; recv_msg->content_length = recv_size; return 0; } void NetModSocket::init_req_rep_req_resp_pool() { /* Initially, there are no connected descriptors */ int i; req_resp_pool.maxi = -1; //line:conc:echoservers:beginempty for (i = 0; i < OPEN_MAX; i++) { req_resp_pool.conns[i].fd = -1; req_resp_pool.conns[i].events = 0; } } int NetModSocket::connect( net_node_t *node) { std::map::iterator mapIter; int connfd; int i; char mapKey[256]; char portstr[32]; sprintf(mapKey, "%s:%d", node->host, node->port); mapIter = req_resp_pool.connectionMap.find(mapKey); if( mapIter != req_resp_pool.connectionMap.end()) { connfd = mapIter->second; // printf("hit: %s\n", mapKey); } else { // printf("mis: %s\n", mapKey); sprintf(portstr, "%d", node->port); // printf("open before: %s\n", mapKey); connfd = open_clientfd(node->host, portstr); // printf("open after: %s\n", mapKey); if(connfd < 0) { LoggerFactory::getLogger()->error(errno, "connect %s:%d ", node->host, node->port); return -1; } req_resp_pool.connectionMap.insert({mapKey, connfd}); } for (i = 0; i < OPEN_MAX; i++) { /* Find an available slot */ if (req_resp_pool.conns[i].fd < 0) { /* Add connected descriptor to the req_resp_pool */ req_resp_pool.conns[i].fd = connfd; req_resp_pool.conns[i].events = POLLIN; /* Add the descriptor to descriptor set */ break; } } if (i > req_resp_pool.maxi) req_resp_pool.maxi = i; if (i == OPEN_MAX) { /* Couldn't find an empty slot */ LoggerFactory::getLogger()->error(errno, "add_client error: Too many clients"); return -1; } return connfd; } void NetModSocket::close_connect(int connfd) { int i; char mapKey[256]; std::map::iterator map_iter; if(close(connfd) != 0) { LoggerFactory::getLogger()->error(errno, "NetModSocket::close_connect close"); } for (i = 0; i <= req_resp_pool.maxi; i++) { if(req_resp_pool.conns[i].fd == connfd) { req_resp_pool.conns[i].fd = -1; } } for ( map_iter = req_resp_pool.connectionMap.begin(); map_iter != req_resp_pool.connectionMap.end(); ) { if(connfd == map_iter->second) { // std::cout << "map_iter->first==" << map_iter->first << std::endl; map_iter = req_resp_pool.connectionMap.erase(map_iter); } else { ++map_iter; } } LoggerFactory::getLogger()->debug( "closed %d\n", connfd); } /** uint32_t mod; char host[NI_MAXHOST]; uint32_t port; uint32_t key; uint32_t content_length; uint32_t topic_length; */ void * NetModSocket::encode_request_head(net_mod_request_head_t & head) { void * headbs = malloc(NET_MODE_REQUEST_HEAD_LENGTH); char *tmp_ptr = (char *)headbs; PUT(tmp_ptr, htonl(head.mod)); tmp_ptr += 4; memcpy(tmp_ptr, head.host, sizeof(head.host)); tmp_ptr += sizeof(head.host); PUT(tmp_ptr, htonl(head.port)); tmp_ptr += 4; PUT(tmp_ptr, htonl(head.key)); tmp_ptr += 4; PUT(tmp_ptr, htonl(head.content_length)); tmp_ptr += 4; PUT(tmp_ptr, htonl(head.topic_length)); return headbs; } net_mod_request_head_t NetModSocket::decode_request_head(void *headbs) { char *tmp_ptr = (char *)headbs; net_mod_request_head_t head; head.mod = ntohl(GET(tmp_ptr)); tmp_ptr += 4; memcpy(head.host, tmp_ptr, sizeof(head.host)); tmp_ptr += sizeof(head.host); head.port = ntohl(GET(tmp_ptr)); tmp_ptr += 4; head.key = ntohl(GET(tmp_ptr)); tmp_ptr += 4; head.content_length = ntohl(GET(tmp_ptr)); tmp_ptr += 4; head.topic_length = ntohl(GET(tmp_ptr)); return head; } /** char host[NI_MAXHOST]; uint32_t port; uint32_t key; uint32_t content_length; uint32_t code; */ void * NetModSocket::encode_response_head(net_mod_response_head_t & response) { void * headbs = malloc(NET_MODE_RESPONSE_HEAD_LENGTH); char *tmp_ptr = (char *)headbs; memcpy(tmp_ptr, response.host, NI_MAXHOST); tmp_ptr += NI_MAXHOST; PUT(tmp_ptr, htonl(response.port)); tmp_ptr += 4; PUT(tmp_ptr , htonl(response.key)); tmp_ptr += 4; PUT(tmp_ptr, htonl(response.content_length)); tmp_ptr += 4; PUT(tmp_ptr, htonl(response.code)); return headbs; } net_mod_response_head_t NetModSocket::decode_response_head(void *headbs) { char *tmp_ptr = (char *)headbs; net_mod_response_head_t head; memcpy(head.host, tmp_ptr, NI_MAXHOST); tmp_ptr += NI_MAXHOST; head.port = ntohl(GET(tmp_ptr)); tmp_ptr += 4; head.key = ntohl(GET(tmp_ptr)); tmp_ptr += 4; head.content_length = ntohl(GET(tmp_ptr)); tmp_ptr += 4; head.code = ntohl(GET(tmp_ptr)); return head; }