#include "shm_socket.h" #include "hashtable.h" #include "logger_factory.h" #include #include #include "bus_error.h" #include "sole.h" static Logger *logger = LoggerFactory::getLogger(); static void print_msg(char *head, shm_packet_t &msg) { // err_msg(0, "%s: key=%d, type=%d\n", head, msg.key, msg.type); } static pthread_once_t _once_ = PTHREAD_ONCE_INIT; static pthread_key_t _perthread_socket_key_; static void _destrory_socket_perthread(void *tmp_socket); static void _create_socket_key_perthread(void); static int shm_recvpakfrom(shm_socket_t *sockt, shm_packet_t *recvpak , const struct timespec *timeout, int flag); static int shm_sendpakto(shm_socket_t *sockt, const shm_packet_t *sendpak, const int key, const struct timespec *timeout, const int flag); // 检查key是否已经被使用, 未被使用则绑定key static LockFreeQueue * shm_socket_bind_queue(int key, bool force) { hashtable_t *hashtable = mm_get_hashtable(); LockFreeQueue *queue; hashtable_lock(hashtable); void *tmp_ptr = hashtable_get(hashtable, key); if (tmp_ptr == NULL || tmp_ptr == (void *)1 ) { queue = new LockFreeQueue(16); hashtable_put(hashtable, key, (void *)queue); hashtable_unlock(hashtable); return queue; } else if(force) { hashtable_unlock(hashtable); return (LockFreeQueue *) queue; } hashtable_unlock(hashtable); return NULL; } /** * 绑定key到队列,但是并不会创建队列。 */ static LockFreeQueue * shm_socket_attach_queue(int key) { LockFreeQueue * queue; hashtable_t *hashtable = mm_get_hashtable(); void *tmp_ptr = hashtable_get(hashtable, key); if (tmp_ptr == NULL || tmp_ptr == (void *)1) { //logger->error("shm_socket._remote_queue_attach:connet at key %d failed!", key); return NULL; } queue = ( LockFreeQueue *)tmp_ptr; // hashtable_unlock(hashtable); return queue; } //删除包含在keys内的queue size_t shm_socket_remove_keys(int keys[], size_t length) { hashtable_t *hashtable = mm_get_hashtable(); LockFreeQueue *mqueue; size_t count = 0; for(int i = 0; i< length; i++) { // 销毁共享内存的queue mqueue = (LockFreeQueue *)hashtable_get(hashtable, keys[i]); delete mqueue; hashtable_remove(hashtable, keys[i]); count++; } return count; } // 删除不在keys内的queue size_t shm_socket_remove_keys_exclude(int keys[], size_t length) { hashtable_t *hashtable = mm_get_hashtable(); std::set *keyset = hashtable_keyset(hashtable); std::set::iterator keyItr; LockFreeQueue *mqueue; bool found; size_t count = 0; for (keyItr = keyset->begin(); keyItr != keyset->end(); keyItr++) { found = false; for (size_t i = 0; i < length; i++) { if (*keyItr == keys[i]) { found = true; break; } } // 100内的是bus内部自己用的 if (!found && *keyItr > 100) { // 销毁共享内存的queue mqueue = (LockFreeQueue *)hashtable_get(hashtable, *keyItr); delete mqueue; hashtable_remove(hashtable, *keyItr); count++; } } delete keyset; return count; } shm_socket_t *shm_open_socket(shm_socket_type_t socket_type) { int s, type; pthread_mutexattr_t mtxAttr; logger->debug("shm_open_socket\n"); // shm_socket_t *socket = (shm_socket_t *)calloc(1, sizeof(shm_socket_t)); shm_socket_t *sockt = new shm_socket_t; sockt->socket_type = socket_type; sockt->key = 0; sockt->force_bind = false; sockt->queue = NULL; s = pthread_mutexattr_init(&mtxAttr); if (s != 0) err_exit(s, "pthread_mutexattr_init"); s = pthread_mutexattr_settype(&mtxAttr, PTHREAD_MUTEX_ERRORCHECK); if (s != 0) err_exit(s, "pthread_mutexattr_settype"); s = pthread_mutex_init(&(sockt->mutex), &mtxAttr); if (s != 0) err_exit(s, "pthread_mutex_init"); s = pthread_mutexattr_destroy(&mtxAttr); if (s != 0) err_exit(s, "pthread_mutexattr_destroy"); return sockt; } int shm_close_socket(shm_socket_t *sockt) { int rv; logger->debug("shm_close_socket\n"); if(sockt->queue != NULL) { delete sockt->queue; sockt->queue = NULL; } rv = pthread_mutex_destroy(&(sockt->mutex) ); if(rv != 0) { err_exit(rv, "shm_close_socket"); } free(sockt); return 0; } int shm_socket_stop(shm_socket_t *sockt) { struct timespec timeout = {5, 0}; shm_packet_t sendpak = {0}; sendpak.key = sockt->key; sendpak.action = BUS_ACTION_STOP; sendpak.size = 0; return shm_sendpakto(sockt, &sendpak, sockt->key, &timeout, BUS_TIMEOUT_FLAG); } int shm_socket_bind(shm_socket_t *sockt, int key) { sockt->key = key; return 0; } int shm_socket_force_bind(shm_socket_t *sockt, int key) { sockt->force_bind = true; sockt->key = key; return 0; } int shm_socket_get_key(shm_socket_t *sockt){ return sockt->key; } // 短连接方式发送 int shm_sendto(shm_socket_t *sockt, const void *buf, const int size, const int key, const struct timespec *timeout, const int flag) { int rv; shm_packet_t sendpak = {0}; sendpak.key = sockt->key; sendpak.size = size; if(buf != NULL) { sendpak.buf = mm_malloc(size); memcpy(sendpak.buf, buf, size); } rv = shm_sendpakto(sockt, &sendpak, key, timeout, flag); return rv; } int shm_sendandrecv(shm_socket_t *sockt, const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int rv, tryn = 3; shm_packet_t sendpak; shm_packet_t recvpak; std::map::iterator recvbufIter; std::string uuid = sole::uuid4().str(); sendpak.key = sockt->key; sendpak.size = send_size; if(send_buf != NULL) { sendpak.buf = mm_malloc(send_size); memcpy(sendpak.buf, send_buf, send_size); } memcpy(sendpak.uuid, uuid.c_str(), uuid.length() + 1); // uuid.copy(sendpak.uuid, sizeof sendpak.uuid); rv = shm_sendpakto(sockt, &sendpak, key, timeout, flags); if(rv != 0) { return rv; } while(tryn > 0) { tryn--; recvbufIter = sockt->recvbuf.find(uuid); if(recvbufIter != sockt->recvbuf.end()) { // 在缓存里查到了UUID匹配成功的 logger->debug("get from recvbuf: %s", uuid.c_str()); recvpak = recvbufIter->second; sockt->recvbuf.erase(recvbufIter); goto LABLE_SUC; } rv = shm_recvpakfrom(sockt, &recvpak, timeout, flags); if (rv != 0) { if(rv == ETIMEDOUT) { return EBUS_TIMEOUT; } logger->debug("%d shm_recvfrom failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } logger->debug("send uuid:%s, recv uuid: %s", uuid.c_str(), recvpak.uuid); if(strlen(recvpak.uuid) == 0) { continue; } else if (strncmp(uuid.c_str(), recvpak.uuid, sizeof recvpak.uuid) == 0) { // 发送与接受的UUID匹配成功 goto LABLE_SUC; } else { // 答非所问,放到缓存里 sockt->recvbuf.insert({recvpak.uuid, recvpak}); continue; } } LABLE_FAIL: return EBUS_RECVFROM_WRONG_END; // return rv; LABLE_SUC: if(recv_buf != NULL) { void *_buf = malloc(recvpak.size); memcpy(_buf, recvpak.buf, recvpak.size); *recv_buf = _buf; } if(recv_size != NULL) *recv_size = recvpak.size; mm_free(recvpak.buf); return 0; } /** * @callback void (*recvandsend_callback_fn)(void *recvbuf, int recvsize, int key, void **sendbuf, int *sendsize) * sendbuf 和 sendsize是callbak_fn回调函数的返回值, 分别表示发送数据,和发送数据的大小。 * */ int shm_recvandsend(shm_socket_t *sockt, recvandsend_callback_fn callback, const struct timespec *timeout, int flag, void *user_data) { int rv; void *sendbuf, *recvbuf = NULL; int sendsize = 0; shm_packet_t recvpak = {0}; rv = shm_recvpakfrom(sockt , &recvpak, timeout, flag); if (rv != 0) { if(rv == ETIMEDOUT){ logger->debug("%d shm_recvandsend failed %s", shm_socket_get_key(sockt), bus_strerror(EBUS_TIMEOUT)); return EBUS_TIMEOUT; } logger->error("%d shm_recvandsend failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } if(recvpak.buf != NULL) { recvbuf = malloc(recvpak.size); memcpy(recvbuf, recvpak.buf, recvpak.size); mm_free(recvpak.buf); } callback(recvbuf, recvpak.size, recvpak.key, &sendbuf, &sendsize, user_data); shm_packet_t sendpak = {0}; sendpak.key = sockt->key; sendpak.size = sendsize; memcpy(sendpak.uuid, recvpak.uuid, sizeof sendpak.uuid); if(sendbuf !=NULL && sendsize > 0) { sendpak.buf = mm_malloc(sendsize); memcpy(sendpak.buf, sendbuf, sendsize); } else { logger->warn("%d shm_recvandsend : sendbuf is null", shm_socket_get_key(sockt)); // return -1; } rv = shm_sendpakto(sockt, &sendpak, recvpak.key, timeout, flag); return 0; } // 短连接方式接受 int shm_recvfrom(shm_socket_t *sockt, void **buf, int *size, int *key, const struct timespec *timeout, int flag) { int rv; shm_packet_t recvpak; rv = shm_recvpakfrom(sockt , &recvpak, timeout, flag); if (rv != 0) { if(rv == ETIMEDOUT) return EBUS_TIMEOUT; else { logger->debug("%d shm_recvfrom failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } } if(buf != NULL && recvpak.buf != NULL) { void *_buf = malloc(recvpak.size); memcpy(_buf, recvpak.buf, recvpak.size); *buf = _buf; } if(size != NULL) *size = recvpak.size; if(key != NULL) *key = recvpak.key; mm_free(recvpak.buf); return 0; } // ================================================================================================= /* Free thread-specific data buffer */ static void _destrory_socket_perthread(void *tmp_socket) { int rv; if(tmp_socket == NULL) return; logger->debug("%d destroy tmp socket\n", pthread_self()); shm_close_socket((shm_socket_t *)tmp_socket); rv = pthread_setspecific(_perthread_socket_key_, NULL); if ( rv != 0) { logger->error(rv, "shm_sendandrecv : pthread_setspecific"); exit(1); } } /* One-time key creation function */ static void _create_socket_key_perthread(void) { int s; /* Allocate a unique thread-specific data key and save the address of the destructor for thread-specific data buffers */ s = pthread_key_create(&_perthread_socket_key_, _destrory_socket_perthread); //s = pthread_key_create(&_perthread_socket_key_, NULL); if (s != 0) { logger->error(s, "pthread_key_create"); exit(1); } } // use thread local int _shm_sendandrecv_thread_local(shm_socket_t *sockt, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int recv_key; int rv; int tryn = 0; // 用thread local 保证每个线程用一个独占的socket接受对方返回的信息 shm_socket_t *tmp_socket; /* If first call from this thread, allocate buffer for thread, and save its location */ // logger->debug("%d create tmp socket\n", pthread_self() ); rv = pthread_once(&_once_, _create_socket_key_perthread); if (rv != 0) { logger->error(rv, "shm_sendandrecv pthread_once"); exit(1); } tmp_socket = (shm_socket_t *)pthread_getspecific(_perthread_socket_key_); if (tmp_socket == NULL) { /* If first call from this thread, allocate buffer for thread, and save its location */ logger->debug("%ld create tmp socket\n", (long)pthread_self() ); tmp_socket = shm_open_socket(SHM_SOCKET_DGRAM); rv = pthread_setspecific(_perthread_socket_key_, tmp_socket); if ( rv != 0) { logger->error(rv, "shm_sendandrecv : pthread_setspecific"); exit(1); } } if ((rv = shm_sendto(tmp_socket, send_buf, send_size, send_key, timeout, flags)) == 0) { while(tryn < 3) { tryn++; rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); if(rv != 0) { logger->error("_shm_sendandrecv_thread_local : %s\n", bus_strerror(rv)); return rv; } // 超时导致接发送对象,与返回对象不对应的情况 if(send_key != recv_key) { logger->debug("======%d use tmp_socket %d, send to %d, receive from %d\n", shm_socket_get_key(sockt), shm_socket_get_key(tmp_socket), send_key, recv_key); // logger->error( "_shm_sendandrecv_alloc_new: send key expect to equal to recv key! send key =%d , recv key=%d", send_key, recv_key); // exit(1); continue; // return EBUS_RECVFROM_WRONG_END; } return 0; } return EBUS_RECVFROM_WRONG_END; } return rv; } int _shm_sendandrecv_alloc_new(shm_socket_t *sockt, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int recv_key; int rv; int tryn = 0; shm_socket_t *tmp_socket; tmp_socket = shm_open_socket(SHM_SOCKET_DGRAM); if ((rv = shm_sendto(tmp_socket, send_buf, send_size, send_key, timeout, flags)) == 0) { rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); while(tryn < 3) { tryn++; rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); if(rv != 0) { logger->error("_shm_sendandrecv_thread_local : %s\n", bus_strerror(rv)); return rv; } // 超时导致接发送对象,与返回对象不对应的情况 if(send_key != recv_key) { // logger->debug("======%d use tmp_socket %d, send to %d, receive from %d\n", shm_socket_get_key(sockt), shm_socket_get_key(tmp_socket), send_key, recv_key); // logger->error( "_shm_sendandrecv_alloc_new: send key expect to equal to recv key! send key =%d , recv key=%d", send_key, recv_key); continue; } return 0; } return EBUS_RECVFROM_WRONG_END; } shm_close_socket(tmp_socket); return rv; } static int shm_sendpakto(shm_socket_t *sockt, const shm_packet_t *sendpak, const int key, const struct timespec *timeout, const int flag) { int rv; hashtable_t *hashtable = mm_get_hashtable(); if( sockt->queue != NULL) goto LABEL_PUSH; if(hashtable_get_queue_count(hashtable) > QUEUE_COUNT_LIMIT) { return EBUS_EXCEED_LIMIT; } { if ((rv = pthread_mutex_lock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_lock"); if (sockt->queue == NULL) { if (sockt->key == 0) { sockt->key = hashtable_alloc_key(hashtable); } sockt->queue = shm_socket_bind_queue( sockt->key, sockt->force_bind); if(sockt->queue == NULL ) { logger->error("%s. key = %d", bus_strerror(EBUS_KEY_INUSED), sockt->key); return EBUS_KEY_INUSED; } } if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_unlock"); } LABEL_PUSH: if (sendpak->action != BUS_ACTION_STOP && key == sockt->key) { logger->error( "can not send to your self!"); return EBUS_SENDTO_SELF; } LockFreeQueue *remoteQueue; if ((remoteQueue = shm_socket_attach_queue(key)) == NULL) { bus_errno = EBUS_CLOSED; logger->error("sendto key %d failed, %s", key, bus_strerror(bus_errno)); return EBUS_CLOSED; } rv = remoteQueue->push(*sendpak, timeout, flag); return rv; } // 短连接方式接受 static int shm_recvpakfrom(shm_socket_t *sockt, shm_packet_t *_recvpak , const struct timespec *timeout, int flag) { int rv; hashtable_t *hashtable = mm_get_hashtable(); shm_packet_t recvpak; if( sockt->queue != NULL) goto LABEL_POP; if(hashtable_get_queue_count(hashtable) > QUEUE_COUNT_LIMIT) { return EBUS_EXCEED_LIMIT; } { if ((rv = pthread_mutex_lock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_lock"); if (sockt->key == 0) { sockt->key = hashtable_alloc_key(hashtable); } sockt->queue = shm_socket_bind_queue( sockt->key, sockt->force_bind); if(sockt->queue == NULL ) { logger->error("%s. key = %d", bus_strerror(EBUS_KEY_INUSED), sockt->key); return EBUS_KEY_INUSED; } if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_unlock"); } LABEL_POP: // // printf("%p start recv.....\n", sockt); rv = sockt->queue->pop(recvpak, timeout, flag); if(rv != 0) return rv; if(recvpak.action == BUS_ACTION_STOP) { return EBUS_STOPED; } *_recvpak = recvpak; return rv; } // int shm_sendandrecv(shm_socket_t *sockt, const void *send_buf, // const int send_size, const int send_key, void **recv_buf, // int *recv_size, const struct timespec *timeout, int flags) { // struct timespec tm = {10, 0}; // return _shm_sendandrecv_thread_local(sockt, send_buf, send_size, send_key,recv_buf, recv_size, &tm, flags); // }