#include "shm_socket.h" #include "hashtable.h" #include "logger_factory.h" #include #include #include "bus_error.h" static Logger *logger = LoggerFactory::getLogger(); static void print_msg(char *head, shm_packet_t &msg) { // err_msg(0, "%s: key=%d, type=%d\n", head, msg.key, msg.type); } static pthread_once_t _once_ = PTHREAD_ONCE_INIT; static pthread_key_t _perthread_socket_key_; static void _destrory_socket_perthread(void *tmp_socket); static void _create_socket_key_perthread(void); // 检查key是否已经被使用, 未被使用则绑定key static LockFreeQueue * shm_socket_bind_queue(int key, bool force) { hashtable_t *hashtable = mm_get_hashtable(); LockFreeQueue *queue; hashtable_lock(hashtable); void *tmp_ptr = hashtable_get(hashtable, key); if (tmp_ptr == NULL || tmp_ptr == (void *)1 ) { queue = new LockFreeQueue(16); hashtable_put(hashtable, key, (void *)queue); hashtable_unlock(hashtable); return queue; } else if(force) { hashtable_unlock(hashtable); return (LockFreeQueue *) queue; } hashtable_unlock(hashtable); return NULL; } /** * 绑定key到队列,但是并不会创建队列。 */ static LockFreeQueue * shm_socket_attach_queue(int key) { LockFreeQueue * queue; hashtable_t *hashtable = mm_get_hashtable(); void *tmp_ptr = hashtable_get(hashtable, key); if (tmp_ptr == NULL || tmp_ptr == (void *)1) { //logger->error("shm_socket._remote_queue_attach:connet at key %d failed!", key); return NULL; } queue = ( LockFreeQueue *)tmp_ptr; // hashtable_unlock(hashtable); return queue; } size_t shm_socket_remove_keys(int keys[], size_t length) { hashtable_t *hashtable = mm_get_hashtable(); LockFreeQueue *mqueue; size_t count = 0; for(int i = 0; i< length; i++) { // 销毁共享内存的queue mqueue = (LockFreeQueue *)hashtable_get(hashtable, keys[i]); delete mqueue; hashtable_remove(hashtable, keys[i]); count++; } return count; } shm_socket_t *shm_open_socket(shm_socket_type_t socket_type) { int s, type; pthread_mutexattr_t mtxAttr; logger->debug("shm_open_socket\n"); shm_socket_t *socket = (shm_socket_t *)calloc(1, sizeof(shm_socket_t)); socket->socket_type = socket_type; socket->key = 0; socket->force_bind = false; socket->queue = NULL; s = pthread_mutexattr_init(&mtxAttr); if (s != 0) err_exit(s, "pthread_mutexattr_init"); s = pthread_mutexattr_settype(&mtxAttr, PTHREAD_MUTEX_ERRORCHECK); if (s != 0) err_exit(s, "pthread_mutexattr_settype"); s = pthread_mutex_init(&(socket->mutex), &mtxAttr); if (s != 0) err_exit(s, "pthread_mutex_init"); s = pthread_mutexattr_destroy(&mtxAttr); if (s != 0) err_exit(s, "pthread_mutexattr_destroy"); return socket; } int shm_close_socket(shm_socket_t *socket) { int s; logger->debug("shm_close_socket\n"); if(socket->queue != NULL) { delete socket->queue; socket->queue = NULL; } s = pthread_mutex_destroy(&(socket->mutex) ); if(s != 0) { err_exit(s, "shm_close_socket"); } free(socket); return 0; } int shm_socket_bind(shm_socket_t *socket, int key) { socket->key = key; return 0; } int shm_socket_force_bind(shm_socket_t *socket, int key) { socket->force_bind = true; socket->key = key; return 0; } int shm_socket_get_key(shm_socket_t *sk){ return sk->key; } // 短连接方式发送 int shm_sendto(shm_socket_t *sockt, const void *buf, const int size, const int key, const struct timespec *timeout, const int flag) { int rv; hashtable_t *hashtable = mm_get_hashtable(); if( sockt->queue != NULL) goto LABEL_PUSH; { if ((rv = pthread_mutex_lock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_lock"); if (sockt->queue == NULL) { if (sockt->key == 0) { sockt->key = hashtable_alloc_key(hashtable); } sockt->queue = shm_socket_bind_queue( sockt->key, sockt->force_bind); if(sockt->queue == NULL ) { logger->error("%s. key = %d", bus_strerror(EBUS_KEY_INUSED), sockt->key); return EBUS_KEY_INUSED; } } if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_unlock"); } LABEL_PUSH: if (key == sockt->key) { logger->error( "can not send to your self!"); return EBUS_SENDTO_SELF; } LockFreeQueue *remoteQueue; if ((remoteQueue = shm_socket_attach_queue(key)) == NULL) { bus_errno = EBUS_CLOSED; logger->error("sendto key %d failed, %s", key, bus_strerror(bus_errno)); return EBUS_CLOSED; } shm_packet_t dest; dest.key = sockt->key; dest.size = size; dest.buf = mm_malloc(size); memcpy(dest.buf, buf, size); rv = remoteQueue->push(dest, timeout, flag); if (rv == 0) { printf("%d sendto %d suc.\n", shm_socket_get_key(sockt), key); return 0; } else { mm_free(dest.buf); if(rv == ETIMEDOUT) return EBUS_TIMEOUT; else { logger->debug("====%d sendto key %d failed %s", shm_socket_get_key(sockt), key, bus_strerror(rv)); return rv; } } } // 短连接方式接受 int shm_recvfrom(shm_socket_t *sockt, void **buf, int *size, int *key, const struct timespec *timeout, int flag) { int rv; hashtable_t *hashtable = mm_get_hashtable(); if( sockt->queue != NULL) goto LABEL_POP; { if ((rv = pthread_mutex_lock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_lock"); if (sockt->key == 0) { sockt->key = hashtable_alloc_key(hashtable); } sockt->queue = shm_socket_bind_queue( sockt->key, sockt->force_bind); if(sockt->queue == NULL ) { logger->error("%s. key = %d", bus_strerror(EBUS_KEY_INUSED), sockt->key); return EBUS_KEY_INUSED; } if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_unlock"); } LABEL_POP: shm_packet_t src; rv = sockt->queue->pop(src, timeout, flag); if (rv == 0) { if(buf != NULL) { void *_buf = malloc(src.size); memcpy(_buf, src.buf, src.size); *buf = _buf; } if(size != NULL) *size = src.size; if(key != NULL) *key = src.key; mm_free(src.buf); return 0; } else { if(rv == ETIMEDOUT) return EBUS_TIMEOUT; else { logger->debug("%d shm_recvfrom failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } } } /* Free thread-specific data buffer */ static void _destrory_socket_perthread(void *tmp_socket) { int rv; if(tmp_socket == NULL) return; logger->debug("%d destroy tmp socket\n", pthread_self()); shm_close_socket((shm_socket_t *)tmp_socket); rv = pthread_setspecific(_perthread_socket_key_, NULL); if ( rv != 0) { logger->error(rv, "shm_sendandrecv : pthread_setspecific"); exit(1); } } /* One-time key creation function */ static void _create_socket_key_perthread(void) { int s; /* Allocate a unique thread-specific data key and save the address of the destructor for thread-specific data buffers */ s = pthread_key_create(&_perthread_socket_key_, _destrory_socket_perthread); //s = pthread_key_create(&_perthread_socket_key_, NULL); if (s != 0) { logger->error(s, "pthread_key_create"); exit(1); } } // use thread local int _shm_sendandrecv_thread_local(shm_socket_t *sockt, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int recv_key; int rv; int tryn = 0; // 用thread local 保证每个线程用一个独占的socket接受对方返回的信息 shm_socket_t *tmp_socket; /* If first call from this thread, allocate buffer for thread, and save its location */ // logger->debug("%d create tmp socket\n", pthread_self() ); rv = pthread_once(&_once_, _create_socket_key_perthread); if (rv != 0) { logger->error(rv, "shm_sendandrecv pthread_once"); exit(1); } tmp_socket = (shm_socket_t *)pthread_getspecific(_perthread_socket_key_); if (tmp_socket == NULL) { /* If first call from this thread, allocate buffer for thread, and save its location */ logger->debug("%ld create tmp socket\n", (long)pthread_self() ); tmp_socket = shm_open_socket(SHM_SOCKET_DGRAM); rv = pthread_setspecific(_perthread_socket_key_, tmp_socket); if ( rv != 0) { logger->error(rv, "shm_sendandrecv : pthread_setspecific"); exit(1); } } if ((rv = shm_sendto(tmp_socket, send_buf, send_size, send_key, timeout, flags)) == 0) { while(tryn < 3) { tryn++; rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); if(rv != 0) { logger->error("_shm_sendandrecv_thread_local : %s\n", bus_strerror(rv)); return rv; } // 超时导致接发送对象,与返回对象不对应的情况 if(send_key != recv_key) { logger->debug("======%d use tmp_socket %d, send to %d, receive from %d\n", shm_socket_get_key(sockt), shm_socket_get_key(tmp_socket), send_key, recv_key); // logger->error( "_shm_sendandrecv_alloc_new: send key expect to equal to recv key! send key =%d , recv key=%d", send_key, recv_key); // exit(1); continue; // return EBUS_RECVFROM_WRONG_END; } return 0; } return EBUS_RECVFROM_WRONG_END; } return rv; } int _shm_sendandrecv_alloc_new(shm_socket_t *sockt, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int recv_key; int rv; int tryn = 0; shm_socket_t *tmp_socket; tmp_socket = shm_open_socket(SHM_SOCKET_DGRAM); if ((rv = shm_sendto(tmp_socket, send_buf, send_size, send_key, timeout, flags)) == 0) { rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); while(tryn < 3) { tryn++; rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); if(rv != 0) { logger->error("_shm_sendandrecv_thread_local : %s\n", bus_strerror(rv)); return rv; } // 超时导致接发送对象,与返回对象不对应的情况 if(send_key != recv_key) { // logger->debug("======%d use tmp_socket %d, send to %d, receive from %d\n", shm_socket_get_key(sockt), shm_socket_get_key(tmp_socket), send_key, recv_key); // logger->error( "_shm_sendandrecv_alloc_new: send key expect to equal to recv key! send key =%d , recv key=%d", send_key, recv_key); continue; } return 0; } return EBUS_RECVFROM_WRONG_END; } shm_close_socket(tmp_socket); return rv; } int shm_sendandrecv(shm_socket_t *socket, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { struct timespec tm = {10, 0}; return _shm_sendandrecv_thread_local(socket, send_buf, send_size, send_key,recv_buf, recv_size, &tm, flags); }