#include "shm_socket.h" #include "hashtable.h" #include "logger_factory.h" #include #include #include "bus_error.h" #include "sole.h" #include "shm_mm.h" #include "key_def.h" static Logger *logger = LoggerFactory::getLogger(); ShmQueueStMap * shmQueueStMap ; static void print_msg(char *head, shm_packet_t &msg) { // err_msg(0, "%s: key=%d, type=%d\n", head, msg.key, msg.type); } static pthread_once_t _once_ = PTHREAD_ONCE_INIT; static pthread_key_t _localthread_socket_key_; static void _destrory_threadlocal_socket_(void *tmp_socket); static void _create_threadlocal_socket_key_(void); static int shm_recvpakfrom(shm_socket_t *sockt, shm_packet_t *recvpak , const struct timespec *timeout, int flag); static int shm_sendpakto(shm_socket_t *sockt, shm_packet_t *sendpak, const int key, const struct timespec *timeout, const int flag); static int _shm_sendandrecv_uuid(shm_socket_t *sockt, const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags); static int _shm_sendandrecv_thread_local(shm_socket_t *sockt, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) ; // 检查key是否已经被使用, 未被使用则绑定key static LockFreeQueue * shm_socket_bind_queue(int key, bool force) { hashtable_t *hashtable = mm_get_hashtable(); LockFreeQueue *queue; void *tmp_ptr = hashtable_get(hashtable, key); if (tmp_ptr == NULL || tmp_ptr == (void *)1 ) { queue = new LockFreeQueue(32); hashtable_put(hashtable, key, (void *)queue); return queue; } else if(force) { return (LockFreeQueue *) tmp_ptr; } return NULL; } /** * 绑定key到队列,但是并不会创建队列。 */ static LockFreeQueue * shm_socket_attach_queue(int key) { LockFreeQueue * queue; hashtable_t *hashtable = mm_get_hashtable(); void *tmp_ptr = hashtable_get(hashtable, key); if (tmp_ptr == NULL || tmp_ptr == (void *)1) { return NULL; } queue = ( LockFreeQueue *)tmp_ptr; // hashtable_unlock(hashtable); return queue; } shm_socket_t *shm_socket_open(shm_socket_type_t socket_type) { int s, type; pthread_mutexattr_t mtxAttr; logger->debug("shm_socket_open\n"); // shm_socket_t *socket = (shm_socket_t *)calloc(1, sizeof(shm_socket_t)); shm_socket_t *sockt = new shm_socket_t; sockt->socket_type = socket_type; sockt->key = 0; sockt->force_bind = false; sockt->queue = NULL; s = pthread_mutexattr_init(&mtxAttr); if (s != 0) err_exit(s, "pthread_mutexattr_init"); s = pthread_mutexattr_settype(&mtxAttr, PTHREAD_MUTEX_ERRORCHECK); if (s != 0) err_exit(s, "pthread_mutexattr_settype"); s = pthread_mutex_init(&(sockt->mutex), &mtxAttr); if (s != 0) err_exit(s, "pthread_mutex_init"); s = pthread_mutexattr_destroy(&mtxAttr); if (s != 0) err_exit(s, "pthread_mutexattr_destroy"); shmQueueStMap = shm_mm_attach(SHM_QUEUE_ST_KEY); return sockt; } static int _shm_socket_close_(shm_socket_t *sockt) { int rv, i; hashtable_t *hashtable = mm_get_hashtable(); logger->debug("shm_socket_close\n"); // if(sockt->key != 0) { // auto it = shmQueueStMap->find(sockt->key); // if(it != shmQueueStMap->end()) { // it->second.status = SHM_QUEUE_ST_CLOSED; // it->second.closeTime = time(NULL); // } // } if(sockt->queue != NULL) { sockt->queue->close(); for( i = 0; i < sockt->queue->size(); i++) { mm_free((*(sockt->queue))[i].buf); logger->info("======= %d free queue element buf\n", sockt->key); } sleep(1); hashtable_remove(hashtable, sockt->key); // sockt->queue = NULL; } pthread_mutex_destroy(&(sockt->mutex) ); free(sockt); return 0; } int shm_socket_close(shm_socket_t *sockt) { shm_socket_t * threadlocal_socket = (shm_socket_t *)pthread_getspecific(_localthread_socket_key_); if(threadlocal_socket != NULL) { _destrory_threadlocal_socket_(threadlocal_socket); } return _shm_socket_close_(sockt); } int shm_socket_stop(shm_socket_t *sockt) { struct timespec timeout = {5, 0}; shm_packet_t sendpak = {0}; sendpak.key = sockt->key; sendpak.action = BUS_ACTION_STOP; sendpak.size = 0; return shm_sendpakto(sockt, &sendpak, sockt->key, &timeout, BUS_TIMEOUT_FLAG); } int shm_socket_bind(shm_socket_t *sockt, int key) { sockt->key = key; return 0; } int shm_socket_force_bind(shm_socket_t *sockt, int key) { sockt->force_bind = true; sockt->key = key; return 0; } int shm_socket_get_key(shm_socket_t *sockt){ return sockt->key; } // 短连接方式发送 int shm_sendto(shm_socket_t *sockt, const void *buf, const int size, const int key, const struct timespec *timeout, const int flag) { int rv; shm_packet_t sendpak = {0}; sendpak.key = sockt->key; sendpak.size = size; if(buf != NULL) { sendpak.buf = mm_malloc(size); memcpy(sendpak.buf, buf, size); } rv = shm_sendpakto(sockt, &sendpak, key, timeout, flag); return rv; } int shm_sendandrecv(shm_socket_t *sockt, const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { // return _shm_sendandrecv_uuid(sockt, send_buf, send_size, key, recv_buf, recv_size, timeout, flags); return _shm_sendandrecv_thread_local(sockt, send_buf, send_size, key, recv_buf, recv_size, timeout, flags); } /** * @callback void (*recvandsend_callback_fn)(void *recvbuf, int recvsize, int key, void **sendbuf, int *sendsize) * sendbuf 和 sendsize是callbak_fn回调函数的返回值, 分别表示发送数据,和发送数据的大小。 * */ int shm_recvandsend(shm_socket_t *sockt, recvandsend_callback_fn callback, const struct timespec *timeout, int flag, void *user_data) { int rv; void *sendbuf, *recvbuf = NULL; int sendsize = 0; shm_packet_t recvpak = {0}; rv = shm_recvpakfrom(sockt , &recvpak, timeout, flag); if (rv != 0) { if(rv == ETIMEDOUT){ logger->debug("%d shm_recvandsend failed %s", shm_socket_get_key(sockt), bus_strerror(EBUS_TIMEOUT)); return EBUS_TIMEOUT; } logger->error("%d shm_recvandsend failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } if(recvpak.buf != NULL) { recvbuf = malloc(recvpak.size); memcpy(recvbuf, recvpak.buf, recvpak.size); mm_free(recvpak.buf); } callback(recvbuf, recvpak.size, recvpak.key, &sendbuf, &sendsize, user_data); shm_packet_t sendpak = {0}; sendpak.key = sockt->key; sendpak.size = sendsize; memcpy(sendpak.uuid, recvpak.uuid, sizeof(sendpak.uuid)); if(sendbuf !=NULL && sendsize > 0) { sendpak.buf = mm_malloc(sendsize); memcpy(sendpak.buf, sendbuf, sendsize); } else { logger->warn("%d shm_recvandsend : sendbuf is null", shm_socket_get_key(sockt)); // return -1; } rv = shm_sendpakto(sockt, &sendpak, recvpak.key, timeout, flag); return 0; } // 短连接方式接受 int shm_recvfrom(shm_socket_t *sockt, void **buf, int *size, int *key, const struct timespec *timeout, int flag) { int rv; shm_packet_t recvpak; rv = shm_recvpakfrom(sockt , &recvpak, timeout, flag); if (rv != 0) { logger->debug("%d shm_recvfrom failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } if(recvpak.buf != NULL) { if (buf == NULL) { logger->warn("!!!Alert: buf should be not NULL!\n"); } else { void *_buf = malloc(recvpak.size); memcpy(_buf, recvpak.buf, recvpak.size); *buf = _buf; } } if(size != NULL) *size = recvpak.size; if(key != NULL) *key = recvpak.key; if(recvpak.key == 0) { err_exit(0, "key = %d, pid= %d, recvpak.key == 0", shm_socket_get_key(sockt), getpid()); } mm_free(recvpak.buf); return 0; } // ================================================================================================= /* Free thread-specific data buffer */ static void _destrory_threadlocal_socket_(void *tmp_socket) { int rv; logger->debug("%lu destroy threadlocal socket\n", pthread_self()); if(tmp_socket == NULL) return; _shm_socket_close_((shm_socket_t *)tmp_socket); rv = pthread_setspecific(_localthread_socket_key_, NULL); if ( rv != 0) { logger->error(rv, "shm_sendandrecv : pthread_setspecific"); exit(1); } } /* One-time key creation function */ static void _create_threadlocal_socket_key_(void) { int s; /* Allocate a unique thread-specific data key and save the address of the destructor for thread-specific data buffers */ s = pthread_key_create(&_localthread_socket_key_, _destrory_threadlocal_socket_); //s = pthread_key_create(&_localthread_socket_key_, NULL); if (s != 0) { logger->error(s, "pthread_key_create"); exit(1); } } static int _shm_sendandrecv_uuid(shm_socket_t *sockt, const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int rv, tryn = 16; shm_packet_t sendpak; shm_packet_t recvpak; std::map::iterator recvbufIter; std::string uuid = sole::uuid4().str(); sendpak.key = sockt->key; sendpak.size = send_size; if(send_buf != NULL) { sendpak.buf = mm_malloc(send_size); memcpy(sendpak.buf, send_buf, send_size); } memcpy(sendpak.uuid, uuid.c_str(), uuid.length() + 1); // uuid.copy(sendpak.uuid, sizeof sendpak.uuid); rv = shm_sendpakto(sockt, &sendpak, key, timeout, flags); if(rv != 0) { return rv; } while(tryn > 0) { tryn--; recvbufIter = sockt->recvbuf.find(uuid); if(recvbufIter != sockt->recvbuf.end()) { // 在缓存里查到了UUID匹配成功的 logger->debug("get from recvbuf: %s", uuid.c_str()); recvpak = recvbufIter->second; sockt->recvbuf.erase(recvbufIter); goto LABLE_SUC; } rv = shm_recvpakfrom(sockt, &recvpak, timeout, flags); if (rv != 0) { if(rv == ETIMEDOUT) { return EBUS_TIMEOUT; } logger->debug("%d shm_recvfrom failed %s", shm_socket_get_key(sockt), bus_strerror(rv)); return rv; } logger->debug("send uuid:%s, recv uuid: %s", uuid.c_str(), recvpak.uuid); if(strlen(recvpak.uuid) == 0) { continue; } else if (strncmp(uuid.c_str(), recvpak.uuid, sizeof(recvpak.uuid)) == 0) { // 发送与接受的UUID匹配成功 goto LABLE_SUC; } else { // 答非所问,放到缓存里 sockt->recvbuf.insert({recvpak.uuid, recvpak}); continue; } } LABLE_FAIL: return EBUS_RECVFROM_WRONG_END; LABLE_SUC: if(recv_buf != NULL) { void *_buf = malloc(recvpak.size); memcpy(_buf, recvpak.buf, recvpak.size); *recv_buf = _buf; mm_free(recvpak.buf); } if(recv_size != NULL) *recv_size = recvpak.size; return 0; } // use thread local static int _shm_sendandrecv_thread_local(shm_socket_t *sockt, const void *send_buf, const int send_size, const int key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int rv = 0, tryn = 16; static int Counter_suc = 0; static int Counter_fail = 0; shm_packet_t sendpak; shm_packet_t recvpak; std::map::iterator recvbufIter; // 用thread local 保证每个线程用一个独占的socket接受对方返回的信息 shm_socket_t *tmp_socket = NULL; rv = pthread_once(&_once_, _create_threadlocal_socket_key_); if (rv != 0) { logger->error(rv, "shm_sendandrecv pthread_once"); exit(1); } tmp_socket = (shm_socket_t *)pthread_getspecific(_localthread_socket_key_); if (tmp_socket == NULL) { /* If first call from this thread, allocate buffer for thread, and save its location */ tmp_socket = shm_socket_open(SHM_SOCKET_DGRAM); rv = pthread_setspecific(_localthread_socket_key_, tmp_socket); if ( rv != 0) { logger->error(rv, "shm_sendandrecv : pthread_setspecific"); exit(1); } } sendpak.key = tmp_socket->key; sendpak.size = send_size; if(send_buf != NULL) { sendpak.buf = mm_malloc(send_size); memcpy(sendpak.buf, send_buf, send_size); } rv = shm_sendpakto(tmp_socket, &sendpak, key, timeout, flags); if(rv != 0) { return rv; } while(tryn > 0) { tryn--; recvbufIter = tmp_socket->recvbuf2.find(key); if(recvbufIter != tmp_socket->recvbuf2.end()) { // 在缓存里查到了key匹配成功的 recvpak = recvbufIter->second; tmp_socket->recvbuf2.erase(recvbufIter); goto LABLE_SUC; } rv = shm_recvpakfrom(tmp_socket, &recvpak, timeout, flags); if (rv != 0) { logger->debug("%d shm_recvfrom failed %s", shm_socket_get_key(tmp_socket), bus_strerror(rv)); return rv; } if (key == recvpak.key) { // 发送与接受的UUID匹配成功 goto LABLE_SUC; } else { // 答非所问,放到缓存里 tmp_socket->recvbuf2.insert({recvpak.key, recvpak}); continue; } } LABLE_FAIL: return EBUS_RECVFROM_WRONG_END; LABLE_SUC: if(recv_buf != NULL) { void *_buf = malloc(recvpak.size); memcpy(_buf, recvpak.buf, recvpak.size); *recv_buf = _buf; mm_free(recvpak.buf); } if(recv_size != NULL) *recv_size = recvpak.size; return 0; } static int _shm_sendandrecv_alloc_new(shm_socket_t *sockt, const void *send_buf, const int send_size, const int send_key, void **recv_buf, int *recv_size, const struct timespec *timeout, int flags) { int recv_key; int rv; int tryn = 0; shm_socket_t *tmp_socket; tmp_socket = shm_socket_open(SHM_SOCKET_DGRAM); if ((rv = shm_sendto(tmp_socket, send_buf, send_size, send_key, timeout, flags)) == 0) { rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); while(tryn < 3) { tryn++; rv = shm_recvfrom(tmp_socket, recv_buf, recv_size, &recv_key, timeout, flags); if(rv != 0) { logger->error("_shm_sendandrecv_alloc_new : %s\n", bus_strerror(rv)); return rv; } // 超时导致接发送对象,与返回对象不对应的情况 if(send_key != recv_key) { // logger->debug("======%d use tmp_socket %d, send to %d, receive from %d\n", shm_socket_get_key(sockt), shm_socket_get_key(tmp_socket), send_key, recv_key); // logger->error( "_shm_sendandrecv_alloc_new: send key expect to equal to recv key! send key =%d , recv key=%d", send_key, recv_key); continue; } return 0; } return EBUS_RECVFROM_WRONG_END; } _shm_socket_close_(tmp_socket); return rv; } static int shm_sendpakto(shm_socket_t *sockt, shm_packet_t *sendpak, const int key, const struct timespec *timeout, const int flag) { int rv; shm_queue_status_t stRecord; LockFreeQueue *remoteQueue; hashtable_t *hashtable = mm_get_hashtable(); if( sockt->queue != NULL) goto LABEL_PUSH; // if(hashtable_get_queue_count(hashtable) > QUEUE_COUNT_LIMIT) { // return EBUS_EXCEED_LIMIT; // } { if ((rv = pthread_mutex_lock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_lock"); if (sockt->queue == NULL) { if (sockt->key == 0) { sockt->key = hashtable_alloc_key(hashtable); } sockt->queue = shm_socket_bind_queue( sockt->key, sockt->force_bind); if(sockt->queue == NULL ) { logger->error("%s. key = %d", bus_strerror(EBUS_KEY_INUSED), sockt->key); if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_unlock"); return EBUS_KEY_INUSED; } // 标记key对应的状态 ,为opened stRecord.status = SHM_QUEUE_ST_OPENED; stRecord.createTime = time(NULL); shmQueueStMap->insert({sockt->key, stRecord}); } if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_sendto : pthread_mutex_unlock"); } LABEL_PUSH: if (sendpak->action != BUS_ACTION_STOP && key == sockt->key) { logger->error( "can not send to your self!"); return EBUS_SENDTO_SELF; } // 检查key标记的状态 auto it = shmQueueStMap->find(key); if(it != shmQueueStMap->end()) { if(it->second.status == SHM_QUEUE_ST_CLOSED) { // key对应的状态是关闭的 goto ERR_CLOSED; } } remoteQueue = shm_socket_attach_queue(key); if (remoteQueue == NULL ) { goto ERR_CLOSED; } else if(remoteQueue->isClosed()) { goto ERR_CLOSED; } sendpak->key = sockt->key; rv = remoteQueue->push(*sendpak, timeout, flag); if(rv != 0) { mm_free(sendpak->buf); } if(rv == ETIMEDOUT) { return EBUS_TIMEOUT; } return rv; ERR_CLOSED: logger->error("sendto key %d failed, %s", key, bus_strerror(EBUS_CLOSED)); return EBUS_CLOSED; } // 短连接方式接受 static int shm_recvpakfrom(shm_socket_t *sockt, shm_packet_t *_recvpak , const struct timespec *timeout, int flag) { int rv; shm_queue_status_t stRecord; hashtable_t *hashtable = mm_get_hashtable(); shm_packet_t recvpak; if( sockt->queue != NULL) goto LABEL_POP; // if(hashtable_get_queue_count(hashtable) > QUEUE_COUNT_LIMIT) { // return EBUS_EXCEED_LIMIT; // } { if ((rv = pthread_mutex_lock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_lock"); if (sockt->key == 0) { sockt->key = hashtable_alloc_key(hashtable); } sockt->queue = shm_socket_bind_queue( sockt->key, sockt->force_bind); if(sockt->queue == NULL ) { logger->error("%s. key = %d", bus_strerror(EBUS_KEY_INUSED), sockt->key); if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_unlock"); return EBUS_KEY_INUSED; } // 标记key对应的状态 ,为opened stRecord.status = SHM_QUEUE_ST_OPENED; stRecord.createTime = time(NULL); shmQueueStMap->insert({sockt->key, stRecord}); if ((rv = pthread_mutex_unlock(&(sockt->mutex))) != 0) err_exit(rv, "shm_recvfrom : pthread_mutex_unlock"); } LABEL_POP: rv = sockt->queue->pop(recvpak, timeout, flag); if(rv != 0) { if(rv == ETIMEDOUT) { return EBUS_TIMEOUT; } return rv; } if(recvpak.action == BUS_ACTION_STOP) { return EBUS_STOPED; } *_recvpak = recvpak; return 0; }