RDMA编程实践-SEND-RECEICVE原语应用

RDMA编程实践

本文描述了RDMA编程过程中的SEND-RECEIVE双边原语的代码实现。包含多个版本，1、client向server发送消息，server回复client收到消息(ACK)，然后两边断开连接。2、server端循环等待客户端建立连接，client发送一次消息后，双方断开连接。3、server端循环等待客户端建立连接，一旦建立，client端可以一直向server端发送消息，直到发送消息为disconnect，server和client断开链接，但是server此时仍然可以等待别的client发送消息。
代码基于代码基于send-receive样例实现。关于代码注释，可以参考代码解释：
Makefile文件、会编译当前目录下的所有.c文件：

.PHONY: all cleanCC := gcc
CFLAGS := -Wall -g
LDLIBS := -lrdmacm -libverbs -lpthread -gSRCS := $(wildcard *.c)
APPS := $(SRCS:.c=)all: $(APPS)%: %.c$(CC) $(CFLAGS) $< -o $@ $(LDLIBS)clean:rm -f $(APPS)

version1 客户端-服务端消息一次传递

在这个阶段，我们希望能实现下面这样一个场景。client与server端相连接，client端能够发送一条消息给server，server收到该条消息之后恢复一条消息给client端表示我已经确认收到。之后两者断开连接。

代码：

// client1.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <netdb.h>
#include <errno.h>
#include <getopt.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>static const char *server = "10.10.10.1";
static const char *port = "7471";static struct rdma_cm_id *id;
static struct ibv_mr *mr, *send_mr;
static int send_flags;
static uint8_t send_msg[16];
static uint8_t recv_msg[16];static int run(void)
{struct rdma_addrinfo hints, *res;struct ibv_qp_init_attr attr;struct ibv_wc wc;int ret;memset(&hints, 0, sizeof hints);hints.ai_port_space = RDMA_PS_TCP;ret = rdma_getaddrinfo(server, port, &hints, &res);if (ret) {printf("rdma_getaddrinfo: %s\n", gai_strerror(ret));goto out;}memset(&attr, 0, sizeof attr);attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;attr.cap.max_inline_data = 16;attr.qp_context = id;attr.sq_sig_all = 1;ret = rdma_create_ep(&id, res, NULL, &attr);// Check to see if we got inline data allowed or notif (attr.cap.max_inline_data >= 16)send_flags = IBV_SEND_INLINE;elseprintf("rdma_client: device doesn't support IBV_SEND_INLINE, ""using sge sends\n");if (ret) {perror("rdma_create_ep");goto out_free_addrinfo;}mr = rdma_reg_msgs(id, recv_msg, 16);if (!mr) {perror("rdma_reg_msgs for recv_msg");ret = -1;goto out_destroy_ep;}if ((send_flags & IBV_SEND_INLINE) == 0) {send_mr = rdma_reg_msgs(id, send_msg, 16);if (!send_mr) {perror("rdma_reg_msgs for send_msg");ret = -1;goto out_dereg_recv;}}ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);if (ret) {perror("rdma_post_recv");goto out_dereg_send;}ret = rdma_connect(id, NULL);if (ret) {perror("rdma_connect");goto out_dereg_send;}printf("client send: %s\n", (char *)send_msg);ret = rdma_post_send(id, NULL, send_msg, 16, send_mr, send_flags);if (ret) {perror("rdma_post_send");goto out_disconnect;}while ((ret = rdma_get_send_comp(id, &wc)) == 0);if (ret < 0) {perror("rdma_get_send_comp");goto out_disconnect;}while ((ret = rdma_get_recv_comp(id, &wc)) == 0);if (ret < 0)perror("rdma_get_recv_comp");elseret = 0;printf("client received: %s\n", (char *) recv_msg);out_disconnect:rdma_disconnect(id);
out_dereg_send:if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);
out_dereg_recv:rdma_dereg_mr(mr);
out_destroy_ep:rdma_destroy_ep(id);
out_free_addrinfo:rdma_freeaddrinfo(res);
out:return ret;
}int main(int argc, char **argv)
{int ret;char *s = "hello world";// printf("client send: %s\n", s);memcpy(send_msg, s , strlen(s));printf("rdma_client: start\n");ret = run();printf("rdma_client: end %d\n", ret);return ret;
}

server端代码

// server1.c
/** Copyright (c) 2005-2009 Intel Corporation.  All rights reserved.** This software is available to you under the OpenIB.org BSD license* below:**     Redistribution and use in source and binary forms, with or*     without modification, are permitted provided that the following*     conditions are met:**      - Redistributions of source code must retain the above*        copyright notice, this list of conditions and the following*        disclaimer.**      - Redistributions in binary form must reproduce the above*        copyright notice, this list of conditions and the following*        disclaimer in the documentation and/or other materials*        provided with the distribution.** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE* SOFTWARE.*/#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <netdb.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>static const char *server = "0.0.0.0";
static const char *port = "7471";static struct rdma_cm_id *listen_id, *id;
static struct ibv_mr *mr, *send_mr;
static int send_flags;
static uint8_t send_msg[16];
static uint8_t recv_msg[16];static int run(void)
{struct rdma_addrinfo hints, *res;struct ibv_qp_init_attr init_attr;struct ibv_qp_attr qp_attr;struct ibv_wc wc;int ret;memset(&hints, 0, sizeof hints);hints.ai_flags = RAI_PASSIVE;hints.ai_port_space = RDMA_PS_TCP;ret = rdma_getaddrinfo(server, port, &hints, &res);if (ret) {printf("rdma_getaddrinfo: %s\n", gai_strerror(ret));return ret;}memset(&init_attr, 0, sizeof init_attr);init_attr.cap.max_send_wr = init_attr.cap.max_recv_wr = 1;init_attr.cap.max_send_sge = init_attr.cap.max_recv_sge = 1;init_attr.cap.max_inline_data = 16;init_attr.sq_sig_all = 1;ret = rdma_create_ep(&listen_id, res, NULL, &init_attr);if (ret) {perror("rdma_create_ep");goto out_free_addrinfo;}ret = rdma_listen(listen_id, 0);if (ret) {perror("rdma_listen");goto out_destroy_listen_ep;}ret = rdma_get_request(listen_id, &id);if (ret) {perror("rdma_get_request");goto out_destroy_listen_ep;}memset(&qp_attr, 0, sizeof qp_attr);memset(&init_attr, 0, sizeof init_attr);ret = ibv_query_qp(id->qp, &qp_attr, IBV_QP_CAP,&init_attr);if (ret) {perror("ibv_query_qp");goto out_destroy_accept_ep;}if (init_attr.cap.max_inline_data >= 16)send_flags = IBV_SEND_INLINE;elseprintf("rdma_server: device doesn't support IBV_SEND_INLINE, ""using sge sends\n");mr = rdma_reg_msgs(id, recv_msg, 16);if (!mr) {ret = -1;perror("rdma_reg_msgs for recv_msg");goto out_destroy_accept_ep;}if ((send_flags & IBV_SEND_INLINE) == 0) {send_mr = rdma_reg_msgs(id, send_msg, 16);if (!send_mr) {ret = -1;perror("rdma_reg_msgs for send_msg");goto out_dereg_recv;}}ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);if (ret) {perror("rdma_post_recv");goto out_dereg_send;}ret = rdma_accept(id, NULL);if (ret) {perror("rdma_accept");goto out_dereg_send;}while ((ret = rdma_get_recv_comp(id, &wc)) == 0);if (ret < 0) {perror("rdma_get_recv_comp");goto out_disconnect;}printf("server received: %s\n" , (char *)recv_msg);char *s = "ACK";memcpy(send_msg, s, strlen(s));printf("server send: %s\n", (char *)send_msg);ret = rdma_post_send(id, NULL, send_msg, 16, send_mr, send_flags);if (ret) {perror("rdma_post_send");goto out_disconnect;}while ((ret = rdma_get_send_comp(id, &wc)) == 0);if (ret < 0)perror("rdma_get_send_comp");elseret = 0;out_disconnect:rdma_disconnect(id);
out_dereg_send:if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);
out_dereg_recv:rdma_dereg_mr(mr);
out_destroy_accept_ep:rdma_destroy_ep(id);
out_destroy_listen_ep:rdma_destroy_ep(listen_id);
out_free_addrinfo:rdma_freeaddrinfo(res);return ret;
}int main(int argc, char **argv)
{int ret;printf("rdma_server: start\n");ret = run();printf("rdma_server: end %d\n", ret);return ret;
}

首先make编译完之后，在server端执行 ./server1，然后在客户端执行./client1
运行结果：
在这里插入图片描述
在这里插入图片描述
可以看到 client向server发送了hello world，server收到之后打印出来并回复给client端ACK消息，client收到之后并打印。最后双方断开连接，完成！

version2-客户端发送一次，服务端循环等待

client2的代码跟上面一样，server2代码不一样。
server2的逻辑：在run函数进来之后记录一个connect点，当远程客户端发送完信息后，释放连接的资源，跳转到connect阶段准备让下一个client连接。

/** Copyright (c) 2005-2009 Intel Corporation.  All rights reserved.** This software is available to you under the OpenIB.org BSD license* below:**     Redistribution and use in source and binary forms, with or*     without modification, are permitted provided that the following*     conditions are met:**      - Redistributions of source code must retain the above*        copyright notice, this list of conditions and the following*        disclaimer.**      - Redistributions in binary form must reproduce the above*        copyright notice, this list of conditions and the following*        disclaimer in the documentation and/or other materials*        provided with the distribution.** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE* SOFTWARE.*/#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <netdb.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>#define N 100
#define MAX_CAP 32static const char *server = "0.0.0.0";
static const char *port = "7471";static struct rdma_cm_id *listen_id, *id;
static struct ibv_mr *mr, *send_mr;
static int send_flags;
static uint8_t send_msg[MAX_CAP];
static uint8_t recv_msg[MAX_CAP];static int run(void)
{struct rdma_addrinfo hints, *res;struct ibv_qp_init_attr init_attr;struct ibv_qp_attr qp_attr;struct ibv_wc wc;int ret;while(1){memset(&hints, 0, sizeof hints);hints.ai_flags = RAI_PASSIVE;hints.ai_port_space = RDMA_PS_TCP;ret = rdma_getaddrinfo(server, port, &hints, &res);if (ret) {printf("rdma_getaddrinfo: %s\n", gai_strerror(ret));return ret;}memset(&init_attr, 0, sizeof init_attr);init_attr.cap.max_send_wr = init_attr.cap.max_recv_wr = N;init_attr.cap.max_send_sge = init_attr.cap.max_recv_sge = 1;init_attr.cap.max_inline_data = MAX_CAP;init_attr.sq_sig_all = 1;ret = rdma_create_ep(&listen_id, res, NULL, &init_attr);if (ret) {perror("rdma_create_ep");goto out_free_addrinfo;}ret = rdma_listen(listen_id, 0);if (ret) {perror("rdma_listen");goto out_destroy_listen_ep;}ret = rdma_get_request(listen_id, &id);if (ret) {perror("rdma_get_request");goto out_destroy_listen_ep;}memset(&qp_attr, 0, sizeof qp_attr);memset(&init_attr, 0, sizeof init_attr);ret = ibv_query_qp(id->qp, &qp_attr, IBV_QP_CAP,&init_attr);if (ret) {perror("ibv_query_qp");goto out_destroy_accept_ep;}if (init_attr.cap.max_inline_data >= MAX_CAP)send_flags = IBV_SEND_INLINE;elseprintf("rdma_server: device doesn't support IBV_SEND_INLINE, ""using sge sends\n");mr = rdma_reg_msgs(id, recv_msg, N);if (!mr) {ret = -1;perror("rdma_reg_msgs for recv_msg");goto out_destroy_accept_ep;}if ((send_flags & IBV_SEND_INLINE) == 0) {send_mr = rdma_reg_msgs(id, send_msg, MAX_CAP);if (!send_mr) {ret = -1;perror("rdma_reg_msgs for send_msg");goto out_dereg_recv;}}   ret = rdma_accept(id, NULL);if (ret) {perror("rdma_accept");goto out_dereg_send;}memset(recv_msg, 0 , sizeof recv_msg);memset(send_msg, 0 , sizeof send_msg);ret = rdma_post_recv(id, NULL, recv_msg, MAX_CAP, mr);if (ret) {perror("rdma_post_recv");goto out_dereg_send;}while ((ret = rdma_get_recv_comp(id, &wc)) == 0);if (ret < 0) {perror("rdma_get_recv_comp");goto out_disconnect;}printf("server received: %s\n", (char *)recv_msg);memcpy(send_msg, recv_msg, sizeof(recv_msg));ret = rdma_post_send(id, NULL, send_msg, MAX_CAP, send_mr, send_flags);if (ret) {perror("rdma_post_send");goto out_disconnect;}while ((ret = rdma_get_send_comp(id, &wc)) == 0); // 确认对方已经收到 对方会发送ackif (ret < 0)perror("rdma_get_send_comp");elseret = 0;rdma_disconnect(id);if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);rdma_dereg_mr(mr);rdma_destroy_ep(id);rdma_destroy_ep(listen_id);rdma_freeaddrinfo(res);  }out_disconnect:rdma_disconnect(id);
out_dereg_send:if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);
out_dereg_recv:rdma_dereg_mr(mr);
out_destroy_accept_ep:rdma_destroy_ep(id);
out_destroy_listen_ep:rdma_destroy_ep(listen_id);
out_free_addrinfo:rdma_freeaddrinfo(res);  return ret;
}int main(int argc, char **argv)
{int ret;printf("rdma_server: start\n");ret = run();printf("rdma_server: end %d\n", ret);return ret;
}

运行结果：
在这里插入图片描述

在这里插入图片描述

可以看到客户端发送一次消息之后便结束了，服务端却一直等待连接，直到按下ctrl+c。

version3-客户端循环发送，服务端循环等待，一次连接

和上述版本2不同的时候，这里client和server只连接一次，然后可以多次发送消息。直到client发送的消息为disconnect

// client3.c
/** Copyright (c) 2010 Intel Corporation.  All rights reserved.** This software is available to you under the OpenIB.org BSD license* below:**     Redistribution and use in source and binary forms, with or*     without modification, are permitted provided that the following*     conditions are met:**      - Redistributions of source code must retain the above*        copyright notice, this list of conditions and the following*        disclaimer.**      - Redistributions in binary form must reproduce the above*        copyright notice, this list of conditions and the following*        disclaimer in the documentation and/or other materials*        provided with the distribution.** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE* SOFTWARE.*/#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <netdb.h>
#include <errno.h>
#include <getopt.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>#define N 100
#define MAX_CAP 32static const char *server = "10.10.10.1";
static const char *port = "7471";static struct rdma_cm_id *id;
static struct ibv_mr *mr, *send_mr;
static int send_flags;
static uint8_t send_msg[MAX_CAP];
static uint8_t recv_msg[MAX_CAP];static int run(void)
{struct rdma_addrinfo hints, *res;struct ibv_qp_init_attr attr;struct ibv_wc wc;int ret;memset(&hints, 0, sizeof hints);hints.ai_port_space = RDMA_PS_TCP;ret = rdma_getaddrinfo(server, port, &hints, &res);if (ret) {printf("rdma_getaddrinfo: %s\n", gai_strerror(ret));goto out;}memset(&attr, 0, sizeof attr);attr.cap.max_send_wr = attr.cap.max_recv_wr = 5;attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;attr.cap.max_inline_data = MAX_CAP;attr.qp_context = id;attr.sq_sig_all = 1;ret = rdma_create_ep(&id, res, NULL, &attr);// Check to see if we got inline data allowed or notif (attr.cap.max_inline_data >= MAX_CAP)send_flags = IBV_SEND_INLINE;elseprintf("rdma_client: device doesn't support IBV_SEND_INLINE, ""using sge sends\n");if (ret) {perror("rdma_create_ep");goto out_free_addrinfo;}mr = rdma_reg_msgs(id, recv_msg, MAX_CAP);if (!mr) {perror("rdma_reg_msgs for recv_msg");ret = -1;goto out_destroy_ep;}if ((send_flags & IBV_SEND_INLINE) == 0) {send_mr = rdma_reg_msgs(id, send_msg, MAX_CAP);if (!send_mr) {perror("rdma_reg_msgs for send_msg");ret = -1;goto out_dereg_recv;}}// ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);// if (ret) {//     perror("rdma_post_recv");//     goto out_dereg_send;// }// printf("123\n");ret = rdma_connect(id, NULL);if (ret) {perror("rdma_connect");goto out_dereg_send;}while(1){// sleep(5);memset(recv_msg, 0 , sizeof recv_msg);memset(send_msg, 0 , sizeof send_msg);printf("input send message: ");scanf("%s", send_msg);getchar();ret = rdma_post_recv(id, NULL, recv_msg, MAX_CAP, mr);if (ret) {perror("rdma_post_recv");goto out_dereg_send;}ret = rdma_post_send(id, NULL, send_msg, MAX_CAP, send_mr, send_flags);if (ret) {perror("rdma_post_send");goto out_disconnect;}while ((ret = rdma_get_send_comp(id, &wc)) == 0);if (ret < 0) {perror("rdma_get_send_comp");goto out_disconnect;}while ((ret = rdma_get_recv_comp(id, &wc)) == 0);if (ret < 0)perror("rdma_get_recv_comp");elseret = 0;if(strcmp((char*)send_msg,"disconnect") == 0){printf("disconnect\n");goto out_disconnect;}else{printf("%s\n", recv_msg);}}out_disconnect:rdma_disconnect(id);
out_dereg_send:if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);
out_dereg_recv:rdma_dereg_mr(mr);
out_destroy_ep:rdma_destroy_ep(id);
out_free_addrinfo:rdma_freeaddrinfo(res);
out:return ret;}int main(int argc, char **argv)
{int ret;//memcpy(send_msg, argv[1], 50);// while ((op = getopt(argc, argv, "s:p:")) != -1) {// 	switch (op) {// 	case 's':// 		server = optarg;// 		break;// 	case 'p':// 		port = optarg;// 		break;// 	default:// 		printf("usage: %s\n", argv[0]);// 		printf("\t[-s server_address]\n");// 		printf("\t[-p port_number]\n");// 		exit(1);// 	}// }printf("rdma_client: start\n");ret = run();printf("rdma_client: end %d\n", ret);return ret;
}

// server3.c
/** Copyright (c) 2005-2009 Intel Corporation.  All rights reserved.** This software is available to you under the OpenIB.org BSD license* below:**     Redistribution and use in source and binary forms, with or*     without modification, are permitted provided that the following*     conditions are met:**      - Redistributions of source code must retain the above*        copyright notice, this list of conditions and the following*        disclaimer.**      - Redistributions in binary form must reproduce the above*        copyright notice, this list of conditions and the following*        disclaimer in the documentation and/or other materials*        provided with the distribution.** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE* SOFTWARE.*/#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <netdb.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>#define N 100
#define MAX_CAP 32static const char *server = "0.0.0.0";
static const char *port = "7471";static struct rdma_cm_id *listen_id, *id;
static struct ibv_mr *mr, *send_mr;
static int send_flags;
static uint8_t send_msg[MAX_CAP];
static uint8_t recv_msg[MAX_CAP];static int run(void)
{struct rdma_addrinfo hints, *res;struct ibv_qp_init_attr init_attr;struct ibv_qp_attr qp_attr;struct ibv_wc wc;int ret;connect:memset(&hints, 0, sizeof hints);hints.ai_flags = RAI_PASSIVE;hints.ai_port_space = RDMA_PS_TCP;ret = rdma_getaddrinfo(server, port, &hints, &res);if (ret) {printf("rdma_getaddrinfo: %s\n", gai_strerror(ret));return ret;}memset(&init_attr, 0, sizeof init_attr);init_attr.cap.max_send_wr = init_attr.cap.max_recv_wr = N;init_attr.cap.max_send_sge = init_attr.cap.max_recv_sge = 1;init_attr.cap.max_inline_data = MAX_CAP;init_attr.sq_sig_all = 1;ret = rdma_create_ep(&listen_id, res, NULL, &init_attr);if (ret) {perror("rdma_create_ep");goto out_free_addrinfo;}ret = rdma_listen(listen_id, 0);if (ret) {perror("rdma_listen");goto out_destroy_listen_ep;}ret = rdma_get_request(listen_id, &id);if (ret) {perror("rdma_get_request");goto out_destroy_listen_ep;}memset(&qp_attr, 0, sizeof qp_attr);memset(&init_attr, 0, sizeof init_attr);ret = ibv_query_qp(id->qp, &qp_attr, IBV_QP_CAP,&init_attr);if (ret) {perror("ibv_query_qp");goto out_destroy_accept_ep;}if (init_attr.cap.max_inline_data >= MAX_CAP)send_flags = IBV_SEND_INLINE;elseprintf("rdma_server: device doesn't support IBV_SEND_INLINE, ""using sge sends\n");mr = rdma_reg_msgs(id, recv_msg, N);if (!mr) {ret = -1;perror("rdma_reg_msgs for recv_msg");goto out_destroy_accept_ep;}if ((send_flags & IBV_SEND_INLINE) == 0) {send_mr = rdma_reg_msgs(id, send_msg, MAX_CAP);if (!send_mr) {ret = -1;perror("rdma_reg_msgs for send_msg");goto out_dereg_recv;}}   ret = rdma_accept(id, NULL);if (ret) {perror("rdma_accept");goto out_dereg_send;}while (1) {memset(recv_msg, 0 , sizeof recv_msg);memset(send_msg, 0 , sizeof send_msg);ret = rdma_post_recv(id, NULL, recv_msg, MAX_CAP, mr);if (ret) {perror("rdma_post_recv");goto out_dereg_send;}while ((ret = rdma_get_recv_comp(id, &wc)) == 0);if (ret < 0) {perror("rdma_get_recv_comp");goto out_disconnect;}char *s = (char *)recv_msg;int total_length = strlen("server get ") + strlen(s); // 加1是为了存储字符串结束符'\0'char *recv_str = (char *)malloc(total_length);  // 分配足够的空间strcpy(recv_str, "server get ");strcat(recv_str, s);//printf("%s\n", recv_str);memcpy(send_msg, recv_str, strlen(recv_str));ret = rdma_post_send(id, NULL, send_msg, MAX_CAP, send_mr, send_flags);if (ret) {perror("rdma_post_send");goto out_disconnect;}if(strcmp((char*)recv_msg,"disconnect") == 0){//printf("%s\n",recv_msg);printf("client disconnect\n");rdma_disconnect(id);if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);rdma_dereg_mr(mr);rdma_destroy_ep(id);rdma_destroy_ep(listen_id);rdma_freeaddrinfo(res);  //goto out_disconnect;goto connect;}else{printf("%s\n", recv_msg);}// while ((ret = rdma_get_send_comp(id, &wc)) == 0); // 确认对方已经收到 对方发送ack// printf("after send\n");// if (ret < 0)//     perror("rdma_get_send_comp");// else//     ret = 0;}out_disconnect:rdma_disconnect(id);
out_dereg_send:if ((send_flags & IBV_SEND_INLINE) == 0)rdma_dereg_mr(send_mr);
out_dereg_recv:rdma_dereg_mr(mr);
out_destroy_accept_ep:rdma_destroy_ep(id);
out_destroy_listen_ep:rdma_destroy_ep(listen_id);
out_free_addrinfo:rdma_freeaddrinfo(res);  return ret;
}int main(int argc, char **argv)
{int op, ret;while ((op = getopt(argc, argv, "s:p:")) != -1) {switch (op) {case 's':server = optarg;break;case 'p':port = optarg;break;default:printf("usage: %s\n", argv[0]);printf("\t[-s server_address]\n");printf("\t[-p port_number]\n");exit(1);}}printf("rdma_server: start\n");ret = run();printf("rdma_server: end %d\n", ret);return ret;
}

运行结果：

在这里插入图片描述
在这里插入图片描述

总结：

本文实现了rdma中send-receive双边原语的三种需求版本，从单次发送到两者都能多次发送。理解其中的代码逻辑，想要发送消息之前对端得创建一个recv队列用来接收消息。发送完了有一个发送完成队列，接收完了也有一个接收完成队列。最后双方断开连接需要一起断开，不能某一方执行disconnect另一方不执行。本次实验有一个关键点：

while ((ret = rdma_get_send_comp(id, &wc)) == 0)

这一行代码是等待发送成功，发送成功之后，对方会给一个隐式信息表示我已经收到。这里耗费得时间比较长一点，在版本3中，如果不注释掉，在server端的receive队列还没有建立好，这就导致client发送了消息，server还没有收到，双方就陷入了死循环中。

本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如若转载，请注明出处：http://www.hqwc.cn/news/412983.html

如若内容造成侵权/违法违规/事实不符，请联系编程知识网进行投诉反馈email:809451989@qq.com，一经查实，立即删除！

相关文章

【Android】为什么在子线程中更新UI不会抛出异常

【Android】为什么在子线程中更新UI不会抛出异常

转载请注明来源&#xff1a;https://blog.csdn.net/devnn/article/details/135638486 前言 众所周知&#xff0c;Android App在子线程中是不允许更新UI的&#xff0c;否则会抛出异常&#xff1a; android.view.ViewRootImpl$CalledFromWrongThreadException: Only the origin…
阅读更多...
NumPy 中数组拼接、合并详解

NumPy 中数组拼接、合并详解

1、np.append() 1.1、语法 将值添加到数组的末端&#xff0c;返回一个新的数组&#xff0c;而原数组不变。 numpy.append(arr, values, axisNone)参数描述arr : 类数组输入的数组values : 类数组向数组 arr 添加的元素&#xff0c;需要与 arr 维度相同axis : 整型添加操作的…
阅读更多...
python对自动驾驶进行模拟

python对自动驾驶进行模拟

使用了 Pygame 库来创建一个简单的游戏环境,模拟了一辆自动驾驶汽车在道路上行驶。汽车的位置和速度通过键盘控制&#xff0c;可以左右移动和加速减速。道路的宽度和颜色可以根据需要进行调整。 import pygame import random # 游戏窗口大小 WINDOW_WIDTH 800 WINDOW_HEIG…
阅读更多...
git 删除 submodule 子模块的步骤

git 删除 submodule 子模块的步骤

实验有效&#xff0c;这里删除了两个 submodule。 1, 执行删除 submodule mkdir tmp1 && cd tmp1 && git clone --recursive ssh://gitaaa.bbb.ccc.git \ && cd ccc/ && git checkout -b abranch_01 \ && git submodule deinit -f…
阅读更多...
我终于学会的前端技能——代码调试、打断点

我终于学会的前端技能——代码调试、打断点

在技术的世界里&#xff0c;要用魔法来打败魔法 说来惭愧我做前端已近三年了竟然还没有学会如何调试代码&#xff0c;也就是给自己的代码打上断点一步步看它的运行状态以达到理清代码运行逻辑、排查问题提升开发效率的目的。直到最近我才学会了这一技能&#xff0c;在这之前我…
阅读更多...
Neos的渗透测试靶机练习——DarkHole-1

Neos的渗透测试靶机练习——DarkHole-1

DarkHole-1 一、实验环境二、开始渗透1. 搜集信息2. sql注入4. 提权 三、总结 一、实验环境 虚拟机软件&#xff1a;VirtualBox 攻击机&#xff1a;kali linux&#xff08;网卡初始为仅主机模式&#xff0c;要有安全意识&#xff09; 靶机&#xff1a;DarkHole-1&#xff08;网…
阅读更多...
磁盘位置不可用怎么修复？

磁盘位置不可用怎么修复？

磁盘位置不可用是计算机使用中经常遇到的问题。造成磁盘位置不可用的原因有多种&#xff0c;其中最常见的是磁盘文件系统损坏。当文件系统损坏时&#xff0c;操作系统无法正常访问磁盘上的数据&#xff0c;导致磁盘位置不可用。 磁盘位置不可用怎么修复&#xff1f; 当磁盘位置…
阅读更多...
2024年北京市安全员-C3证证模拟考试题库及北京市安全员-C3证理论考试试题

2024年北京市安全员-C3证证模拟考试题库及北京市安全员-C3证理论考试试题

题库来源&#xff1a;安全生产模拟考试一点通公众号小程序 2024年北京市安全员-C3证证模拟考试题库及北京市安全员-C3证理论考试试题是由安全生产模拟考试一点通提供&#xff0c;北京市安全员-C3证证模拟考试题库是根据北京市安全员-C3证最新版教材&#xff0c;北京市安全员-C…
阅读更多...
探索数据之美：深入Seaborn的数据可视化艺术与技巧【第26篇—python：Seaborn】

探索数据之美：深入Seaborn的数据可视化艺术与技巧【第26篇—python：Seaborn】

文章目录 1. 引言2. Seaborn基础2.1 安装和环境设置2.2 常用数据可视化函数2.3 设置样式和颜色主题 3. 数据准备与导入3.1 使用Pandas库加载和处理数据3.2 数据清理和缺失值处理 4. Seaborn中的常见图表4.1 折线图和散点图&#xff1a;展示趋势和变量关系4.2 条形图和箱线图&am…
阅读更多...
个性化定制的知识付费小程序，为用户提供个性化的知识服务

个性化定制的知识付费小程序，为用户提供个性化的知识服务

明理信息科技知识付费saas租户平台 随着知识经济的兴起&#xff0c;越来越多的人开始重视知识付费&#xff0c;并希望通过打造自己的知识付费平台来实现自己的知识变现。本文将介绍如何打造自己的知识付费平台&#xff0c;并从定位、内容制作、渠道推广、运营维护四个方面进行…
阅读更多...
Spring Boot 优雅实现统一数据返回格式+统一异常处理+统一日志处理

Spring Boot 优雅实现统一数据返回格式+统一异常处理+统一日志处理

在我们的项目开发中&#xff0c;我们都会对数据返回格式进行统一的处理&#xff0c;这样可以方便前端人员取数据&#xff0c;当然除了正常流程的数据返回格式需要统一以外&#xff0c;我们也需要对异常的情况进行统一的处理&#xff0c;以及项目必备的日志。 1. 统一返回格式 …
阅读更多...
归并排序（C语言）

归并排序（C语言）

目录 1.归并排序图解 2.归并排序&#xff08;递归版&#xff09; 3.归并排序&#xff08;非递归版&#xff09; 1.归并排序图解 归并排序的核心思想是让左右两边有序的部分进行合并比较排序&#xff0c;具体什么意思呢&#xff1f;分两点&#xff1a; 1.分&#xff1a;左右两边…
阅读更多...
推荐文章
最新文章

Copyright @ 2022~2023