summaryrefslogtreecommitdiffstats
path: root/xlators/protocol/transport/ib-verbs/src/ib-verbs.h
diff options
context:
space:
mode:
Diffstat (limited to 'xlators/protocol/transport/ib-verbs/src/ib-verbs.h')
-rw-r--r--xlators/protocol/transport/ib-verbs/src/ib-verbs.h220
1 files changed, 220 insertions, 0 deletions
diff --git a/xlators/protocol/transport/ib-verbs/src/ib-verbs.h b/xlators/protocol/transport/ib-verbs/src/ib-verbs.h
new file mode 100644
index 00000000000..c385b62e5cb
--- /dev/null
+++ b/xlators/protocol/transport/ib-verbs/src/ib-verbs.h
@@ -0,0 +1,220 @@
+/*
+ Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
+ This file is part of GlusterFS.
+
+ GlusterFS is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ GlusterFS is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see
+ <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _XPORT_IB_VERBS_H
+#define _XPORT_IB_VERBS_H
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifndef MAX_IOVEC
+#define MAX_IOVEC 16
+#endif /* MAX_IOVEC */
+
+#include "xlator.h"
+#include "event.h"
+#include "ib-verbs-mem-types.h"
+
+#include <stdio.h>
+#include <list.h>
+#include <arpa/inet.h>
+#include <infiniband/verbs.h>
+
+#define GF_DEFAULT_IBVERBS_LISTEN_PORT 6997
+
+/* options per transport end point */
+struct _ib_verbs_options {
+ int32_t port;
+ char *device_name;
+ enum ibv_mtu mtu;
+ int32_t send_count;
+ int32_t recv_count;
+ uint64_t recv_size;
+ uint64_t send_size;
+};
+typedef struct _ib_verbs_options ib_verbs_options_t;
+
+
+struct _ib_verbs_header {
+ char colonO[3];
+ uint32_t size1;
+ uint32_t size2;
+ char version;
+} __attribute__((packed));
+typedef struct _ib_verbs_header ib_verbs_header_t;
+
+struct _ib_verbs_ioq {
+ union {
+ struct list_head list;
+ struct {
+ struct _ib_verbs_ioq *next;
+ struct _ib_verbs_ioq *prev;
+ };
+ };
+ ib_verbs_header_t header;
+ struct iovec vector[MAX_IOVEC];
+ int count;
+ char *buf;
+ struct iobref *iobref;
+};
+typedef struct _ib_verbs_ioq ib_verbs_ioq_t;
+
+/* represents one communication peer, two per transport_t */
+struct _ib_verbs_peer {
+ transport_t *trans;
+ struct ibv_qp *qp;
+
+ int32_t recv_count;
+ int32_t send_count;
+ int32_t recv_size;
+ int32_t send_size;
+
+ int32_t quota;
+ union {
+ struct list_head ioq;
+ struct {
+ ib_verbs_ioq_t *ioq_next;
+ ib_verbs_ioq_t *ioq_prev;
+ };
+ };
+
+ /* QP attributes, needed to connect with remote QP */
+ int32_t local_lid;
+ int32_t local_psn;
+ int32_t local_qpn;
+ int32_t remote_lid;
+ int32_t remote_psn;
+ int32_t remote_qpn;
+};
+typedef struct _ib_verbs_peer ib_verbs_peer_t;
+
+
+struct _ib_verbs_post {
+ struct _ib_verbs_post *next, *prev;
+ struct ibv_mr *mr;
+ char *buf;
+ int32_t buf_size;
+ char aux;
+ int32_t reused;
+ pthread_barrier_t wait;
+};
+typedef struct _ib_verbs_post ib_verbs_post_t;
+
+
+struct _ib_verbs_queue {
+ ib_verbs_post_t active_posts, passive_posts;
+ int32_t active_count, passive_count;
+ pthread_mutex_t lock;
+};
+typedef struct _ib_verbs_queue ib_verbs_queue_t;
+
+
+struct _ib_verbs_qpreg {
+ pthread_mutex_t lock;
+ int32_t count;
+ struct _qpent {
+ struct _qpent *next, *prev;
+ int32_t qp_num;
+ ib_verbs_peer_t *peer;
+ } ents[42];
+};
+typedef struct _ib_verbs_qpreg ib_verbs_qpreg_t;
+
+/* context per device, stored in global glusterfs_ctx_t->ib */
+struct _ib_verbs_device {
+ struct _ib_verbs_device *next;
+ const char *device_name;
+ struct ibv_context *context;
+ int32_t port;
+ struct ibv_pd *pd;
+ struct ibv_srq *srq;
+ ib_verbs_qpreg_t qpreg;
+ struct ibv_comp_channel *send_chan, *recv_chan;
+ struct ibv_cq *send_cq, *recv_cq;
+ ib_verbs_queue_t sendq, recvq;
+ pthread_t send_thread, recv_thread;
+};
+typedef struct _ib_verbs_device ib_verbs_device_t;
+
+typedef enum {
+ IB_VERBS_HANDSHAKE_START = 0,
+ IB_VERBS_HANDSHAKE_SENDING_DATA,
+ IB_VERBS_HANDSHAKE_RECEIVING_DATA,
+ IB_VERBS_HANDSHAKE_SENT_DATA,
+ IB_VERBS_HANDSHAKE_RECEIVED_DATA,
+ IB_VERBS_HANDSHAKE_SENDING_ACK,
+ IB_VERBS_HANDSHAKE_RECEIVING_ACK,
+ IB_VERBS_HANDSHAKE_RECEIVED_ACK,
+ IB_VERBS_HANDSHAKE_COMPLETE,
+} ib_verbs_handshake_state_t;
+
+struct ib_verbs_nbio {
+ int state;
+ char *buf;
+ int count;
+ struct iovec vector;
+ struct iovec *pending_vector;
+ int pending_count;
+};
+
+
+struct _ib_verbs_private {
+ int32_t sock;
+ int32_t idx;
+ unsigned char connected;
+ unsigned char tcp_connected;
+ unsigned char ib_connected;
+ in_addr_t addr;
+ unsigned short port;
+
+ /* IB Verbs Driver specific variables, pointers */
+ ib_verbs_peer_t peer;
+ ib_verbs_device_t *device;
+ ib_verbs_options_t options;
+
+ /* Used by trans->op->receive */
+ char *data_ptr;
+ int32_t data_offset;
+ int32_t data_len;
+
+ /* Mutex */
+ pthread_mutex_t read_mutex;
+ pthread_mutex_t write_mutex;
+ pthread_barrier_t handshake_barrier;
+ char handshake_ret;
+
+ pthread_mutex_t recv_mutex;
+ pthread_cond_t recv_cond;
+
+ /* used during ib_verbs_handshake */
+ struct {
+ struct ib_verbs_nbio incoming;
+ struct ib_verbs_nbio outgoing;
+ int state;
+ ib_verbs_header_t header;
+ char *buf;
+ size_t size;
+ } handshake;
+};
+typedef struct _ib_verbs_private ib_verbs_private_t;
+
+#endif /* _XPORT_IB_VERBS_H */