summaryrefslogtreecommitdiffstats
path: root/rpc
diff options
context:
space:
mode:
Diffstat (limited to 'rpc')
-rw-r--r--rpc/rpc-lib/src/rpc-clnt-ping.c87
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.h1
-rw-r--r--rpc/rpc-lib/src/rpc-transport.c44
-rw-r--r--rpc/rpc-lib/src/rpc-transport.h3
-rw-r--r--rpc/rpc-lib/src/rpcsvc.c89
-rw-r--r--rpc/rpc-lib/src/rpcsvc.h5
-rw-r--r--rpc/rpc-transport/rdma/src/name.c5
-rw-r--r--rpc/rpc-transport/socket/src/name.c18
-rw-r--r--rpc/rpc-transport/socket/src/socket.c21
-rw-r--r--rpc/xdr/src/glusterfs-fops.x1
10 files changed, 256 insertions, 18 deletions
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
index a7ff866ac99..7ce066dec5f 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.c
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -18,6 +18,7 @@
#include "mem-pool.h"
#include "xdr-rpc.h"
#include "rpc-common-xdr.h"
+#include "timespec.h"
char *clnt_ping_procs[GF_DUMP_MAXVALUE] = {
@@ -30,6 +31,11 @@ struct rpc_clnt_program clnt_ping_prog = {
.procnames = clnt_ping_procs,
};
+struct ping_local {
+ struct rpc_clnt *rpc;
+ struct timespec submit_time;
+};
+
/* Must be called under conn->lock */
static int
__rpc_clnt_rearm_ping_timer (struct rpc_clnt *rpc, gf_timer_cbk_t cbk)
@@ -166,16 +172,48 @@ out:
return;
}
+void
+_update_client_latency (const rpc_clnt_connection_t *conn,
+ call_frame_t *frame,
+ uint64_t elapsed_usec)
+{
+ fop_latency_t *lat;
+
+ lat = &frame->this->client_latency;
+
+ if (elapsed_usec < lat->min) {
+ lat->min = elapsed_usec;
+ }
+
+ if (elapsed_usec > lat->max) {
+ lat->max = elapsed_usec;
+ }
+
+ lat->total += elapsed_usec;
+ lat->count++;
+ lat->mean = lat->mean + (elapsed_usec - lat->mean) / lat->count;
+ gf_log (THIS->name, GF_LOG_DEBUG, "%s - Ping latency is %0.6lf ms, "
+ "avg: %0.6lf ms, count:%ld",
+ conn->trans->peerinfo.identifier, elapsed_usec / 1000.0,
+ lat->mean / 1000.0, lat->count);
+}
+
int
rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
void *myframe)
{
- struct rpc_clnt *rpc = NULL;
+ struct ping_local *local = NULL;
xlator_t *this = NULL;
rpc_clnt_connection_t *conn = NULL;
+
call_frame_t *frame = NULL;
struct timespec timeout = {0, };
+ struct timespec now;
+ struct timespec delta;
+ int64_t latency_usec = 0;
+ int ret = 0;
int unref = 0;
+ gf_boolean_t call_notify = _gf_false;
if (!myframe) {
gf_log (THIS->name, GF_LOG_WARNING,
@@ -185,14 +223,13 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
frame = myframe;
this = frame->this;
- rpc = frame->local;
- frame->local = NULL; /* Prevent STACK_DESTROY from segfaulting */
- conn = &rpc->conn;
+ local = frame->local;
+ conn = &local->rpc->conn;
pthread_mutex_lock (&conn->lock);
{
if (req->rpc_status == -1) {
- unref = rpc_clnt_remove_ping_timer_locked (rpc);
+ unref = rpc_clnt_remove_ping_timer_locked (local->rpc);
if (unref) {
gf_log (this->name, GF_LOG_WARNING,
"socket or ib related error");
@@ -207,8 +244,15 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
goto unlock;
}
- unref = rpc_clnt_remove_ping_timer_locked (rpc);
- if (__rpc_clnt_rearm_ping_timer (rpc,
+ timespec_now (&now);
+ timespec_sub (&local->submit_time, &now, &delta);
+ latency_usec = delta.tv_sec * 1000000UL +
+ delta.tv_nsec / 1000UL;
+
+ _update_client_latency (conn, frame, latency_usec);
+ call_notify = _gf_true;
+ unref = rpc_clnt_remove_ping_timer_locked (local->rpc);
+ if (__rpc_clnt_rearm_ping_timer (local->rpc,
rpc_clnt_start_ping) == -1) {
gf_log (this->name, GF_LOG_WARNING,
"failed to set the ping timer");
@@ -217,12 +261,24 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
}
unlock:
pthread_mutex_unlock (&conn->lock);
+
+ if (call_notify) {
+ ret = local->rpc->notifyfn (local->rpc, this,
+ RPC_CLNT_PING, NULL);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "RPC_CLNT_PING notify failed");
+ }
+ }
out:
if (unref)
- rpc_clnt_unref (rpc);
+ rpc_clnt_unref (local->rpc);
- if (frame)
+ if (frame) {
+ GF_FREE (frame->local);
+ frame->local = NULL;
STACK_DESTROY (frame->root);
+ }
return 0;
}
@@ -232,18 +288,27 @@ rpc_clnt_ping (struct rpc_clnt *rpc)
call_frame_t *frame = NULL;
int32_t ret = -1;
rpc_clnt_connection_t *conn = NULL;
+ struct ping_local *local = NULL;
conn = &rpc->conn;
+ local = GF_MALLOC (sizeof(struct ping_local), gf_common_ping_local_t);
+ if (!local)
+ return ret;
frame = create_frame (THIS, THIS->ctx->pool);
- if (!frame)
+ if (!frame) {
+ GF_FREE (local);
return ret;
+ }
- frame->local = rpc;
+ local->rpc = rpc;
+ timespec_now (&local->submit_time);
+ frame->local = local;
ret = rpc_clnt_submit (rpc, &clnt_ping_prog,
GF_DUMP_PING, rpc_clnt_ping_cbk, NULL, 0,
NULL, 0, NULL, frame, NULL, 0, NULL, 0, NULL);
if (ret) {
+ /* FIXME: should we free the frame here? Methinks so! */
gf_log (THIS->name, GF_LOG_ERROR,
"failed to start ping timer");
}
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 3a5b287cd49..2ccaa56e4cb 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -19,6 +19,7 @@
typedef enum {
RPC_CLNT_CONNECT,
RPC_CLNT_DISCONNECT,
+ RPC_CLNT_PING,
RPC_CLNT_MSG,
RPC_CLNT_DESTROY
} rpc_clnt_event_t;
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index e224dcc022e..5556740ca81 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -166,6 +166,19 @@ out:
+int rpc_transport_lib_path (char **name, char *type)
+{
+ int ret = -1;
+ char *libdir_env = getenv ("GLUSTER_LIBDIR");
+
+ ret = libdir_env == NULL
+ ? gf_asprintf (name, "%s/%s.so", RPC_TRANSPORTDIR, type)
+ : gf_asprintf (name, "%s/rpc-transport/%s.so", libdir_env, type);
+ return ret;
+}
+
+
+
rpc_transport_t *
rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
{
@@ -274,7 +287,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
goto fail;
}
- ret = gf_asprintf (&name, "%s/%s.so", RPC_TRANSPORTDIR, type);
+ ret = rpc_transport_lib_path (&name, type);
if (-1 == ret) {
goto fail;
}
@@ -652,18 +665,37 @@ out:
return ret;
}
+/** @brief build a dictionary containing basic transport options.
+ *
+ * @param[out] options: will be set to a newly created dictionary on success.
+ * @param[in] hostname: desired target hostname.
+ * @param[in] port: desired target port.
+ * @param[in] addr_family (optional): desired address family. If NULL,
+ * default will be used.
+ *
+ * @returns zero on success.
+ */
int
rpc_transport_inet_options_build (dict_t **options, const char *hostname,
- int port)
+ int port, const char *addr_family)
{
dict_t *dict = NULL;
char *host = NULL;
int ret = -1;
+#ifdef IPV6_DEFAULT
+ const char *addr_family_default = "inet6";
+#else
+ const char *addr_family_default = "inet";
+#endif
GF_ASSERT (options);
GF_ASSERT (hostname);
GF_ASSERT (port >= 1024);
+ if (!addr_family) {
+ addr_family = addr_family_default;
+ }
+
dict = dict_new ();
if (!dict)
goto out;
@@ -688,6 +720,14 @@ rpc_transport_inet_options_build (dict_t **options, const char *hostname,
goto out;
}
+ ret = dict_set_str (dict, "transport.address-family",
+ (char *)addr_family);
+ if (ret) {
+ gf_log (THIS->name, GF_LOG_WARNING,
+ "failed to set address-family to %s", addr_family);
+ goto out;
+ }
+
ret = dict_set_str (dict, "transport-type", "socket");
if (ret) {
gf_log (THIS->name, GF_LOG_WARNING,
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index f0add065065..0f555462ea4 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -311,5 +311,6 @@ rpc_transport_unix_options_build (dict_t **options, char *filepath,
int frame_timeout);
int
-rpc_transport_inet_options_build (dict_t **options, const char *hostname, int port);
+rpc_transport_inet_options_build (dict_t **options, const char *hostname,
+ int port, const char *addr_family);
#endif /* __RPC_TRANSPORT_H__ */
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index f07e745a4b3..9dc3bff427c 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -37,6 +37,10 @@
#include <stdarg.h>
#include <stdio.h>
+#ifdef IPV6_DEFAULT
+#include <netconfig.h>
+#endif
+
#include "xdr-rpcclnt.h"
#include "glusterfs-acl.h"
@@ -1363,6 +1367,82 @@ rpcsvc_error_reply (rpcsvc_request_t *req)
return rpcsvc_submit_generic (req, &dummyvec, 0, NULL, 0, NULL);
}
+#ifdef IPV6_DEFAULT
+int
+rpcsvc_program_register_rpcbind6 (rpcsvc_program_t *newprog, uint32_t port)
+{
+ const int IP_BUF_LEN = 64;
+ char addr_buf[IP_BUF_LEN];
+
+ int err = 0;
+ bool_t success = 0;
+ struct netconfig *nc;
+ struct netbuf *nb;
+
+ if (!newprog) {
+ goto out;
+ }
+
+ nc = getnetconfigent ("tcp6");
+ if (!nc) {
+ err = -1;
+ goto out;
+ }
+
+
+ err = sprintf (addr_buf, "::.%d.%d", port >> 8 & 0xff,
+ port & 0xff);
+ if (err < 0) {
+ err = -1;
+ goto out;
+ }
+
+ nb = uaddr2taddr (nc, addr_buf);
+ if (!nb) {
+ err = -1;
+ goto out;
+ }
+
+ success = rpcb_set (newprog->prognum, newprog->progver, nc, nb);
+ if (!success) {
+ gf_log (GF_RPCSVC, GF_LOG_ERROR, "Could not register the IPv6"
+ " service with rpcbind");
+ }
+
+ err = 0;
+
+out:
+ return err;
+}
+
+int
+rpcsvc_program_unregister_rpcbind6 (rpcsvc_program_t *newprog)
+{
+ int err = 0;
+ bool_t success = 0;
+ struct netconfig *nc;
+
+ if (!newprog) {
+ goto out;
+ }
+
+ nc = getnetconfigent ("tcp6");
+ if (!nc) {
+ err = -1;
+ goto out;
+ }
+
+ success = rpcb_unset (newprog->prognum, newprog->progver, nc);
+ if (!success) {
+ gf_log (GF_RPCSVC, GF_LOG_ERROR, "Could not unregister the IPv6"
+ " service with rpcbind");
+ }
+
+ err = 0;
+out:
+ return err;
+}
+#endif
/* Register the program with the local portmapper service. */
int
@@ -1527,7 +1607,14 @@ rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *program)
" program failed");
goto out;
}
-
+#ifdef IPV6_DEFAULT
+ ret = rpcsvc_program_unregister_rpcbind6 (program);
+ if (ret == -1) {
+ gf_log (GF_RPCSVC, GF_LOG_ERROR, "rpcbind (ipv6)"
+ " unregistration of program failed");
+ goto out;
+ }
+#endif
pthread_mutex_lock (&svc->rpclock);
{
list_for_each_entry (prog, &svc->programs, program) {
diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h
index 02e467e68a7..1032df03b0e 100644
--- a/rpc/rpc-lib/src/rpcsvc.h
+++ b/rpc/rpc-lib/src/rpcsvc.h
@@ -437,6 +437,11 @@ rpcsvc_listener_destroy (rpcsvc_listener_t *listener);
extern int
rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port);
+#ifdef IPV6_DEFAULT
+extern int
+rpcsvc_program_register_rpcbind6 (rpcsvc_program_t *newprog, uint32_t port);
+#endif
+
extern int
rpcsvc_program_unregister_portmap (rpcsvc_program_t *newprog);
diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c
index 8003b1c87a0..b9d3269eb73 100644
--- a/rpc/rpc-transport/rdma/src/name.c
+++ b/rpc/rpc-transport/rdma/src/name.c
@@ -54,6 +54,10 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
struct sockaddr *sockaddr,
socklen_t sockaddr_len, uint32_t ceiling)
{
+#if defined(NO_PRIVPORT)
+ _assign_port(sockaddr, 0);
+ return rdma_bind_addr (cm_id, sockaddr);
+#else
int32_t ret = -1;
uint16_t port = ceiling - 1;
gf_boolean_t ports[GF_PORT_MAX];
@@ -100,6 +104,7 @@ loop:
}
return ret;
+#endif /* NO_PRIVPORT */
}
#if 0
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index 0e34dc211fe..cab4161c076 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -42,6 +42,10 @@ static int32_t
af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
socklen_t sockaddr_len, uint32_t ceiling)
{
+#if defined(NO_PRIVPORT)
+ _assign_port(sockaddr, 0);
+ return bind (fd, sockaddr, sockaddr_len);
+#else
int32_t ret = -1;
uint16_t port = ceiling - 1;
gf_boolean_t ports[GF_PORT_MAX];
@@ -88,6 +92,7 @@ loop:
}
return ret;
+#endif /* NO_PRIVPORT */
}
static int32_t
@@ -557,6 +562,14 @@ server_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
data_t *address_family_data = NULL;
int32_t ret = -1;
+#ifdef IPV6_DEFAULT
+ char *addr_family = "inet6";
+ sa_family_t default_family = AF_INET6;
+#else
+ char *addr_family = "inet";
+ sa_family_t default_family = AF_INET;
+#endif
+
GF_VALIDATE_OR_GOTO ("socket", sa_family, out);
address_family_data = dict_get (this->options,
@@ -581,8 +594,9 @@ server_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
}
} else {
gf_log (this->name, GF_LOG_DEBUG,
- "option address-family not specified, defaulting to inet");
- *sa_family = AF_INET;
+ "option address-family not specified, "
+ "defaulting to %s", addr_family);
+ *sa_family = default_family;
}
ret = 0;
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index ae551dcfae7..76609fbbc7a 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -55,7 +55,11 @@
/* TBD: do automake substitutions etc. (ick) to set these. */
#if !defined(DEFAULT_ETC_SSL)
# ifdef GF_LINUX_HOST_OS
+# ifdef GF_FBEXTRAS
+# define DEFAULT_ETC_SSL "/var/lib/glusterd/ssl"
+# else
# define DEFAULT_ETC_SSL "/etc/ssl"
+# endif
# endif
# ifdef GF_BSD_HOST_OS
# define DEFAULT_ETC_SSL "/etc/openssl"
@@ -866,7 +870,7 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
goto err;
}
#else
- if (family != AF_INET)
+ if (family != AF_INET && family != AF_INET6)
goto done;
ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle,
@@ -3009,6 +3013,21 @@ socket_connect (rpc_transport_t *this, int port)
}
}
+ /* Make sure we are not vulnerable to someone setting
+ * net.ipv6.bindv6only to 1 so that gluster services are
+ * avalable over IPv4 & IPv6.
+ */
+ int disable_v6only = 0;
+
+ if (setsockopt (priv->sock, IPPROTO_IPV6, IPV6_V6ONLY,
+ (void *)&disable_v6only,
+ sizeof (disable_v6only)) < 0) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Error disabling sockopt IPV6_V6ONLY: \"%s\"",
+ strerror (errno));
+ }
+
+
if (priv->nodelay && (sa_family != AF_UNIX)) {
ret = __socket_nodelay (priv->sock);
diff --git a/rpc/xdr/src/glusterfs-fops.x b/rpc/xdr/src/glusterfs-fops.x
index 8462dcc258a..5ec8109d828 100644
--- a/rpc/xdr/src/glusterfs-fops.x
+++ b/rpc/xdr/src/glusterfs-fops.x
@@ -84,6 +84,7 @@ enum glusterfs_event_t {
GF_EVENT_UPCALL,
GF_EVENT_SCRUB_STATUS,
GF_EVENT_SOME_CHILD_DOWN,
+ GF_EVENT_CHILD_PING,
GF_EVENT_MAXVAL
};
generated by cgit (git 2.34.1) at 2025-11-10 17:23:45 +0000