summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--configure.ac22
-rw-r--r--glusterfs.spec.in19
-rw-r--r--xlators/features/Makefile.am7
-rw-r--r--xlators/features/cloudsync/src/Makefile.am13
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am3
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am7
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am3
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am12
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h21
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c553
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h51
-rw-r--r--xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym1
-rw-r--r--xlators/features/cloudsync/src/cloudsync.c25
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-set.c22
14 files changed, 743 insertions, 16 deletions
diff --git a/configure.ac b/configure.ac
index fe25ec2bc02..c20309b232e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -182,6 +182,10 @@ AC_CONFIG_FILES([Makefile
xlators/features/cloudsync/src/Makefile
xlators/features/utime/Makefile
xlators/features/utime/src/Makefile
+ xlators/features/cloudsync/src/cloudsync-plugins/Makefile
+ xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile
+ xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile
+ xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile
xlators/playground/Makefile
xlators/playground/template/Makefile
xlators/playground/template/src/Makefile
@@ -970,6 +974,23 @@ else
fi
# end of xml-output
+dnl cloudsync section
+BUILD_CLOUDSYNC="no"
+AC_CHECK_LIB([curl], [curl_easy_setopt], [LIBCURL="-lcurl"])
+if test "x$LIBCURL" != "x";then
+ HAVE_LIBCURL="yes"
+fi
+AC_CHECK_HEADERS([openssl/hmac.h openssl/evp.h openssl/bio.h openssl/buffer.h], [HAVE_OPENSSL="yes"])
+if test "x$HAVE_LIBCURL" = "xyes" -a "x$HAVE_OPENSSL" = "xyes";then
+ HAVE_AMAZONS3="yes"
+fi
+AM_CONDITIONAL([BUILD_AMAZONS3_PLUGIN], [test "x$HAVE_AMAZONS3" = "xyes"])
+if test "x$HAVE_AMAZONS3" = "xyes";then
+ BUILD_CLOUDSYNC="yes"
+fi
+AM_CONDITIONAL([BUILD_CLOUDSYNC], [test "x$BUILD_CLOUDSYNC" = "xyes"])
+dnl end cloudsync section
+
dnl SELinux feature enablement
case $host_os in
linux*)
@@ -1718,4 +1739,5 @@ echo "Legacy gNFS server : $BUILD_GNFS"
echo "IPV6 default : $with_ipv6_default"
echo "Use TIRPC : $with_libtirpc"
echo "With Python : ${PYTHON_VERSION}"
+echo "Cloudsync : $BUILD_CLOUDSYNC"
echo
diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index a749ea37534..7f83c686e27 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -431,6 +431,21 @@ is in user space and easily manageable.
This package provides support to FUSE based clients and inlcudes the
glusterfs(d) binary.
+%package cloudsync-plugins
+Summary: Cloudsync Plugins
+BuildRequires: libcurl-devel
+
+%description cloudsync-plugins
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility. It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
+
+This package provides cloudsync plugins for archival feature.
+
%if ( 0%{!?_without_georeplication:1} )
%package geo-replication
Summary: GlusterFS Geo-replication
@@ -1201,6 +1216,10 @@ exit 0
%endif
%endif
+%files cloudsync-plugins
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins
+ %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsyncs3.so
+
%if ( 0%{?_with_gnfs:1} && 0%{!?_without_server:1} )
%files gnfs
%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 81fbe856e78..6ef19af0860 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,6 +1,11 @@
+if BUILD_CLOUDSYNC
+ CLOUDSYNC_DIR = cloudsync
+endif
+
+
SUBDIRS = locks quota read-only quiesce marker index barrier \
arbiter compress changelog changetimerecorder \
gfid-access $(GLUPY_SUBDIR) upcall snapview-client snapview-server \
- trash shard bit-rot leases selinux sdfs namespace cloudsync thin-arbiter \
+ trash shard bit-rot leases selinux sdfs namespace $(CLOUDSYNC_DIR) thin-arbiter \
utime
CLEANFILES =
diff --git a/xlators/features/cloudsync/src/Makefile.am b/xlators/features/cloudsync/src/Makefile.am
index 4e8319c9d34..0f3381f1d7d 100644
--- a/xlators/features/cloudsync/src/Makefile.am
+++ b/xlators/features/cloudsync/src/Makefile.am
@@ -1,3 +1,5 @@
+SUBDIRS = cloudsync-plugins
+
xlator_LTLIBRARIES = cloudsync.la
xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
@@ -10,9 +12,9 @@ CLOUDSYNC_BLD = $(top_builddir)/xlators/features/cloudsync/src
cloudsynccommon_sources = $(CLOUDSYNC_SRC)/cloudsync-common.c
noinst_HEADERS = $(CLOUDSYNC_BLD)/cloudsync.h \
- $(CLOUDSYNC_BLD)/cloudsync-mem-types.h \
- $(CLOUDSYNC_BLD)/cloudsync-messages.h \
- $(CLOUDSYNC_BLD)/cloudsync-common.h
+ $(CLOUDSYNC_BLD)/cloudsync-mem-types.h \
+ $(CLOUDSYNC_BLD)/cloudsync-messages.h \
+ $(CLOUDSYNC_BLD)/cloudsync-common.h
cloudsync_la_SOURCES = $(cloudsync_sources) $(cloudsynccommon_sources)
@@ -23,8 +25,8 @@ cloudsync_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
-
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+ -DCS_PLUGINDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins\"
AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
noinst_PYTHON = cloudsync-fops-c.py cloudsync-fops-h.py
@@ -42,4 +44,3 @@ CLEANFILES = $(nodist_cloudsync_la_SOURCES)
uninstall-local:
rm -f $(DESTDIR)$(xlatordir)/cloudsync.so
-
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
new file mode 100644
index 00000000000..4deefb651eb
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
@@ -0,0 +1,7 @@
+if BUILD_AMAZONS3_PLUGIN
+ AMAZONS3_DIR = cloudsyncs3
+endif
+
+SUBDIRS = ${AMAZONS3_DIR}
+
+CLEANFILES =
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
new file mode 100644
index 00000000000..93fb2eecbf5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsyncs3.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsyncs3_la_SOURCES = libcloudsyncs3.c $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsyncs3_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsyncs3_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src -lcurlpp -lcryptopp
+noinst_HEADERS = libcloudsyncs3.h libcloudsyncs3-mem-types.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -lcurl -lcrypto -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsyncs3.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
new file mode 100644
index 00000000000..dd9314ec8d8
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+
+#ifndef __LIBAWS_MEM_TYPES_H__
+#define __LIBAWS_MEM_TYPES_H__
+
+#include "mem-types.h"
+enum libaws_mem_types_ {
+ gf_libaws_mt_aws_private_t = gf_common_mt_end + 1,
+ gf_libaws_mt_end
+};
+#endif /* __CLOUDSYNC_MEM_TYPES_H__ */
+
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
new file mode 100644
index 00000000000..019f98017dc
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
@@ -0,0 +1,553 @@
+/*
+ Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#include <stdlib.h>
+#include <openssl/hmac.h>
+#include <openssl/evp.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#include <openssl/crypto.h>
+#include <curl/curl.h>
+#include "xlator.h"
+#include "glusterfs.h"
+#include "libcloudsyncs3.h"
+#include "cloudsync-common.h"
+
+#define RESOURCE_SIZE 4096
+
+store_methods_t store_ops = {
+ .fop_download = aws_download_s3,
+ .fop_init = aws_init,
+ .fop_reconfigure = aws_reconfigure,
+ .fop_fini = aws_fini,
+};
+
+typedef struct aws_private {
+ char *hostname;
+ char *bucketid;
+ char *awssekey;
+ char *awskeyid;
+ gf_boolean_t abortdl;
+ pthread_spinlock_t lock;
+} aws_private_t;
+
+void *
+aws_init (xlator_t *this)
+{
+ aws_private_t *priv = NULL;
+ char *temp_str = NULL;
+ int ret = 0;
+
+ priv = GF_CALLOC (1, sizeof (aws_private_t),
+ gf_libaws_mt_aws_private_t);
+ if (!priv) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+ ret = -1;
+ goto out;
+ }
+
+ priv->abortdl = _gf_false;
+
+ pthread_spin_init (&priv->lock, PTHREAD_PROCESS_PRIVATE);
+
+ pthread_spin_lock (&(priv->lock));
+ {
+ if (dict_get_str (this->options, "s3plugin-seckey",
+ &temp_str) == 0) {
+ priv->awssekey = gf_strdup (temp_str);
+ if (!priv->awssekey) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws secret key failed");
+ ret = -1;
+ goto unlock;
+ }
+ }
+
+ if (dict_get_str (this->options, "s3plugin-keyid",
+ &temp_str) == 0) {
+ priv->awskeyid = gf_strdup (temp_str);
+ if (!priv->awskeyid) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws key ID failed");
+ ret = -1;
+ goto unlock;
+ }
+ }
+
+ if (dict_get_str (this->options, "s3plugin-bucketid",
+ &temp_str) == 0) {
+ priv->bucketid = gf_strdup (temp_str);
+ if (!priv->bucketid) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws bucketid failed");
+
+ ret = -1;
+ goto unlock;
+ }
+ }
+
+ if (dict_get_str (this->options, "s3plugin-hostname",
+ &temp_str) == 0) {
+ priv->hostname = gf_strdup (temp_str);
+ if (!priv->hostname) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws hostname failed");
+
+ ret = -1;
+ goto unlock;
+ }
+ }
+
+ gf_msg_debug (this->name, 0, "stored key: %s id: %s "
+ "bucketid %s hostname: %s", priv->awssekey,
+ priv->awskeyid, priv->bucketid, priv->hostname);
+
+ }
+unlock:
+ pthread_spin_unlock (&(priv->lock));
+
+out:
+ if (ret == -1) {
+ GF_FREE (priv->awskeyid);
+ GF_FREE (priv->awssekey);
+ GF_FREE (priv->bucketid);
+ GF_FREE (priv->hostname);
+ GF_FREE (priv);
+ priv = NULL;
+ }
+
+ return (void *)priv;
+}
+
+int
+aws_reconfigure (xlator_t *this, dict_t *options)
+{
+ aws_private_t *priv = NULL;
+ char *temp_str = NULL;
+ int ret = 0;
+ cs_private_t *cspriv = NULL;
+
+ cspriv = this->private;
+
+ priv = cspriv->stores->config;
+
+ if (!priv) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "null priv");
+ ret = -1;
+ goto out;
+ }
+
+ pthread_spin_lock (&(priv->lock));
+ {
+ if (dict_get_str (options, "s3plugin-seckey",
+ &temp_str) == 0) {
+ priv->awssekey = gf_strdup (temp_str);
+ if (!priv->awssekey) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws secret key failed");
+ ret = -1;
+ goto out;
+ }
+ }
+
+ if (dict_get_str (options, "s3plugin-keyid",
+ &temp_str) == 0) {
+ priv->awskeyid = gf_strdup (temp_str);
+ if (!priv->awskeyid) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws key ID failed");
+ ret = -1;
+ goto out;
+ }
+ }
+
+ if (dict_get_str (options, "s3plugin-bucketid",
+ &temp_str) == 0) {
+ priv->bucketid = gf_strdup (temp_str);
+ if (!priv->bucketid) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws bucketid failed");
+ ret = -1;
+ goto out;
+ }
+ }
+
+ if (dict_get_str (options, "s3plugin-hostname",
+ &temp_str) == 0) {
+ priv->hostname = gf_strdup (temp_str);
+ if (!priv->hostname) {
+ gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0,
+ "initializing aws hostname failed");
+ ret = -1;
+ goto out;
+ }
+ }
+
+ }
+out:
+ pthread_spin_unlock (&(priv->lock));
+
+ gf_msg_debug (this->name, 0, "stored key: %s id: %s "
+ "bucketid %s hostname: %s", priv->awssekey,
+ priv->awskeyid, priv->bucketid, priv->hostname);
+
+ return ret;
+}
+
+void
+aws_fini (void *config)
+{
+ aws_private_t *priv = NULL;
+
+ priv = (aws_private_t *)priv;
+
+ if (priv) {
+ GF_FREE (priv->hostname);
+ GF_FREE (priv->bucketid);
+ GF_FREE (priv->awssekey);
+ GF_FREE (priv->awskeyid);
+
+ pthread_spin_destroy (&priv->lock);
+ GF_FREE (priv);
+ }
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+ int ret = -1;
+
+ GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+ ret = xlator_mem_acct_init (this, gf_libaws_mt_end + 1);
+
+ if (ret != 0) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+ "Memory accounting init failed");
+ return ret;
+ }
+out:
+ return ret;
+}
+char *
+aws_form_request (char *resource, char **date, char *reqtype, char *bucketid,
+ char *filepath)
+{
+ char httpdate[256];
+ time_t ctime;
+ struct tm *gtime = NULL;
+ char *sign_req = NULL;
+
+ ctime = time(NULL);
+ gtime = gmtime(&ctime);
+
+ memset (httpdate, 0, sizeof(httpdate));
+ strftime (httpdate, sizeof(httpdate), "%a, %d %b %Y %H:%M:%S +0000",
+ gtime);
+ *date = gf_strdup (httpdate);
+
+ memset (resource, 0, RESOURCE_SIZE);
+
+ snprintf(resource, RESOURCE_SIZE, "%s/%s", bucketid, filepath);
+
+ gf_msg_debug ("CS", 0, "resource %s", resource);
+
+ sign_req = GF_CALLOC (1, 256, gf_common_mt_char);
+
+ snprintf(sign_req, 256, "%s\n\n%s\n%s\n/%s",
+ reqtype,
+ "",
+ *date,
+ resource);
+
+ return sign_req;
+}
+
+char*
+aws_b64_encode(const unsigned char *input, int length)
+{
+ BIO *bio, *b64;
+ BUF_MEM *bptr;
+ char *buff = NULL;
+
+ b64 = BIO_new(BIO_f_base64());
+ bio = BIO_new(BIO_s_mem());
+ b64 = BIO_push(b64, bio);
+ BIO_write(b64, input, length);
+ BIO_flush(b64);
+ BIO_get_mem_ptr(b64, &bptr);
+
+ buff = GF_CALLOC(1, (bptr->length), gf_common_mt_char);
+ memcpy(buff, bptr->data, bptr->length - 1);
+ buff[bptr->length - 1] = 0;
+
+ BIO_free_all(b64);
+
+ return buff;
+}
+
+char *
+aws_sign_request(char *const str, char *awssekey)
+{
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+ HMAC_CTX ctx;
+#endif
+ HMAC_CTX *pctx = NULL;;
+
+ unsigned char md[256];
+ unsigned len;
+ char *base64 = NULL;
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+ HMAC_CTX_init (&ctx);
+ pctx = &ctx;
+#else
+ pctx = HMAC_CTX_new ();
+#endif
+ HMAC_Init_ex (pctx, awssekey, strlen(awssekey), EVP_sha1(), NULL);
+ HMAC_Update (pctx, (unsigned char *)str, strlen(str));
+ HMAC_Final (pctx, (unsigned char *)md, &len);
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+ HMAC_CTX_cleanup (pctx);
+#else
+ HMAC_CTX_free (pctx);
+#endif
+ base64 = aws_b64_encode(md, len);
+
+ return base64;
+}
+
+int
+aws_dlwritev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno, struct iatt *prebuf,
+ struct iatt *postbuf, dict_t *xdata)
+{
+ aws_private_t *priv = NULL;
+
+ if (op_ret == -1) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, op_errno, "write failed "
+ ". Aborting Download");
+
+ priv = this->private;
+ pthread_spin_lock (&(priv->lock));
+ {
+ priv->abortdl = _gf_true;
+ }
+ pthread_spin_unlock (&(priv->lock));
+ }
+
+ CS_STACK_DESTROY (frame);
+
+ return op_ret;
+}
+
+size_t
+aws_write_callback (void *dlbuf, size_t size, size_t nitems, void *mainframe)
+{
+ call_frame_t *frame = NULL;
+ fd_t *dlfd = NULL;
+ int ret = 0;
+ cs_local_t *local = NULL;
+ struct iovec iov = {0,};
+ struct iobref *iobref = NULL;
+ struct iobuf *iobuf = NULL;
+ struct iovec dliov = {0, };
+ size_t tsize = 0;
+ xlator_t *this = NULL;
+ cs_private_t *xl_priv = NULL;
+ aws_private_t *priv = NULL;
+ call_frame_t *dlframe = NULL;
+
+ frame = (call_frame_t *)mainframe;
+ this = frame->this;
+ xl_priv = this->private;
+ priv = xl_priv->stores->config;
+
+ pthread_spin_lock (&(priv->lock));
+ {
+ /* returning size other than the size passed from curl will
+ * abort further download*/
+ if (priv->abortdl) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+ "aborting download");
+ pthread_spin_unlock (&(priv->lock));
+ return 0;
+ }
+ }
+ pthread_spin_unlock (&(priv->lock));
+
+ local = frame->local;
+ dlfd = local->dlfd;
+ tsize = size * nitems;
+
+ dliov.iov_base = (void *)dlbuf;
+ dliov.iov_len = tsize;
+
+ ret = iobuf_copy (this->ctx->iobuf_pool, &dliov, 1, &iobref, &iobuf,
+ &iov);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "iobuf_copy failed");
+ goto out;
+ }
+
+ /* copy frame */
+ dlframe = copy_frame (frame);
+ if (!dlframe) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "copy_frame failed");
+ tsize = 0;
+ goto out;
+ }
+
+ STACK_WIND (dlframe, aws_dlwritev_cbk, FIRST_CHILD (this),
+ FIRST_CHILD (this)->fops->writev, dlfd,
+ &iov, 1, local->dloffset, 0, iobref, NULL);
+
+ local->dloffset += tsize;
+
+out:
+ if (iobuf)
+ iobuf_unref (iobuf);
+ if (iobref)
+ iobref_unref (iobref);
+
+ return tsize;
+}
+
+int
+aws_download_s3 (call_frame_t *frame, void *config)
+{
+ char buf[1024];
+ CURL *handle = NULL;
+ struct curl_slist *slist = NULL;
+ struct curl_slist *tmp = NULL;
+ xlator_t *this = NULL;
+ int ret = 0;
+ int debug = 1;
+ CURLcode res;
+ char errbuf[CURL_ERROR_SIZE];
+ size_t len = 0;
+ long responsecode;
+ char *sign_req = NULL;
+ char *date = NULL;
+ char *const reqtype = "GET";
+ char *signature = NULL;
+ cs_local_t *local = NULL;
+ char resource[4096] = {0,};
+ aws_private_t *priv = NULL;
+
+ local = frame->local;
+
+ priv = (aws_private_t *)config;
+
+ if (!priv->bucketid || !priv->hostname || !priv->awssekey ||
+ !priv->awskeyid) {
+ ret = -1;
+ goto out;
+ }
+
+ sign_req = aws_form_request (resource, &date, reqtype, priv->bucketid,
+ local->remotepath);
+ if (!sign_req) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "null sign_req, "
+ "aborting download");
+ ret = -1;
+ goto out;
+ }
+
+ gf_msg_debug ("CS", 0, "sign_req %s date %s", sign_req, date);
+
+ signature = aws_sign_request (sign_req, priv->awssekey);
+ if (!signature) {
+ gf_msg ("CS", GF_LOG_ERROR, 0, 0, "null signature, "
+ "aborting download");
+ ret = -1;
+ goto out;
+ }
+
+ handle = curl_easy_init();
+ this = frame->this;
+
+ snprintf (buf, 1024, "Date: %s", date);
+ slist = curl_slist_append(slist, buf);
+ snprintf (buf, sizeof(buf), "Authorization: AWS %s:%s", priv->awskeyid,
+ signature);
+ slist = curl_slist_append(slist, buf);
+ snprintf(buf, sizeof(buf), "https://%s/%s", priv->hostname, resource);
+
+ if (gf_log_get_loglevel () >= GF_LOG_DEBUG) {
+ tmp = slist;
+ while (tmp) {
+ gf_msg_debug (this->name, 0, "slist for curl - %s",
+ tmp->data);
+ tmp = tmp->next;
+ }
+ }
+
+ curl_easy_setopt (handle, CURLOPT_HTTPHEADER, slist);
+ curl_easy_setopt (handle, CURLOPT_URL, buf);
+ curl_easy_setopt (handle, CURLOPT_WRITEFUNCTION, aws_write_callback);
+ curl_easy_setopt (handle, CURLOPT_WRITEDATA, frame);
+ curl_easy_setopt (handle, CURLOPT_VERBOSE, debug);
+ curl_easy_setopt (handle, CURLOPT_ERRORBUFFER, errbuf);
+
+ res = curl_easy_perform(handle);
+ if (res != CURLE_OK) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+ "download failed. err: %s\n", curl_easy_strerror(res));
+ ret = -1;
+ len = strlen(errbuf);
+ if (len) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+ "curl failure %s", errbuf);
+ } else {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "curl error "
+ "%s\n", curl_easy_strerror(res));
+ }
+ }
+
+ if (res == CURLE_OK) {
+ curl_easy_getinfo (handle, CURLINFO_RESPONSE_CODE,
+ &responsecode);
+ gf_msg_debug (this->name, 0, "response code %ld", responsecode);
+ if (responsecode != 200) {
+ ret = -1;
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+ "curl download failed");
+ }
+ }
+
+ curl_slist_free_all(slist);
+ curl_easy_cleanup(handle);
+
+out:
+ return ret;
+}
+
+struct volume_options cs_options[] = {
+ { .key = {"s3plugin-seckey"},
+ .type = GF_OPTION_TYPE_STR,
+ .description = "aws secret key"
+ },
+ { .key = {"s3plugin-keyid"},
+ .type = GF_OPTION_TYPE_STR,
+ .description = "aws key ID"
+
+ },
+ { .key = {"s3plugin-bucketid"},
+ .type = GF_OPTION_TYPE_STR,
+ .description = "aws bucketid"
+ },
+ { .key = {"s3plugin-hostname"},
+ .type = GF_OPTION_TYPE_STR,
+ .description = "aws hostname e.g. s3.amazonaws.com"
+ },
+ { .key = {NULL} },
+};
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
new file mode 100644
index 00000000000..c233e1c96f7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
@@ -0,0 +1,51 @@
+/*
+ Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBAWS_H
+#define _LIBAWS_H
+
+#include "glusterfs.h"
+#include "call-stub.h"
+#include "xlator.h"
+#include "syncop.h"
+#include <curl/curl.h>
+#include "cloudsync-common.h"
+#include "libcloudsyncs3-mem-types.h"
+
+
+char*
+aws_b64_encode(const unsigned char *input, int length);
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe);
+
+int
+aws_download_s3 (call_frame_t *frame, void *config);
+
+int
+aws_dlwritev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno, struct iatt *prebuf,
+ struct iatt *postbuf, dict_t *xdata);
+
+void *
+aws_init (xlator_t *this);
+
+int
+aws_reconfigure (xlator_t *this, dict_t *options);
+
+char *
+aws_form_request (char *resource, char **date, char *reqtype, char *bucketid,
+ char *filepath);
+char *
+aws_sign_request(char *const str, char *awssekey);
+
+void
+aws_fini (void *config);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/features/cloudsync/src/cloudsync.c b/xlators/features/cloudsync/src/cloudsync.c
index e14fc544957..e32181c22ab 100644
--- a/xlators/features/cloudsync/src/cloudsync.c
+++ b/xlators/features/cloudsync/src/cloudsync.c
@@ -36,9 +36,9 @@ cs_cleanup_private (cs_private_t *priv)
struct cs_plugin plugins[] = {
{
- .name = "amazons3",
- .library = "libamazons3.so",
- .description = "amazon s3 store."
+ .name = "cloudsyncs3",
+ .library = "cloudsyncs3.so",
+ .description = "cloudsync s3 store."
},
{.name = NULL},
@@ -55,6 +55,7 @@ cs_init (xlator_t *this)
void *handle = NULL;
char *temp_str = NULL;
int index = 0;
+ char *libname = NULL;
priv = GF_CALLOC (1, sizeof (*priv), gf_cs_mt_cs_private_t);
if (!priv) {
@@ -82,7 +83,7 @@ cs_init (xlator_t *this)
&temp_str) == 0) {
for (index = 0; plugins[index].name; index++) {
if (!strcmp (temp_str, plugins[index].name)) {
- libpath = plugins[index].library;
+ libname = plugins[index].library;
break;
}
}
@@ -90,21 +91,27 @@ cs_init (xlator_t *this)
ret = 0;
}
- if (!libpath) {
+ if (!libname) {
gf_msg (this->name, GF_LOG_WARNING, 0, 0,
"no plugin enabled");
ret = 0;
goto out;
}
+ ret = gf_asprintf (&libpath, "%s/%s", CS_PLUGINDIR, libname);
+ if (ret == -1) {
+ goto out;
+ }
+
handle = dlopen (libpath, RTLD_NOW);
if (!handle) {
- gf_msg (this->name, GF_LOG_ERROR, 0, 0, "could not load"
- " the required library. %s", dlerror ());
+ gf_msg (this->name, GF_LOG_WARNING, 0, 0, "could not "
+ "load the required library. %s", dlerror ());
+ ret = 0;
goto out;
} else {
gf_msg (this->name, GF_LOG_INFO, 0, 0,
- "loading library:%s successful", libpath);
+ "loading library:%s successful", libname);
}
@@ -166,6 +173,8 @@ cs_init (xlator_t *this)
}
}
+ ret = 0;
+
out:
if (ret == -1) {
if (this->local_pool)
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 8b2b1f80239..b2a3ceb2bed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -3822,7 +3822,27 @@ struct volopt_map_entry glusterd_volopt_map[] = {
},
{ .key = "feature.cloudsync-storetype",
.voltype = "features/cloudsync",
- .op_version = GD_OP_VERSION_4_1_0,
+ .op_version = GD_OP_VERSION_4_2_0,
+ .flags = VOLOPT_FLAG_CLIENT_OPT
+ },
+ { .key = "features.s3plugin-seckey",
+ .voltype = "features/cloudsync",
+ .op_version = GD_OP_VERSION_4_2_0,
+ .flags = VOLOPT_FLAG_CLIENT_OPT
+ },
+ { .key = "features.s3plugin-keyid",
+ .voltype = "features/cloudsync",
+ .op_version = GD_OP_VERSION_4_2_0,
+ .flags = VOLOPT_FLAG_CLIENT_OPT
+ },
+ { .key = "features.s3plugin-bucketid",
+ .voltype = "features/cloudsync",
+ .op_version = GD_OP_VERSION_4_2_0,
+ .flags = VOLOPT_FLAG_CLIENT_OPT
+ },
+ { .key = "features.s3plugin-hostname",
+ .voltype = "features/cloudsync",
+ .op_version = GD_OP_VERSION_4_2_0,
.flags = VOLOPT_FLAG_CLIENT_OPT
},
{ .key = NULL