summaryrefslogtreecommitdiffstats
path: root/xlators/nfs/server/src/nfs.c
diff options
context:
space:
mode:
authorShehjar Tikoo <shehjart@gluster.com>2010-09-22 03:32:17 +0000
committerVijay Bellur <vijay@dev.gluster.com>2010-09-22 04:14:27 -0700
commitf47b0c55de9941823fbefe4b3a7e37179d6d4329 (patch)
treeab22e40749a8173a869bfc1567fb2ef816196f56 /xlators/nfs/server/src/nfs.c
parent6b2e2fb0903fc56105c267881f202a4865a8b6c7 (diff)
nfs, nfs3: Base volume access on CHILD-UP-DOWN event
Overall, the aim of this patch is to change the result of an nfs op depending on whether the underlying volume is up or down as notified by CHILD_UP and CHILD_DOWN events. This patch contains three intertwined changes: o Only when the lookup on the root of a volume is successful does gnfs now export the subvolume. Till now the result of the lookup was not used to determine whether we should export that volume. Not accounting for root lookup failure resulted in ESTALEs on first access because some children of distribute were down at the time of the root lookup. o Only when lookups on all the subvolumes have succeeded are these exports enabled through NFS. o When a child of say distribute goes down, on CHILD_DOWN event nfs will ignore all incoming requests from the client because ignoring these will prevent ESTALEs for those requests and in the hope that ignoring the requests will make the client retransmit. There are risks in this approach absent the DRC but we're willing to live with that for now. When a child goes down, the mount exports list will continue to show it but mount requests will be denied. Signed-off-by: Shehjar Tikoo <shehjart@gluster.com> Signed-off-by: Vijay Bellur <vijay@dev.gluster.com> BUG: 1643 (Initial requests after mount ESTALE if DHT subvolumes connect after nfs startup) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1643
Diffstat (limited to 'xlators/nfs/server/src/nfs.c')
-rw-r--r--xlators/nfs/server/src/nfs.c379
1 files changed, 296 insertions, 83 deletions
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 431765332..3fa46662f 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -169,7 +169,7 @@ ret:
int
-nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
+__is_nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
{
int x = 0;
int started = 0;
@@ -177,16 +177,32 @@ nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
if ((!nfs) || (!xl))
return 1;
+ for (;x < nfs->allsubvols; ++x) {
+ if (!((&nfs->subvols[x])->subvol == xl))
+ continue;
+
+ if (gf_nfs_subvolume_lookupsent (&nfs->subvols[x])) {
+ started = 1;
+ goto out;
+ }
+ }
+out:
+ return started;
+}
+
+
+int
+is_nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
+{
+ int started = 0;
+
+ if ((!nfs) || (!xl))
+ return 1;
+
LOCK (&nfs->svinitlock);
{
- for (;x < nfs->allsubvols; ++x) {
- if (nfs->initedxl[x] == xl) {
- started = 1;
- goto unlock;
- }
- }
+ started = __is_nfs_subvolume_lookup_sent (nfs, xl);
}
-unlock:
UNLOCK (&nfs->svinitlock);
return started;
@@ -194,27 +210,187 @@ unlock:
int
-nfs_subvolume_set_started (struct nfs_state *nfs, xlator_t *xl)
+__is_nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
{
int x = 0;
+ int started = 0;
+
+ if ((!nfs) || (!xl))
+ return 1;
+
+ for (;x < nfs->allsubvols; ++x) {
+ if (!((&nfs->subvols[x])->subvol == xl))
+ continue;
+
+ if (gf_nfs_subvolume_started (&nfs->subvols[x])) {
+ started = 1;
+ goto out;
+ }
+ }
+out:
+ return started;
+}
+
+
+int
+is_nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
+{
+ int started = 0;
if ((!nfs) || (!xl))
return 1;
LOCK (&nfs->svinitlock);
{
- for (;x < nfs->allsubvols; ++x) {
- if (nfs->initedxl[x] == NULL) {
- nfs->initedxl[x] = xl;
- ++nfs->upsubvols;
- gf_log (GF_NFS, GF_LOG_DEBUG, "Starting up: %s "
- ", vols started till now: %d", xl->name,
- nfs->upsubvols);
- goto unlock;
- }
- }
+ started = __is_nfs_subvolume_started (nfs, xl);
+ }
+ UNLOCK (&nfs->svinitlock);
+
+ return started;
+}
+
+
+int
+__nfs_subvolume_lookup_again_later (struct nfs_state *nfs, xlator_t *xl)
+{
+ int x = 0;
+ struct nfs_subvolume *sv = NULL;
+
+ if ((!nfs) || (!xl))
+ return -1;
+
+ for (;x < nfs->allsubvols; ++x) {
+ if (((&nfs->subvols[x])->subvol == xl)) {
+ sv = &nfs->subvols[x];
+ break;
+ }
+ }
+
+ if (!sv)
+ goto err;
+ gf_log (GF_NFS, GF_LOG_TRACE, "Will lookup subvol later: %s",
+ xl->name);
+ sv->status = GF_NFS_SUBVOLUME_NOTSTARTED;
+err:
+ return 0;
+}
+
+
+int
+__nfs_subvolume_start (struct nfs_state *nfs, xlator_t *xl)
+{
+ int x = 0;
+ int ret = 0;
+ struct nfs_subvolume *sv = NULL;
+
+ if ((!nfs) || (!xl))
+ return -1;
+
+ for (;x < nfs->allsubvols; ++x) {
+ if (((&nfs->subvols[x])->subvol == xl)) {
+ sv = &nfs->subvols[x];
+ break;
+ }
+ }
+
+ if (!sv) {
+ gf_log (GF_NFS, GF_LOG_ERROR, "Could not find subvol slot");
+ goto err;
+ }
+
+ /* If the subvolume got disconnected or sent a CHILD-DOWN between the
+ * lookup fop and the lookup callback.
+ */
+ if (gf_nfs_subvolume_notstarted (sv)) {
+ goto err;
+ }
+
+ sv->status = GF_NFS_SUBVOLUME_STARTED;
+ ++nfs->upsubvols;
+ gf_log (GF_NFS, GF_LOG_DEBUG, "Starting up: %s, vols started till now: "
+ "%d", xl->name, nfs->upsubvols);
+
+ if (gf_nfs_all_subvolumes_started (nfs)) {
+ nfs->subvols_started = 1;
+ gf_log (GF_NFS, GF_LOG_INFO, "All exports up");
+ ret = nfs_init_versions (nfs, nfs->nfsx);
+ if (ret == -1)
+ gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to initialize "
+ "protocols");
+ }
+err:
+ return 0;
+}
+
+
+int
+nfs_subvolume_start (struct nfs_state *nfs, xlator_t *xl)
+{
+ if ((!nfs) || (!xl))
+ return 1;
+
+ LOCK (&nfs->svinitlock);
+ {
+ __nfs_subvolume_start (nfs, xl);
+ }
+ UNLOCK (&nfs->svinitlock);
+
+ return 0;
+}
+
+
+int
+__nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
+{
+ struct nfs_subvolume *sv = NULL;
+ int x = 0;
+
+ if ((!nfs) || (!xl))
+ return -1;
+
+ for (;x < nfs->allsubvols; ++x) {
+ if (((&nfs->subvols[x])->subvol == xl)) {
+ sv = &nfs->subvols[x];
+ break;
+ }
+ }
+
+ if (sv) {
+ sv->status = GF_NFS_SUBVOLUME_LOOKUPSENT;
+ } else
+ gf_log (GF_NFS, GF_LOG_ERROR, "Could not find subvol slot: %s",
+ xl->name);
+
+ return 0;
+}
+
+
+int
+nfs_subvolume_lookup_sent (struct nfs_state *nfs, xlator_t *xl)
+{
+ if ((!nfs) || (!xl))
+ return 1;
+
+ LOCK (&nfs->svinitlock);
+ {
+ __nfs_subvolume_lookup_sent (nfs, xl);
+ }
+ UNLOCK (&nfs->svinitlock);
+
+ return 0;
+}
+
+
+int
+nfs_subvolume_lookup_again_later (struct nfs_state *nfs, xlator_t *xl)
+{
+ if ((!nfs) || (!xl))
+ return 1;
+
+ LOCK (&nfs->svinitlock);
+ {
+ __nfs_subvolume_lookup_again_later (nfs, xl);
}
-unlock:
UNLOCK (&nfs->svinitlock);
return 0;
@@ -227,36 +403,71 @@ nfs_start_subvol_lookup_cbk (call_frame_t *frame, void *cookie,
inode_t *inode, struct iatt *buf, dict_t *xattr,
struct iatt *postparent)
{
+ struct nfs_state *nfs = frame->local;
+
if (op_ret == -1) {
gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to lookup root: %s",
strerror (op_errno));
+ nfs_subvolume_lookup_again_later (nfs, (xlator_t *)cookie);
goto err;
}
- gf_log (GF_NFS, GF_LOG_TRACE, "Started %s", ((xlator_t *)cookie)->name);
+ nfs_subvolume_start (nfs, (xlator_t *)cookie);
err:
return 0;
}
int
+__nfs_startup_subvolume_check (struct nfs_state *nfs, xlator_t *xl)
+{
+ int ret = 0;
+
+ if (__is_nfs_subvolume_started (nfs, xl)) {
+ gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume already started: %s",
+ xl->name);
+ ret = 1;
+ goto out;
+ }
+
+ if (__is_nfs_subvolume_lookup_sent (nfs, xl)) {
+ gf_log (GF_NFS,GF_LOG_TRACE,"Subvolume lookup already sent: %s",
+ xl->name);
+ ret = 1;
+ goto out;
+ }
+
+out:
+ return ret;
+}
+
+
+int
nfs_startup_subvolume (xlator_t *nfsx, xlator_t *xl)
{
- int ret = -1;
- loc_t rootloc = {0, };
- nfs_user_t nfu = {0, };
+ int ret = -1;
+ loc_t rootloc = {0, };
+ nfs_user_t nfu = {0, };
+ struct nfs_state *nfs = NULL;
if ((!nfsx) || (!xl))
return -1;
- if (nfs_subvolume_started (nfsx->private, xl)) {
- gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume already started: %s",
- xl->name);
- ret = 0;
- goto err;
+ nfs = nfsx->private;
+ LOCK (&nfs->svinitlock);
+ {
+ ret = __nfs_startup_subvolume_check (nfsx->private, xl);
+ if (ret)
+ goto unlock;
+
+ __nfs_subvolume_lookup_sent (nfs, xl);
}
+unlock:
+ UNLOCK (&nfs->svinitlock);
+
+ if (ret)
+ goto err;
- nfs_subvolume_set_started (nfsx->private, xl);
ret = nfs_inode_loc_fill (xl->itable->root, &rootloc);
if (ret == -1) {
gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init root loc");
@@ -279,35 +490,6 @@ err:
return ret;
}
-int
-nfs_startup_subvolumes (xlator_t *nfsx)
-{
- int ret = -1;
- xlator_list_t *cl = NULL;
- struct nfs_state *nfs = NULL;
-
- if (!nfsx)
- return -1;
-
- nfs = nfsx->private;
- cl = nfs->subvols;
- while (cl) {
- gf_log (GF_NFS, GF_LOG_DEBUG, "Starting subvolume: %s",
- cl->xlator->name);
- ret = nfs_startup_subvolume (nfsx, cl->xlator);
- if (ret == -1) {
- gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to start-up "
- "xlator: %s", cl->xlator->name);
- goto err;
- }
- cl = cl->next;
- }
-
- ret = 0;
-err:
- return ret;
-}
-
int
nfs_init_subvolume (struct nfs_state *nfs, xlator_t *xl)
@@ -333,17 +515,19 @@ err:
int
nfs_init_subvolumes (struct nfs_state *nfs, xlator_list_t *cl)
{
- int ret = -1;
- unsigned int lrusize = 0;
- int svcount = 0;
+ int ret = -1;
+ unsigned int lrusize = 0;
+ int svcount = 0;
+ int x = 0;
+ xlator_list_t *tmpcl = NULL;
+ struct nfs_subvolume *sv = NULL;
if ((!nfs) || (!cl))
return -1;
lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
- nfs->subvols = cl;
gf_log (GF_NFS, GF_LOG_TRACE, "inode table lru: %d", lrusize);
-
+ tmpcl = cl;
while (cl) {
gf_log (GF_NFS, GF_LOG_DEBUG, "Initing subvolume: %s",
cl->xlator->name);
@@ -358,16 +542,23 @@ nfs_init_subvolumes (struct nfs_state *nfs, xlator_list_t *cl)
}
LOCK_INIT (&nfs->svinitlock);
- nfs->initedxl = GF_CALLOC (svcount, sizeof (xlator_t *),
- gf_nfs_mt_xlator_t );
- if (!nfs->initedxl) {
- gf_log (GF_NFS, GF_LOG_ERROR, "Failed to allocated inited xls");
+ nfs->subvols = GF_CALLOC (svcount, sizeof (struct nfs_subvolume),
+ gf_nfs_mt_subvolumes);
+ if (!nfs->subvols) {
+ gf_log (GF_NFS, GF_LOG_ERROR, "Failed to allocate subvol list");
ret = -1;
goto err;
}
- gf_log (GF_NFS, GF_LOG_TRACE, "Inited volumes: %d", svcount);
+ cl = tmpcl;
nfs->allsubvols = svcount;
+ for (;x < nfs->allsubvols; x++) {
+ sv = &nfs->subvols[x];
+ sv->subvol = cl->xlator;
+ sv->status = GF_NFS_SUBVOLUME_NOTSTARTED;
+ }
+
+ gf_log (GF_NFS, GF_LOG_TRACE, "inited volumes: %d", svcount);
ret = 0;
err:
return ret;
@@ -482,6 +673,7 @@ nfs_init_state (xlator_t *this)
return NULL;
}
+ nfs->nfsx = this;
/* RPC service needs to be started before NFS versions can be
* inited. */
nfs->rpcsvc = nfs_rpcsvc_init (this->ctx, this->options);
@@ -580,33 +772,48 @@ err:
int
+nfs_disable_subvolume (xlator_t *nfsx, xlator_t *xl)
+{
+ struct nfs_state *nfs = NULL;
+
+ if ((!nfsx) || (!xl))
+ return -1;
+
+ nfs = nfsx->private;
+ gf_log (GF_NFS, GF_LOG_TRACE, "Disabling subvolume: %s", xl->name);
+ LOCK (&nfs->svinitlock);
+ {
+ if (!__is_nfs_subvolume_started (nfsx->private, xl)) {
+ gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume not started, "
+ "no point disabling: %s", xl->name);
+ goto unlock;
+ }
+ --nfs->upsubvols;
+ __nfs_subvolume_lookup_again_later (nfs, xl);
+ }
+unlock:
+ UNLOCK (&nfs->svinitlock);
+
+ return 0;
+}
+
+
+int
notify (xlator_t *this, int32_t event, void *data, ...)
{
struct nfs_state *nfs = NULL;
xlator_t *subvol = NULL;
- int ret = -1;
nfs = (struct nfs_state *)this->private;
subvol = (xlator_t *)data;
- gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %d",
- event);
+ gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %s",
+ glusterfs_strevent (event));
switch (event)
{
case GF_EVENT_CHILD_UP:
{
nfs_startup_subvolume (this, subvol);
- if ((nfs->upsubvols == nfs->allsubvols) &&
- (!nfs->subvols_started)) {
- nfs->subvols_started = 1;
- gf_log (GF_NFS, GF_LOG_TRACE, "All children up,"
- " starting RPC");
- ret = nfs_init_versions (nfs, this);
- if (ret == -1)
- gf_log (GF_NFS, GF_LOG_CRITICAL,
- "Failed to initialize "
- "protocols");
- }
break;
}
@@ -615,6 +822,12 @@ notify (xlator_t *this, int32_t event, void *data, ...)
default_notify (this, GF_EVENT_PARENT_UP, data);
break;
}
+
+ case GF_EVENT_CHILD_DOWN:
+ {
+ nfs_disable_subvolume (this, subvol);
+ break;
+ }
}
return 0;