Introduced DiscoveredServerService, and fetching discovered server details in parallel through multiple threads to improve performance.

5 files changed, 280 insertions, 206 deletions

diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/DiscoveredServersResource.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/DiscoveredServersResource.java
index 5a038670..f47675d8 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/DiscoveredServersResource.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/DiscoveredServersResource.java
@@ -22,7 +22,6 @@ import static com.gluster.storage.management.core.constants.RESTConstants.PATH_P
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DETAILS;
 import static com.gluster.storage.management.core.constants.RESTConstants.RESOURCE_PATH_DISCOVERED_SERVERS;
 
-import java.util.ArrayList;
 import java.util.List;
 
 import javax.ws.rs.GET;
@@ -33,14 +32,12 @@ import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
-import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
 import com.gluster.storage.management.core.model.Server;
 import com.gluster.storage.management.core.response.ServerListResponse;
 import com.gluster.storage.management.core.response.ServerNameListResponse;
-import com.gluster.storage.management.gateway.utils.GlusterUtil;
-import com.gluster.storage.management.gateway.utils.ServerUtil;
+import com.gluster.storage.management.gateway.services.DiscoveredServerService;
 import com.sun.jersey.api.core.InjectParam;
 import com.sun.jersey.spi.resource.Singleton;
 
@@ -49,33 +46,8 @@ import com.sun.jersey.spi.resource.Singleton;
 @Path(RESOURCE_PATH_DISCOVERED_SERVERS)
 public class DiscoveredServersResource extends AbstractResource {
 	@InjectParam
-	protected ServerUtil serverUtil;
+	private DiscoveredServerService discoveredServerService;
 	
-	@InjectParam
-	protected GlusterUtil glusterUtil;
-	
-	private List<String> discoveredServerNames = new ArrayList<String>();
-	
-	private static final Logger logger = Logger.getLogger(DiscoveredServersResource.class);
-	
-	public List<String> getDiscoveredServerNames() {
-		return discoveredServerNames;
-	}
-	
-	public void setDiscoveredServerNames(List<String> discoveredServerNames) {
-		synchronized (discoveredServerNames) {
-			this.discoveredServerNames = discoveredServerNames;
-		}
-	}
-	
-	public void removeDiscoveredServer(String serverName) {
-		discoveredServerNames.remove(serverName);
-	}
-	
-	public void addDiscoveredServer(String serverName) {
-		discoveredServerNames.add(serverName);
-	}
-
 	@GET
 	@Produces(MediaType.APPLICATION_XML)
 	public Response getDiscoveredServersXML(@QueryParam(QUERY_PARAM_DETAILS) Boolean details) {
@@ -91,29 +63,16 @@ public class DiscoveredServersResource extends AbstractResource {
 	private Response getDiscoveredServersResponse(Boolean details, String mediaType) {
 		if(details != null && details == true) {
 			try {
-				List<Server> discoveredServers = getDiscoveredServerDetails();
+				List<Server> discoveredServers = discoveredServerService.getDiscoveredServerDetails();
 				return okResponse(new ServerListResponse(discoveredServers), mediaType);
 			} catch(Exception e) {
 				return errorResponse(e.getMessage());
 			}
 		} else {		
-			return okResponse(new ServerNameListResponse(getDiscoveredServerNames()), mediaType);
+			return okResponse(new ServerNameListResponse(discoveredServerService.getDiscoveredServerNames()), mediaType);
 		}
 	}
 
-	private List<Server> getDiscoveredServerDetails() {
-		List<Server> discoveredServers = new ArrayList<Server>();
-		for (String serverName : getDiscoveredServerNames()) {
-			try {
-				discoveredServers.add(getDiscoveredServer(serverName));
-			} catch(Exception e) {
-				logger.warn("Could not fetch details of discovered server [ " + serverName + "]", e);
-				// continue with next discovered server
-			}
-		}
-		return discoveredServers;
-	}
-	
 	@Path("{" + PATH_PARAM_SERVER_NAME + "}")
 	@GET
 	@Produces(MediaType.APPLICATION_XML)
@@ -133,21 +92,10 @@ public class DiscoveredServersResource extends AbstractResource {
 			return badRequestResponse("Server name must not be empty!");
 		}
 		try {
-			return okResponse(getDiscoveredServer(serverName), mediaType);
+			return okResponse(discoveredServerService.getDiscoveredServer(serverName), mediaType);
 		} catch (Exception e) {
 			// TODO: Log the exception
 			return errorResponse(e.getMessage());
 		}
 	}
-	
-	private Server getDiscoveredServer(String serverName) {
-		Server server = new Server(serverName);
-		serverUtil.fetchServerDetails(server);
-		return server;
-	}
-	
-	public static void main(String[] args) {
-		Response response = (Response)new DiscoveredServersResource().getDiscoveredServersXML(false);
-		System.out.println(response.getEntity());
-	}
 }
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java
index 89143129..e95a8858 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java
@@ -91,18 +91,12 @@ public class GlusterServersResource extends AbstractResource {
 	public static final String HOSTNAMETAG = "hostname:";
 
 	@InjectParam
-	private DiscoveredServersResource discoveredServersResource;
-
-	@InjectParam
 	private TasksResource taskResource;
 
 	@InjectParam
 	private ClusterService clusterService;
 
 	@InjectParam
-	private SshUtil sshUtil;
-	
-	@InjectParam
 	private CpuStatsFactory cpuStatsFactory;
 
 	@InjectParam
@@ -112,12 +106,6 @@ public class GlusterServersResource extends AbstractResource {
 	private NetworkStatsFactory networkStatsFactory;
 	
 	@InjectParam
-	private ServerUtil serverUtil;
-	
-	@InjectParam
-	private GlusterUtil glusterUtil;
-	
-	@InjectParam
 	private GlusterServerService glusterServerService;
 	
 	@GET
@@ -204,86 +192,10 @@ public class GlusterServersResource extends AbstractResource {
 		}
 	}
 
-	private void performAddServer(String clusterName, String serverName) {
-		GlusterServer onlineServer = clusterService.getOnlineServer(clusterName);
-		if (onlineServer == null) {
-			throw new GlusterRuntimeException("No online server found in cluster [" + clusterName + "]");
-		}
-
-		try {
-			glusterUtil.addServer(onlineServer.getName(), serverName);
-		} catch (Exception e) {
-			// check if online server has gone offline. If yes, try again one more time.
-			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
-				// online server has gone offline! try with a different one.
-				onlineServer = clusterService.getNewOnlineServer(clusterName);
-				if (onlineServer == null) {
-					throw new GlusterRuntimeException("No online server found in cluster [" + clusterName + "]");
-				}
-				glusterUtil.addServer(onlineServer.getName(), serverName);
-			} else {
-				throw new GlusterRuntimeException(e.getMessage());
-			}
-		}
-	}
-
 	@POST
 	public Response addServer(@PathParam(PATH_PARAM_CLUSTER_NAME) String clusterName,
 			@FormParam(FORM_PARAM_SERVER_NAME) String serverName) {
-		if (clusterName == null || clusterName.isEmpty()) {
-			return badRequestResponse("Cluster name must not be empty!");
-		}
-
-		if (serverName == null || serverName.isEmpty()) {
-			return badRequestResponse("Parameter [" + FORM_PARAM_SERVER_NAME + "] is missing in request!");
-		}
-
-		ClusterInfo cluster = clusterService.getCluster(clusterName);
-		if (cluster == null) {
-			return notFoundResponse("Cluster [" + clusterName + "] not found!");
-		}
-
-		boolean publicKeyInstalled = sshUtil.isPublicKeyInstalled(serverName);
-		if (!publicKeyInstalled && !sshUtil.hasDefaultPassword(serverName)) {
-			// public key not installed, default password doesn't work. return with error.
-			return errorResponse("Gluster Management Gateway uses the default password to set up keys on the server."
-					+ CoreConstants.NEWLINE + "However it seems that the password on server [" + serverName
-					+ "] has been changed manually." + CoreConstants.NEWLINE
-					+ "Please reset it back to the standard default password and try again.");
-		}
-
-		String hostName = serverUtil.fetchHostName(serverName);
-		List<ServerInfo> servers = cluster.getServers();
-		if (servers != null && !servers.isEmpty()) {
-			// cluster has at least one existing server, so that peer probe can be performed
-			performAddServer(clusterName, hostName);
-		} else {
-			// this is the first server to be added to the cluster, which means no
-			// gluster CLI operation required. just add it to the cluster-server mapping
-		}
-
-		try {
-			// add the cluster-server mapping
-			clusterService.mapServerToCluster(clusterName, hostName);
-		} catch (Exception e) {
-			return errorResponse(e.getMessage());
-		}
-
-		// since the server is added to a cluster, it should not more be considered as a
-		// discovered server available to other clusters
-		discoveredServersResource.removeDiscoveredServer(hostName);
-
-		if (!publicKeyInstalled) {
-			try {
-				// install public key (this will also disable password based ssh login)
-				sshUtil.installPublicKey(hostName);
-			} catch (Exception e) {
-				return errorResponse("Public key could not be installed on [" + hostName + "]! Error: ["
-						+ e.getMessage() + "]");
-			}
-		}
-
-		return createdResponse(hostName);
+		return createdResponse(glusterServerService.addServerToCluster(clusterName, serverName));
 	}
 
 	@DELETE
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/DiscoveredServerService.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/DiscoveredServerService.java
new file mode 100644
index 00000000..6fad758c
--- /dev/null
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/DiscoveredServerService.java
@@ -0,0 +1,130 @@
+/*******************************************************************************
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of Gluster Management Console.
+ *
+ * Gluster Management Console is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *  
+ * Gluster Management Console is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
+ * for more details.
+ *  
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *******************************************************************************/
+package com.gluster.storage.management.gateway.services;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.gluster.storage.management.core.exceptions.GlusterRuntimeException;
+import com.gluster.storage.management.core.model.Server;
+import com.gluster.storage.management.core.utils.ProcessUtil;
+import com.gluster.storage.management.gateway.utils.ServerUtil;
+
+/**
+ *
+ */
+@Component
+public class DiscoveredServerService {
+	@Autowired
+	protected ServerUtil serverUtil;
+
+	private List<String> discoveredServerNames = new ArrayList<String>();
+	private static final Logger logger = Logger.getLogger(DiscoveredServerService.class);
+
+	public List<Server> getDiscoveredServerDetails() {
+		try {
+			List<Server> discoveredServers = Collections.synchronizedList(new ArrayList<Server>());
+			List<Thread> threads = createThreads(discoveredServers);
+			ProcessUtil.waitForThreads(threads);
+			return discoveredServers;
+		} catch (Exception e) {
+			String errMsg = "Exception while fetching details of discovered servers! Error: [" + e.getMessage() + "]";
+			logger.error(errMsg, e);
+			throw new GlusterRuntimeException(errMsg, e);
+		}
+	}
+
+	/**
+	 * Creates threads that will run in parallel and fetch details of all discovered servers
+	 * @param discoveredServers The list to be populated with details of discovered servers
+	 * @return
+	 * @throws InterruptedException
+	 */
+	private List<Thread> createThreads(List<Server> discoveredServers) throws InterruptedException {
+		List<String> discoveredServerNames = getDiscoveredServerNames();
+		List<Thread> threads = new ArrayList<Thread>();
+		for (int i = discoveredServerNames.size() - 1; i >= 0; i--) {
+			Thread thread = new DiscoveredServerDetailsThread(discoveredServers, discoveredServerNames.get(i));
+			threads.add(thread);
+			thread.start();
+			if (i >= 5 && i % 5 == 0) {
+				// After every 5 servers, wait for 1 second so that we don't end up with too many running threads
+				Thread.sleep(1000);
+			}
+		}
+		return threads;
+	}
+	
+	public List<String> getDiscoveredServerNames() {
+		return discoveredServerNames;
+	}
+	
+	public void setDiscoveredServerNames(List<String> discoveredServerNames) {
+		synchronized (discoveredServerNames) {
+			this.discoveredServerNames = discoveredServerNames;
+		}
+	}
+	
+	public void removeDiscoveredServer(String serverName) {
+		discoveredServerNames.remove(serverName);
+	}
+	
+	public void addDiscoveredServer(String serverName) {
+		discoveredServerNames.add(serverName);
+	}
+
+	public Server getDiscoveredServer(String serverName) {
+		Server server = new Server(serverName);
+		serverUtil.fetchServerDetails(server);
+		return server;
+	}
+	
+	public class DiscoveredServerDetailsThread extends Thread {
+		private List<Server> servers;
+		private String serverName;
+		private final Logger logger = Logger.getLogger(DiscoveredServerDetailsThread.class);
+
+		/**
+		 * Private constructor called on each thread
+		 * @param servers The list to be populated with fetched server details
+		 * @param serverName Name of the server whose details should be fetched by this thread
+		 */
+		private DiscoveredServerDetailsThread(List<Server> servers, String serverName) {
+			this.servers = servers;
+			this.serverName = serverName;
+		}
+
+		@Override
+		public void run() {
+			try {
+				logger.info("fetching details of discovered server [" + serverName + "] - start");
+				servers.add(getDiscoveredServer(serverName));
+				logger.info("fetching details of discovered server [" + serverName + "] - end");
+			} catch(Exception e) {
+				logger.warn("fetching details of discovered server [" + serverName + "] - error", e);
+				// eat the exception as we can't consider this server as a discovered server any more
+			}
+		}
+	}
+}
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java
index 7d5f0fff..f69b4a75 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java
@@ -18,10 +18,11 @@
  *******************************************************************************/
 package com.gluster.storage.management.gateway.services;
 
+import static com.gluster.storage.management.core.constants.RESTConstants.FORM_PARAM_SERVER_NAME;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Vector;
 
 import org.apache.log4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -35,11 +36,12 @@ import com.gluster.storage.management.core.model.GlusterServer;
 import com.gluster.storage.management.core.model.Server;
 import com.gluster.storage.management.core.model.Server.SERVER_STATUS;
 import com.gluster.storage.management.core.utils.GlusterCoreUtil;
+import com.gluster.storage.management.core.utils.ProcessUtil;
 import com.gluster.storage.management.gateway.data.ClusterInfo;
 import com.gluster.storage.management.gateway.data.ServerInfo;
-import com.gluster.storage.management.gateway.resources.v1_0.DiscoveredServersResource;
 import com.gluster.storage.management.gateway.utils.GlusterUtil;
 import com.gluster.storage.management.gateway.utils.ServerUtil;
+import com.gluster.storage.management.gateway.utils.SshUtil;
 
 /**
  *
@@ -56,11 +58,15 @@ public class GlusterServerService {
 	private GlusterUtil glusterUtil;
 	
 	@Autowired
+	private SshUtil sshUtil;
+
+	@Autowired
 	private VolumeService volumeService;
 	
-	// TODO: create a discovered servers "service" instead of injecting the resource directly
 	@Autowired
-	private DiscoveredServersResource discoveredServersResource;
+	private DiscoveredServerService discoveredServerService;
+	
+	private static final Logger logger = Logger.getLogger(GlusterServerService.class);
 	
 	public void fetchServerDetails(GlusterServer server) {
 		try {
@@ -121,7 +127,7 @@ public class GlusterServerService {
 		glusterServers = GlusterCoreUtil.skipEntities(glusterServers, maxCount, previousServerName);
 		
 		if (fetchDetails) {
-			String errMsg = fetchDetailsOfServers(glusterServers);
+			String errMsg = fetchDetailsOfServers(Collections.synchronizedList(glusterServers));
 			if (!errMsg.isEmpty()) {
 				throw new GlusterRuntimeException("Couldn't fetch details for server(s): " + errMsg);
 			}
@@ -131,10 +137,16 @@ public class GlusterServerService {
 
 	private String fetchDetailsOfServers(List<GlusterServer> glusterServers) {
 		try {
-			return new ServerDetailsThread(glusterServers).fetchDetails();
+			List<String> errors = Collections.synchronizedList(new ArrayList<String>());
+
+			List<Thread> threads = createThreads(glusterServers, errors);
+			ProcessUtil.waitForThreads(threads);
+			
+			return prepareErrorMessage(errors);
 		} catch(InterruptedException e) {
-			throw new GlusterRuntimeException("Exception while fetching details of servers! Error: [" + e.getMessage()
-					+ "]", e);
+			String errMsg = "Exception while fetching details of servers! Error: [" + e.getMessage() + "]";
+			logger.error(errMsg, e);
+			throw new GlusterRuntimeException(errMsg, e);
 		}
 //		String errMsg = "";
 //
@@ -147,66 +159,56 @@ public class GlusterServerService {
 //		}
 //		return errMsg;
 	}
-	
+
+	private String prepareErrorMessage(List<String> errors) {
+		String errMsg = "";
+		for(String error : errors) {
+			if(!errMsg.isEmpty()) {
+				errMsg += CoreConstants.NEWLINE;
+			}
+			errMsg +=  error;
+		}
+		
+		return errMsg;
+	}
+
+	/**
+	 * Creates threads that will run in parallel and fetch details of given gluster servers
+	 * @param discoveredServers The list to be populated with details of gluster servers
+	 * @param errors List to be populated with errors if any
+	 * @return
+	 * @throws InterruptedException
+	 */
+	private List<Thread> createThreads(List<GlusterServer> glusterServers, List<String> errors)
+			throws InterruptedException {
+		List<Thread> threads = new ArrayList<Thread>();
+		for (int i = glusterServers.size()-1; i >= 0 ; i--) {
+			Thread thread = new ServerDetailsThread(glusterServers.get(i), errors);
+			threads.add(thread);
+			thread.start();
+			if(i >= 5 && i % 5 == 0) {
+				// After every 5 servers, wait for 1 second so that we don't end up with too many running threads
+				Thread.sleep(1000);
+			}
+		}
+		return threads;
+	}
+
 	public class ServerDetailsThread extends Thread {
 		private List<String> errors;
-		private List<GlusterServer> glusterServers;
 		private GlusterServer server;
 		private final Logger logger = Logger.getLogger(ServerDetailsThread.class);
 
 		/**
-		 * This constructor should be called by clients that need to fetch details of the servers in parallel
-		 * @param glusterServers
-		 */
-		public ServerDetailsThread(List<GlusterServer> glusterServers) {
-			// create a synchronized "copy" so that the original list remains untouched
-			this(Collections.synchronizedList(new ArrayList<GlusterServer>(glusterServers)), Collections
-					.synchronizedList(new ArrayList<String>()), null);
-		}
-		
-		/**
 		 * Private constructor called on each thread
-		 * @param glusterServers
+		 * @param server The server whose details are to be fetched by this thread
 		 * @param errors
 		 */
-		private ServerDetailsThread(List<GlusterServer> glusterServers, List<String> errors, GlusterServer server) {
-			this.glusterServers = glusterServers;
+		private ServerDetailsThread(GlusterServer server, List<String> errors) {
 			this.errors = errors;
 			this.server = server;
 		}
 
-		/**
-		 * Call this method to fetch details of all the servers passed in the constructor. Internally creates one thread
-		 * for each server
-		 * 
-		 * @param glusterServers
-		 * @return
-		 */
-		public String fetchDetails() throws InterruptedException {
-			for (int i = glusterServers.size()-1; i >= 0 ; i--) {
-				new ServerDetailsThread(glusterServers, errors, glusterServers.get(i)).start();
-				if(i >= 5 && i % 5 == 0) {
-					// After every 5 servers, wait for 1 second so that we don't end up with too many running threads
-					Thread.sleep(1000);
-				}
-			}
-			
-			// Wait till all servers have been processed
-			while (!glusterServers.isEmpty()) {
-				Thread.sleep(500);
-			}
-			
-			String errMsg = "";
-			for(String error : errors) {
-				if(!errMsg.isEmpty()) {
-					errMsg += CoreConstants.NEWLINE;
-				}
-				errMsg +=  error;
-			}
-			
-			return errMsg;
-		}
-		
 		@Override
 		public void run() {
 			try {
@@ -217,7 +219,6 @@ public class GlusterServerService {
 				logger.error("fetching details of server [" + server.getName() + "] - error", e);
 				errors.add(server.getName() + " : [" + e.getMessage() + "]");
 			}
-			glusterServers.remove(server);
 		}
 	}
 	
@@ -340,7 +341,7 @@ public class GlusterServerService {
 
 			// since the server is removed from the cluster, it is now available to be added to other clusters.
 			// Hence add it back to the discovered servers list.
-			discoveredServersResource.addDiscoveredServer(serverName);
+			discoveredServerService.addDiscoveredServer(serverName);
 		}
 
 		clusterService.unmapServerFromCluster(clusterName, serverName);
@@ -354,4 +355,87 @@ public class GlusterServerService {
 		}
 		return false;
 	}
+
+	/**
+	 * Adds given server to cluster and returns its host name. e.g. If serverName passed is an IP address, this method
+	 * will return the host name of the machine with given IP address.
+	 * 
+	 * @param clusterName
+	 * @param serverName
+	 * @return
+	 */
+	public String addServerToCluster(String clusterName, String serverName) {
+		if (clusterName == null || clusterName.isEmpty()) {
+			throw new GlusterValidationException("Cluster name must not be empty!");
+		}
+
+		if (serverName == null || serverName.isEmpty()) {
+			throw new GlusterValidationException("Parameter [" + FORM_PARAM_SERVER_NAME + "] is missing in request!");
+		}
+
+		ClusterInfo cluster = clusterService.getCluster(clusterName);
+		if (cluster == null) {
+			throw new GlusterValidationException("Cluster [" + clusterName + "] not found!");
+		}
+
+		boolean publicKeyInstalled = sshUtil.isPublicKeyInstalled(serverName);
+		if (!publicKeyInstalled && !sshUtil.hasDefaultPassword(serverName)) {
+			// public key not installed, default password doesn't work. return with error.
+			throw new GlusterRuntimeException("Gluster Management Gateway uses the default password to set up keys on the server."
+					+ CoreConstants.NEWLINE + "However it seems that the password on server [" + serverName
+					+ "] has been changed manually." + CoreConstants.NEWLINE
+					+ "Please reset it back to the standard default password and try again.");
+		}
+
+		String hostName = serverUtil.fetchHostName(serverName);
+		List<ServerInfo> servers = cluster.getServers();
+		if (servers != null && !servers.isEmpty()) {
+			// cluster has at least one existing server, so that peer probe can be performed
+			performAddServer(clusterName, hostName);
+		} else {
+			// this is the first server to be added to the cluster, which means no
+			// gluster CLI operation required. just add it to the cluster-server mapping
+		}
+
+		// add the cluster-server mapping
+		clusterService.mapServerToCluster(clusterName, hostName);
+
+		// since the server is added to a cluster, it should not more be considered as a
+		// discovered server available to other clusters
+		discoveredServerService.removeDiscoveredServer(hostName);
+
+		if (!publicKeyInstalled) {
+			try {
+				// install public key (this will also disable password based ssh login)
+				sshUtil.installPublicKey(hostName);
+			} catch (Exception e) {
+				throw new GlusterRuntimeException("Public key could not be installed on [" + hostName + "]! Error: ["
+						+ e.getMessage() + "]");
+			}
+		}
+		return hostName;
+	}
+
+	private void performAddServer(String clusterName, String serverName) {
+		GlusterServer onlineServer = clusterService.getOnlineServer(clusterName);
+		if (onlineServer == null) {
+			throw new GlusterRuntimeException("No online server found in cluster [" + clusterName + "]");
+		}
+
+		try {
+			glusterUtil.addServer(onlineServer.getName(), serverName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+				if (onlineServer == null) {
+					throw new GlusterRuntimeException("No online server found in cluster [" + clusterName + "]");
+				}
+				glusterUtil.addServer(onlineServer.getName(), serverName);
+			} else {
+				throw new GlusterRuntimeException(e.getMessage());
+			}
+		}
+	}
 }
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/ServerSyncTask.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/ServerSyncTask.java
index 429981b1..8d788925 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/ServerSyncTask.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/ServerSyncTask.java
@@ -35,8 +35,8 @@ import com.gluster.storage.management.core.utils.ProcessResult;
 import com.gluster.storage.management.gateway.data.ClusterInfo;
 import com.gluster.storage.management.gateway.data.PersistenceDao;
 import com.gluster.storage.management.gateway.data.ServerInfo;
-import com.gluster.storage.management.gateway.resources.v1_0.DiscoveredServersResource;
 import com.gluster.storage.management.gateway.services.ClusterService;
+import com.gluster.storage.management.gateway.services.DiscoveredServerService;
 import com.gluster.storage.management.gateway.services.GlusterServerService;
 import com.gluster.storage.management.gateway.utils.ServerUtil;
 
@@ -54,7 +54,7 @@ public class ServerSyncTask {
 	private ServerUtil serverUtil;
 	
 	@Autowired
-	private DiscoveredServersResource discoveredServersResource;
+	private DiscoveredServerService discoveredServersService;
 	
 	@Autowired
 	private GlusterServerService glusterServerService;
@@ -163,6 +163,6 @@ public class ServerSyncTask {
 			}
 		}
 		
-		discoveredServersResource.setDiscoveredServerNames(serverNameList);
+		discoveredServersService.setDiscoveredServerNames(serverNameList);
 	}
 }
\ No newline at end of file