method that retrieve list of users have been speed up by exploiting executors

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/vo-management/usermanagement-core@151262 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/vomanagement/usermanagement/impl/ws/LiferayWSUserManager.java

@@ -6,10 +6,12 @@ import java.net.URLEncoder;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.http.HttpHost;
@@ -83,6 +85,7 @@ public class LiferayWSUserManager implements UserManager{
 
 	// some pre-defined constants
 	private static final String USER_LOCATION_INDUSTRY_KEY = "industry";
+	private static final int USERS_EXECUTOR_FACTOR = 20;
 
 	/**
 	 * In order to contact the json ws of Liferay, user and password are needed. The host in which the current JVM
@@ -225,7 +228,7 @@ public class LiferayWSUserManager implements UserManager{
 	}
 
 	/**
-	 * Given the contactId value, retrieves the json object releted to this information
+	 * Given the contactId value, retrieves the json object related to this information
 	 * @param contactId
 	 * @return
 	 */
@@ -368,23 +371,64 @@ public class LiferayWSUserManager implements UserManager{
 	}
 
 	@Override
+	@SuppressWarnings("rawtypes")
 	public List<GCubeUser> listUsersByGroup(long groupId)
 			throws UserManagementSystemException, GroupRetrievalFault,
 			UserRetrievalFault {
 
 		try{
-			List<GCubeUser> toReturn = new ArrayList<GCubeUser>();
+			final List<GCubeUser> toReturn = new ArrayList<GCubeUser>();
 			String jsonUsers = 
 					HttpUtils.executeHTTPGETRequest(API_BASE_URL + GET_USERS_BY_GROUP.replace("$GROUP_ID", String.valueOf(groupId)), 
 							credsProvider, localContext, target);
 
 			if(jsonUsers != null){
+
 				logger.debug("Trying to parse json users array ");
 				JSONParser parser = new JSONParser();
 				JSONArray array = (JSONArray)parser.parse(jsonUsers);
-				for (int i = 0; i < array.size(); i++) {
+
-					toReturn.add(mapLRUser(((JSONObject)array.get(i)).toJSONString()));
+				// use executors to speed up this process
+				int numberOfThreads = (int) Math.ceil((double)array.size()/(double)USERS_EXECUTOR_FACTOR);
+				logger.trace("Number of concurrent threads is going to be " + numberOfThreads);
+
+				// let do the job by some threads
+				ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
+				int start = 0;
+				int offset = USERS_EXECUTOR_FACTOR;
+
+				for(int i = 0; i < numberOfThreads; i++){
+					start = USERS_EXECUTOR_FACTOR * i;
+					offset = (start + offset) > array.size() ? array.size() - start: offset;
+					logger.trace("Start = " + start + ", offset=" + offset);
+					final List usersPortion = array.subList(start, start + offset);
+
+					executor.submit(new Runnable() {
+						@Override
+						public void run() {
+							logger.debug("Thread is " + Thread.currentThread().getName());
+
+							ArrayList<GCubeUser> localList = new ArrayList<GCubeUser>();
+							for(int i = 0; i < usersPortion.size(); i++){
+								localList.add(mapLRUser(((JSONObject)usersPortion.get(i)).toJSONString()));
 							}
+
+							// avoid contention on a single retrieved value but add them all at the end
+							synchronized (toReturn){
+								toReturn.addAll(localList);
+							}
+						}
+					});
+
+					// stop when ...
+					if((start + offset) >= array.size())
+						break;
+				}
+
+				// wait threads to finish
+				executor.shutdown();
+				executor.awaitTermination(2, TimeUnit.MINUTES);
+
 			}else
 				return null;
 
@@ -478,26 +522,54 @@ public class LiferayWSUserManager implements UserManager{
 	}
 
 	@Override
-	public List<GCubeUser> listUsersByGroupAndRole(long groupId, long roleId)
+	public List<GCubeUser> listUsersByGroupAndRole(final long groupId, final long roleId)
 			throws UserManagementSystemException, RoleRetrievalFault,
 			GroupRetrievalFault, UserRetrievalFault {
 
-		List<GCubeUser> toReturn = listUsersByGroup(groupId);
+		final List<GCubeUser> toReturn = new ArrayList<GCubeUser>(0);
+
 		try {
 
+			final List<GCubeUser> usersInGroup = listUsersByGroup(groupId);
+			logger.debug("Number of users is " + usersInGroup.size());
 
-			Iterator<GCubeUser> iterator = toReturn.iterator();
+			if(usersInGroup == null || usersInGroup.isEmpty())
-			while (iterator.hasNext()) {
+				return toReturn;
-				GCubeUser gCubeUser = (GCubeUser) iterator.next();
-				long userId = gCubeUser.getUserId();
 
+			// use executors to speed up this process
+			int numberOfThreads = (int) Math.ceil((double)usersInGroup.size()/(double)USERS_EXECUTOR_FACTOR);
+			logger.trace("Number of concurrent threads is going to be " + numberOfThreads);
+
+			// let do the job by some threads
+			ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
+			int start = 0;
+			int offset = USERS_EXECUTOR_FACTOR;
+
+			for(int i = 0; i < numberOfThreads; i++){
+				start = USERS_EXECUTOR_FACTOR * i;
+				offset = (start + offset) > usersInGroup.size() ? usersInGroup.size() - start: offset;
+				logger.trace("Start = " + start + ", offset=" + offset);
+				final List<GCubeUser> usersPortion = usersInGroup.subList(start, start + offset);
+
+				executor.submit(new Runnable() {
+
+					@Override
+					public void run() {
+						logger.debug("Thread is " + Thread.currentThread().getName());
+						ArrayList<GCubeUser> localUsersHavingRole = new ArrayList<GCubeUser>(0);
+
+						for(GCubeUser user: usersPortion){
+							long userId = user.getUserId();
 							String userRoles = 
 									HttpUtils.executeHTTPGETRequest(API_BASE_URL + GET_ROLES_IN_GROUP_BY_USER.replace("$GROUP_ID", String.valueOf(groupId)).replace("$USER_ID", String.valueOf(userId)), 
 											credsProvider, localContext, target);
 
 							boolean userHasRole = false;
 							JSONParser parser = new JSONParser();
-				JSONArray array = (JSONArray)parser.parse(userRoles);
+							JSONArray array;
+							try {
+								array = (JSONArray)parser.parse(userRoles);
+
 								for (int i = 0; i < array.size(); i++) {
 									try {
 										GCubeRole role = LiferayWSRoleManager.mapLRRole(array.get(i).toString());
@@ -506,13 +578,34 @@ public class LiferayWSUserManager implements UserManager{
 											break;
 										}
 									} catch (PortalException | SystemException | ParseException e) {
-						logger.warn("Failed to retrieve a role for user " + gCubeUser.getUsername());
+										logger.warn("Failed to retrieve a role for user " + user.getUsername());
 									}
 								}
 
-				if(!userHasRole)
+								// add if he/she has
-					iterator.remove();
+								if(userHasRole)
+									localUsersHavingRole.add(user);
+
+							} catch (ParseException e1) {
+								logger.warn("Failed to parse role for user " + user.getUsername());
 							}
+						}
+
+						// avoid contention on a single retrieved value but add them all at the end
+						synchronized (toReturn) {
+							toReturn.addAll(localUsersHavingRole);
+						}
+					}
+				});
+
+				// stop when ...
+				if((start + offset) >= usersInGroup.size())
+					break;
+			}
+
+			// wait threads to finish
+			executor.shutdown();
+			executor.awaitTermination(2, TimeUnit.MINUTES);
 
 		} catch (Exception e) {
 			logger.warn("Failed to retrieve users that have role whit id " + roleId + " in group with id " + groupId);

src/test/java/org/gcube/vomanagement/usermanagement/test/LiferayWSUserTest.java

@@ -57,8 +57,26 @@ public class LiferayWSUserTest{
 	//@Test
 	public void listUsersByGroup() throws UserManagementSystemException, UserRetrievalFault, GroupRetrievalFault{
 
+		long init = System.currentTimeMillis();
 		List<GCubeUser> gcubeUsers = userManager.listUsersByGroup(21660);
-		logger.debug("Retrieved user object " + gcubeUsers);
+		logger.debug("Retrieved user object " + gcubeUsers.size());
+		logger.debug("Time is " + (System.currentTimeMillis() - init) );
+	}
+	
+	//@Test
+	public void getUsersHavingRole() throws UserManagementSystemException, RoleRetrievalFault, GroupRetrievalFault, UserRetrievalFault{
+
+		long roleId  = 29548512; // data miner manager
+		long groupId = 21660;
+		long init = System.currentTimeMillis();
+		List<GCubeUser> users = userManager.listUsersByGroupAndRole(groupId, roleId);
+		logger.debug("Time is " + (System.currentTimeMillis() - init) );
+		logger.debug("Users are " + users);
+		
+		for (GCubeUser user:users) {
+			logger.debug("Username is " + user.getUsername() + "\n");
+			
+		}
 
 	}
 
@@ -80,18 +98,6 @@ public class LiferayWSUserTest{
 
 	}
 
-	//@Test
-	public void getUsersHavingRole() throws UserManagementSystemException, RoleRetrievalFault, GroupRetrievalFault, UserRetrievalFault{
-
-		long roleId  = 25184135;
-		long groupId = 21660;
-		long init = System.currentTimeMillis();
-		List<GCubeUser> users = userManager.listUsersByGroupAndRole(groupId, roleId);
-		logger.debug("Time is " + (System.currentTimeMillis() - init) );
-		logger.debug("Users are " + users);
-
-	}
-
 	//@Test
 	public void listUsersByGlobalRole(){