From 3e2f0eaa0bbdf1ac7b13c7ae443c70937933a4d3 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Wed, 12 Jul 2017 17:04:42 +0000 Subject: [PATCH] ref 8819: Extend The Algorithms Importer to Manage Many Processes as Black Boxes https://support.d4science.org/issues/8819 Updated the support for Processes as Black Boxes git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/portlets/user/statistical-algorithms-importer@151031 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../client/tools/input/ProjectInfoPanel.java | 13 +- .../server/generator/ProjectDeploy.java | 28 ++-- .../server/is/InformationSystemUtils.java | 5 +- .../poolmanager/DataMinerPoolManager.java | 155 ++++++++++++++++++ .../server/poolmanager/PoolManager.java | 35 ---- .../server/storage/DeploySave.java | 15 +- .../shared/Constants.java | 4 +- 7 files changed, 199 insertions(+), 56 deletions(-) create mode 100644 src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/DataMinerPoolManager.java delete mode 100644 src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/PoolManager.java diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/client/tools/input/ProjectInfoPanel.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/client/tools/input/ProjectInfoPanel.java index 02bb502..80f7fcb 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/client/tools/input/ProjectInfoPanel.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/client/tools/input/ProjectInfoPanel.java @@ -61,6 +61,12 @@ public class ProjectInfoPanel extends ContentPanel { algorithmName.setAllowBlank(false); algorithmName.addValidator( new RegExValidator("^[a-zA-Z0-9_]*$", "Attention only characters a-z,A-Z,0-9 are allowed")); + algorithmName.addValidator( + new RegExValidator("^((?!(TEST|test|Test)).)*$", "Attention the words that contain TEST are invalid names")); + + algorithmName.addValidator(new MaxLengthValidator(100)); + + algorithmName.setEmptyText("Enter name..."); algorithmName.addChangeHandler(new ChangeHandler() { @@ -79,6 +85,8 @@ public class ProjectInfoPanel extends ContentPanel { algorithmDescription.setAllowBlank(false); algorithmDescription.setEmptyText("Enter description..."); algorithmDescription.addValidator(new RegExValidator("^[^\"]*$", "Attention character \" is not allowed")); + algorithmDescription.addValidator(new RegExValidator("^[^|]*$", "Attention character | is not allowed")); + FieldLabel descriptionLabel = new FieldLabel(algorithmDescription, "Description"); descriptionLabel.setLabelWidth(LABAEL_WIDTH); @@ -86,7 +94,10 @@ public class ProjectInfoPanel extends ContentPanel { algorithmCategory = new TextField(); algorithmCategory.setAllowBlank(false); algorithmCategory.setEmptyText("Enter description..."); - algorithmCategory.addValidator(new RegExValidator("^[^\"]*$", "Attention character \" is not allowed")); + algorithmCategory.addValidator( + new RegExValidator("^[a-zA-Z0-9_]*$", "Attention only characters a-z,A-Z,0-9 are allowed")); + algorithmCategory.addValidator( + new RegExValidator("^((?!(TEST|test|Test)).)*$", "Attention the words that contain TEST are invalid names")); algorithmCategory.addValidator(new MaxLengthValidator(32)); FieldLabel categoryLabel = new FieldLabel(algorithmCategory, "Category"); categoryLabel.setLabelWidth(LABAEL_WIDTH); diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectDeploy.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectDeploy.java index 618d8de..8282eb4 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectDeploy.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectDeploy.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import javax.servlet.http.HttpServletRequest; +import org.gcube.portlets.user.statisticalalgorithmsimporter.server.poolmanager.DataMinerPoolManager; import org.gcube.portlets.user.statisticalalgorithmsimporter.server.social.AlgorithmNotification; import org.gcube.portlets.user.statisticalalgorithmsimporter.server.social.Recipient; import org.gcube.portlets.user.statisticalalgorithmsimporter.server.storage.DeploySave; @@ -16,29 +17,27 @@ import org.slf4j.LoggerFactory; /** * - * @author Giancarlo Panichi + * @author Giancarlo Panichi * * */ public class ProjectDeploy { private static Logger logger = LoggerFactory.getLogger(ProjectDeploy.class); - private ServiceCredentials serviceCredentials; private Project project; private ArrayList recipients; private HttpServletRequest httpServletRequest; - public ProjectDeploy(HttpServletRequest httpServletRequest, - ServiceCredentials serviceCredentials, Project project, + public ProjectDeploy(HttpServletRequest httpServletRequest, ServiceCredentials serviceCredentials, Project project, ArrayList recipients) { this.serviceCredentials = serviceCredentials; this.project = project; this.recipients = recipients; this.httpServletRequest = httpServletRequest; - + } - + public void deploy() throws StatAlgoImporterServiceException{ InfoGenerator infoGenerator=new InfoGenerator(project, serviceCredentials); InfoData infoData=infoGenerator.readInfo(); @@ -47,14 +46,19 @@ public class ProjectDeploy { deploySave.save(); logger.debug("Send notify"); sendNotify(deploySave.getInfoText()); + logger.debug("CodeJarAdminCopy for PoolManager: "+deploySave.getCodeJarAdminCopy()); + DataMinerPoolManager poolManager=new DataMinerPoolManager(serviceCredentials); + String operationId=poolManager.deployAlgorithm(infoData, deploySave.getCodeJarAdminCopy()); + logger.debug("Deploy operationId: "+operationId); + String deployStatus=poolManager.monitorAlgorithmDeploy(operationId); + logger.debug("Deploy status: "+deployStatus); + } - - private void sendNotify(String body){ - AlgorithmNotification notify = new AlgorithmNotification(httpServletRequest, - serviceCredentials, recipients, body); + + private void sendNotify(String body) { + AlgorithmNotification notify = new AlgorithmNotification(httpServletRequest, serviceCredentials, recipients, + body); notify.run(); } - - } diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/is/InformationSystemUtils.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/is/InformationSystemUtils.java index fffd9a2..89d2d84 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/is/InformationSystemUtils.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/is/InformationSystemUtils.java @@ -107,8 +107,9 @@ public class InformationSystemUtils { logger.debug("Retrieve DataMiner Pool Manager resource in scope: " + scope); SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class); - query.addCondition("$resource/Profile/SecondaryType/text() eq '" + Constants.POOL_MANAGER_CATEGORY + "'") - .addCondition("$resource/Profile/Name/text() eq '" + Constants.POOL_MANAGER_NAME + "'") + + query.addCondition("$resource/Profile/ServiceClass/text() eq '" + Constants.POOL_MANAGER_SERVICE_CLASS + "'") + .addCondition("$resource/Profile/ServiceName/text() eq '" + Constants.POOL_MANAGER_SERVICE_NAME + "'") .setResult("$resource"); DiscoveryClient client = ICFactory.clientFor(GCoreEndpoint.class); diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/DataMinerPoolManager.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/DataMinerPoolManager.java new file mode 100644 index 0000000..3255b0b --- /dev/null +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/DataMinerPoolManager.java @@ -0,0 +1,155 @@ +package org.gcube.portlets.user.statisticalalgorithmsimporter.server.poolmanager; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; + +import org.gcube.portlets.user.statisticalalgorithmsimporter.server.is.InformationSystemUtils; +import org.gcube.portlets.user.statisticalalgorithmsimporter.server.util.ServiceCredentials; +import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.exception.StatAlgoImporterServiceException; +import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.info.InfoData; +import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.workspace.ItemDescription; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author Giancarlo Panichi + * + * + */ +public class DataMinerPoolManager { + private static Logger logger = LoggerFactory.getLogger(DataMinerPoolManager.class); + private String serverUrl; + private ServiceCredentials serviceCredentials; + + public DataMinerPoolManager(ServiceCredentials serviceCredentials) throws StatAlgoImporterServiceException { + this.serviceCredentials=serviceCredentials; + serverUrl = InformationSystemUtils.retrieveDataMinerPoolManager(serviceCredentials.getScope()); + } + + public String deployAlgorithm(InfoData infoData, + ItemDescription codeJarAdminCopy) throws StatAlgoImporterServiceException { + String operationId = sendRequest(infoData, codeJarAdminCopy, serverUrl); + return operationId; + } + + public String monitorAlgorithmDeploy(String operationId) throws StatAlgoImporterServiceException { + String deployStatus = deployStatus(serverUrl, operationId); + return deployStatus; + } + + + + private String sendRequest(InfoData infoData, + ItemDescription codeJarAdminCopy, String serverUrl) throws StatAlgoImporterServiceException { + logger.info("Send request to DataMinerPoolManager: " + serverUrl); + /* + * http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-2.0.0- + * SNAPSHOT/api/algorithm/stage? + * gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548 + * &algorithmPackageURL=http://data.d4science.org/ + * dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0 + * &category=ICHTHYOP_MODEL + */ + + try { + String requestUrl = serverUrl + "/algorithm/stage?gcube-token=" + serviceCredentials.getToken() + + "&algorithmPackageURL=" + codeJarAdminCopy.getPublicLink() + + "&algorithm_type=transducers&category=" + infoData.getAlgorithmCategory(); + logger.debug("DataMinerPoolManager request=" + requestUrl); + + // String authString = user + ":" + password; + // logger.debug("auth string: " + authString); + // byte[] authEncBytes = Base64.encodeBase64(authString.getBytes()); + // String encoded = new String(authEncBytes); + // logger.debug("Base64 encoded auth string: " + encoded); + + URL urlObj = new URL(requestUrl); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + // connection.setRequestProperty("Authorization", "Basic " + + // encoded); + InputStream is = connection.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(is)); + String line = null; + StringBuffer operationId = new StringBuffer(); + logger.info("DataMinerPoolManager response: "); + while ((line = reader.readLine()) != null) { + logger.info(line); + operationId.append(line); + } + + return operationId.toString(); + } catch (MalformedURLException e) { + logger.error("DataMinerPoolManager URL seems to be invalid: " + e.getLocalizedMessage(), e); + throw new StatAlgoImporterServiceException( + "DataMinerPoolManager URL seems to be invalid: " + e.getLocalizedMessage(), e); + } catch (IOException e) { + logger.error("DataMinerPoolManager error occured in request: " + e.getLocalizedMessage(), e); + throw new StatAlgoImporterServiceException( + "DataMinerPoolManager error occured in request: " + e.getLocalizedMessage(), e); + + } catch (Throwable e) { + logger.error("DataMinerPoolManager error occured: " + e.getLocalizedMessage(), e); + throw new StatAlgoImporterServiceException("DataMinerPoolManager error occured: " + e.getLocalizedMessage(), + e); + + } + } + + private String deployStatus(String serverUrl, String operationId) + throws StatAlgoImporterServiceException { + logger.info("Send monitor request to DataMinerPoolManager: " + serverUrl); + /* + * http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-2.0.0- + * SNAPSHOT/api/log? + * gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548 + * &logUrl=426c8e35-a624-4710-b612-c90929c32c27 + */ + + try { + String requestUrl = serverUrl + "/log?gcube-token=" + serviceCredentials.getToken() + "&logUrl=" + + operationId; + logger.debug("DataMinerPoolManager monitor request=" + requestUrl); + + URL urlObj = new URL(requestUrl); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + + InputStream is = connection.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(is)); + String line = null; + StringBuffer operationStatus = new StringBuffer(); + logger.info("DataMinerPoolManager response: "); + while ((line = reader.readLine()) != null) { + logger.info(line); + operationStatus.append(line); + } + + return operationStatus.toString(); + } catch (MalformedURLException e) { + logger.error("DataMinerPoolManager monitor URL seems to be invalid: " + e.getLocalizedMessage(), e); + throw new StatAlgoImporterServiceException( + "DataMinerPoolManager monitor URL seems to be invalid: " + e.getLocalizedMessage(), e); + } catch (IOException e) { + logger.error("DataMinerPoolManager error occured in monitor request: " + e.getLocalizedMessage(), e); + throw new StatAlgoImporterServiceException( + "DataMinerPoolManager error occured in monitor request: " + e.getLocalizedMessage(), e); + + } catch (Throwable e) { + logger.error("DataMinerPoolManager monitor error occured: " + e.getLocalizedMessage(), e); + throw new StatAlgoImporterServiceException( + "DataMinerPoolManager monitor error occured: " + e.getLocalizedMessage(), e); + + } + + } + +} diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/PoolManager.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/PoolManager.java deleted file mode 100644 index bcc1356..0000000 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/PoolManager.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.portlets.user.statisticalalgorithmsimporter.server.poolmanager; - -import org.gcube.portlets.user.statisticalalgorithmsimporter.server.is.InformationSystemUtils; -import org.gcube.portlets.user.statisticalalgorithmsimporter.server.util.ServiceCredentials; -import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.exception.StatAlgoImporterServiceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - * @author Giancarlo Panichi - * - * - */ -public class PoolManager { - private static Logger logger = LoggerFactory.getLogger(PoolManager.class); - - private ServiceCredentials serviceCredentials; - - public PoolManager(ServiceCredentials serviceCredentials) { - this.serviceCredentials=serviceCredentials; - } - - public void deployAlgorithm() throws StatAlgoImporterServiceException{ - retrievePoolManagerURI(); - } - - private String retrievePoolManagerURI() throws StatAlgoImporterServiceException { - String poolManagerURI=InformationSystemUtils.retrieveDataMinerPoolManager(serviceCredentials.getScope()); - logger.debug("PoolManagerURI: "+poolManagerURI); - return poolManagerURI; - } - - -} diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/DeploySave.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/DeploySave.java index 95d590c..7262c1d 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/DeploySave.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/DeploySave.java @@ -39,12 +39,15 @@ public class DeploySave { private Project project; private InfoData infoData; private String infoText; + private ItemDescription codeJarAdminCopy; + public DeploySave(ServiceCredentials serviceCredentials, Project project, InfoData infoData) { this.serviceCredentials = serviceCredentials; this.project = project; this.infoData = infoData; this.infoText = null; + this.codeJarAdminCopy=null; filesStorage = new FilesStorage(); } @@ -58,7 +61,6 @@ public class DeploySave { WorkspaceItem codeJarItem = filesStorage.createItemOnWorkspaceHowAdmin(ADMIN_USER, codeJarInputStream, codeJar.getName(), CODE_JAR_DESCRIPTION, CODE_JAR_MIMETYPE, DESTINATION_FOLDER); - ItemDescription codeJarAdminCopy; try { codeJarAdminCopy = new ItemDescription(codeJarItem.getId(), codeJarItem.getName(), codeJarItem.getOwner().getPortalLogin(), codeJarItem.getPath(), codeJarItem.getType().name()); @@ -69,8 +71,7 @@ public class DeploySave { } - createInfoText(codeJarAdminCopy); - + createInfoText(); try { int codeJarNamelenght = codeJar.getName().length(); @@ -98,7 +99,7 @@ public class DeploySave { } - private void createInfoText(ItemDescription codeJarAdminCopy) { + private void createInfoText() { infoText = "Username: " + serviceCredentials.getUserName() + "\nFull Name: " + serviceCredentials.getFullName() + "\nEmail: " + serviceCredentials.getEmail() + "\n\nin VRE: " + serviceCredentials.getScope() + "\n\nhas requested to publish the algorithm: " + "\nLanguage: " + infoData.getLanguage() @@ -117,4 +118,10 @@ public class DeploySave { return infoText; } + public ItemDescription getCodeJarAdminCopy() { + return codeJarAdminCopy; + } + + + } diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/shared/Constants.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/shared/Constants.java index a3b674f..059de75 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/shared/Constants.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/shared/Constants.java @@ -35,8 +35,8 @@ public class Constants { public static final String SAI_NAME = "SAIProfile"; public static final String SAI_CATEGORY = "SAI"; - public static final String POOL_MANAGER_NAME = "dataminer-pool-manager"; - public static final String POOL_MANAGER_CATEGORY = "DataAnalysis"; + public static final String POOL_MANAGER_SERVICE_NAME = "dataminer-pool-manager"; + public static final String POOL_MANAGER_SERVICE_CLASS = "DataAnalysis"; // Main Generator public static final String PRE_INSTALLED = "Pre-Installed";