gCubeSystem
/
statistical-algorithms-importer
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ref 8819: Extend The Algorithms Importer to Manage Many Processes as Black Boxes 

https://support.d4science.org/issues/8819

Updated the support for Processes as Black Boxes

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/portlets/user/statistical-algorithms-importer@151031 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Giancarlo Panichi 2017-07-12 17:04:42 +00:00
parent 6244c81883
commit 3e2f0eaa0b
7 changed files with 199 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/client/tools/input/ProjectInfoPanel.java
View File

@ -61,6 +61,12 @@ public class ProjectInfoPanel extends ContentPanel {
algorithmName.setAllowBlank(false);
algorithmName.addValidator(
new RegExValidator("^[a-zA-Z0-9_]*$", "Attention only characters a-z,A-Z,0-9 are allowed"));
algorithmName.addValidator(
new RegExValidator("^((?!(TEST|test|Test)).)*$", "Attention the words that contain TEST are invalid names"));
algorithmName.addValidator(new MaxLengthValidator(100));
algorithmName.setEmptyText("Enter name...");
algorithmName.addChangeHandler(new ChangeHandler() {
@ -79,6 +85,8 @@ public class ProjectInfoPanel extends ContentPanel {
algorithmDescription.setAllowBlank(false);
algorithmDescription.setEmptyText("Enter description...");
algorithmDescription.addValidator(new RegExValidator("^[^\"]*$", "Attention character \" is not allowed"));
algorithmDescription.addValidator(new RegExValidator("^[^|]*$", "Attention character | is not allowed"));
FieldLabel descriptionLabel = new FieldLabel(algorithmDescription, "Description");
descriptionLabel.setLabelWidth(LABAEL_WIDTH);
@ -86,7 +94,10 @@ public class ProjectInfoPanel extends ContentPanel {
algorithmCategory = new TextField();
algorithmCategory.setAllowBlank(false);
algorithmCategory.setEmptyText("Enter description...");
algorithmCategory.addValidator(new RegExValidator("^[^\"]*$", "Attention character \" is not allowed"));
algorithmCategory.addValidator(
new RegExValidator("^[a-zA-Z0-9_]*$", "Attention only characters a-z,A-Z,0-9 are allowed"));
algorithmCategory.addValidator(
new RegExValidator("^((?!(TEST|test|Test)).)*$", "Attention the words that contain TEST are invalid names"));
algorithmCategory.addValidator(new MaxLengthValidator(32));
FieldLabel categoryLabel = new FieldLabel(algorithmCategory, "Category");
categoryLabel.setLabelWidth(LABAEL_WIDTH);

28
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectDeploy.java
View File

@ -4,6 +4,7 @@ import java.util.ArrayList;
import javax.servlet.http.HttpServletRequest;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.poolmanager.DataMinerPoolManager;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.social.AlgorithmNotification;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.social.Recipient;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.storage.DeploySave;
@ -16,29 +17,27 @@ import org.slf4j.LoggerFactory;
/**
*
* @author Giancarlo Panichi
* @author Giancarlo Panichi
*
*
*/
public class ProjectDeploy {
private static Logger logger = LoggerFactory.getLogger(ProjectDeploy.class);
private ServiceCredentials serviceCredentials;
private Project project;
private ArrayList<Recipient> recipients;
private HttpServletRequest httpServletRequest;
public ProjectDeploy(HttpServletRequest httpServletRequest,
ServiceCredentials serviceCredentials, Project project,
public ProjectDeploy(HttpServletRequest httpServletRequest, ServiceCredentials serviceCredentials, Project project,
ArrayList<Recipient> recipients) {
this.serviceCredentials = serviceCredentials;
this.project = project;
this.recipients = recipients;
this.httpServletRequest = httpServletRequest;
}
public void deploy() throws StatAlgoImporterServiceException{
InfoGenerator infoGenerator=new InfoGenerator(project, serviceCredentials);
InfoData infoData=infoGenerator.readInfo();
@ -47,14 +46,19 @@ public class ProjectDeploy {
deploySave.save();
logger.debug("Send notify");
sendNotify(deploySave.getInfoText());
logger.debug("CodeJarAdminCopy for PoolManager: "+deploySave.getCodeJarAdminCopy());
DataMinerPoolManager poolManager=new DataMinerPoolManager(serviceCredentials);
String operationId=poolManager.deployAlgorithm(infoData, deploySave.getCodeJarAdminCopy());
logger.debug("Deploy operationId: "+operationId);
String deployStatus=poolManager.monitorAlgorithmDeploy(operationId);
logger.debug("Deploy status: "+deployStatus);
}
private void sendNotify(String body){
AlgorithmNotification notify = new AlgorithmNotification(httpServletRequest,
serviceCredentials, recipients, body);
private void sendNotify(String body) {
AlgorithmNotification notify = new AlgorithmNotification(httpServletRequest, serviceCredentials, recipients,
body);
notify.run();
}
}

5
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/is/InformationSystemUtils.java
View File

@ -107,8 +107,9 @@ public class InformationSystemUtils {
logger.debug("Retrieve DataMiner Pool Manager resource in scope: " + scope);
SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq '" + Constants.POOL_MANAGER_CATEGORY + "'")
.addCondition("$resource/Profile/Name/text() eq '" + Constants.POOL_MANAGER_NAME + "'")
query.addCondition("$resource/Profile/ServiceClass/text() eq '" + Constants.POOL_MANAGER_SERVICE_CLASS + "'")
.addCondition("$resource/Profile/ServiceName/text() eq '" + Constants.POOL_MANAGER_SERVICE_NAME + "'")
.setResult("$resource");
DiscoveryClient<GCoreEndpoint> client = ICFactory.clientFor(GCoreEndpoint.class);

155
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/DataMinerPoolManager.java Normal file
View File

@ -0,0 +1,155 @@
package org.gcube.portlets.user.statisticalalgorithmsimporter.server.poolmanager;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.is.InformationSystemUtils;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.util.ServiceCredentials;
import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.exception.StatAlgoImporterServiceException;
import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.info.InfoData;
import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.workspace.ItemDescription;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Giancarlo Panichi
*
*
*/
public class DataMinerPoolManager {
private static Logger logger = LoggerFactory.getLogger(DataMinerPoolManager.class);
private String serverUrl;
private ServiceCredentials serviceCredentials;
public DataMinerPoolManager(ServiceCredentials serviceCredentials) throws StatAlgoImporterServiceException {
this.serviceCredentials=serviceCredentials;
serverUrl = InformationSystemUtils.retrieveDataMinerPoolManager(serviceCredentials.getScope());
}
public String deployAlgorithm(InfoData infoData,
ItemDescription codeJarAdminCopy) throws StatAlgoImporterServiceException {
String operationId = sendRequest(infoData, codeJarAdminCopy, serverUrl);
return operationId;
}
public String monitorAlgorithmDeploy(String operationId) throws StatAlgoImporterServiceException {
String deployStatus = deployStatus(serverUrl, operationId);
return deployStatus;
}
private String sendRequest(InfoData infoData,
ItemDescription codeJarAdminCopy, String serverUrl) throws StatAlgoImporterServiceException {
logger.info("Send request to DataMinerPoolManager: " + serverUrl);
/*
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-2.0.0-
* SNAPSHOT/api/algorithm/stage?
* gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548
* &algorithmPackageURL=http://data.d4science.org/
* dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0
* &category=ICHTHYOP_MODEL
*/
try {
String requestUrl = serverUrl + "/algorithm/stage?gcube-token=" + serviceCredentials.getToken()
+ "&algorithmPackageURL=" + codeJarAdminCopy.getPublicLink()
+ "&algorithm_type=transducers&category=" + infoData.getAlgorithmCategory();
logger.debug("DataMinerPoolManager request=" + requestUrl);
// String authString = user + ":" + password;
// logger.debug("auth string: " + authString);
// byte[] authEncBytes = Base64.encodeBase64(authString.getBytes());
// String encoded = new String(authEncBytes);
// logger.debug("Base64 encoded auth string: " + encoded);
URL urlObj = new URL(requestUrl);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
connection.setRequestMethod("GET");
connection.setDoOutput(true);
// connection.setRequestProperty("Authorization", "Basic " +
// encoded);
InputStream is = connection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
String line = null;
StringBuffer operationId = new StringBuffer();
logger.info("DataMinerPoolManager response: ");
while ((line = reader.readLine()) != null) {
logger.info(line);
operationId.append(line);
}
return operationId.toString();
} catch (MalformedURLException e) {
logger.error("DataMinerPoolManager URL seems to be invalid: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException(
"DataMinerPoolManager URL seems to be invalid: " + e.getLocalizedMessage(), e);
} catch (IOException e) {
logger.error("DataMinerPoolManager error occured in request: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException(
"DataMinerPoolManager error occured in request: " + e.getLocalizedMessage(), e);
} catch (Throwable e) {
logger.error("DataMinerPoolManager error occured: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException("DataMinerPoolManager error occured: " + e.getLocalizedMessage(),
e);
}
}
private String deployStatus(String serverUrl, String operationId)
throws StatAlgoImporterServiceException {
logger.info("Send monitor request to DataMinerPoolManager: " + serverUrl);
/*
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-2.0.0-
* SNAPSHOT/api/log?
* gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548
* &logUrl=426c8e35-a624-4710-b612-c90929c32c27
*/
try {
String requestUrl = serverUrl + "/log?gcube-token=" + serviceCredentials.getToken() + "&logUrl="
+ operationId;
logger.debug("DataMinerPoolManager monitor request=" + requestUrl);
URL urlObj = new URL(requestUrl);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
connection.setRequestMethod("GET");
connection.setDoOutput(true);
InputStream is = connection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
String line = null;
StringBuffer operationStatus = new StringBuffer();
logger.info("DataMinerPoolManager response: ");
while ((line = reader.readLine()) != null) {
logger.info(line);
operationStatus.append(line);
}
return operationStatus.toString();
} catch (MalformedURLException e) {
logger.error("DataMinerPoolManager monitor URL seems to be invalid: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException(
"DataMinerPoolManager monitor URL seems to be invalid: " + e.getLocalizedMessage(), e);
} catch (IOException e) {
logger.error("DataMinerPoolManager error occured in monitor request: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException(
"DataMinerPoolManager error occured in monitor request: " + e.getLocalizedMessage(), e);
} catch (Throwable e) {
logger.error("DataMinerPoolManager monitor error occured: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException(
"DataMinerPoolManager monitor error occured: " + e.getLocalizedMessage(), e);
}
}
}

35
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/poolmanager/PoolManager.java
View File

@ -1,35 +0,0 @@
package org.gcube.portlets.user.statisticalalgorithmsimporter.server.poolmanager;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.is.InformationSystemUtils;
import org.gcube.portlets.user.statisticalalgorithmsimporter.server.util.ServiceCredentials;
import org.gcube.portlets.user.statisticalalgorithmsimporter.shared.exception.StatAlgoImporterServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Giancarlo Panichi
*
*
*/
public class PoolManager {
private static Logger logger = LoggerFactory.getLogger(PoolManager.class);
private ServiceCredentials serviceCredentials;
public PoolManager(ServiceCredentials serviceCredentials) {
this.serviceCredentials=serviceCredentials;
}
public void deployAlgorithm() throws StatAlgoImporterServiceException{
retrievePoolManagerURI();
}
private String retrievePoolManagerURI() throws StatAlgoImporterServiceException {
String poolManagerURI=InformationSystemUtils.retrieveDataMinerPoolManager(serviceCredentials.getScope());
logger.debug("PoolManagerURI: "+poolManagerURI);
return poolManagerURI;
}
}

15
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/DeploySave.java
View File

@ -39,12 +39,15 @@ public class DeploySave {
private Project project;
private InfoData infoData;
private String infoText;
private ItemDescription codeJarAdminCopy;
public DeploySave(ServiceCredentials serviceCredentials, Project project, InfoData infoData) {
this.serviceCredentials = serviceCredentials;
this.project = project;
this.infoData = infoData;
this.infoText = null;
this.codeJarAdminCopy=null;
filesStorage = new FilesStorage();
}
@ -58,7 +61,6 @@ public class DeploySave {
WorkspaceItem codeJarItem = filesStorage.createItemOnWorkspaceHowAdmin(ADMIN_USER, codeJarInputStream,
codeJar.getName(), CODE_JAR_DESCRIPTION, CODE_JAR_MIMETYPE, DESTINATION_FOLDER);
ItemDescription codeJarAdminCopy;
try {
codeJarAdminCopy = new ItemDescription(codeJarItem.getId(), codeJarItem.getName(),
codeJarItem.getOwner().getPortalLogin(), codeJarItem.getPath(), codeJarItem.getType().name());
@ -69,8 +71,7 @@ public class DeploySave {
}
createInfoText(codeJarAdminCopy);
createInfoText();
try {
int codeJarNamelenght = codeJar.getName().length();
@ -98,7 +99,7 @@ public class DeploySave {
}
private void createInfoText(ItemDescription codeJarAdminCopy) {
private void createInfoText() {
infoText = "Username: " + serviceCredentials.getUserName() + "\nFull Name: " + serviceCredentials.getFullName()
+ "\nEmail: " + serviceCredentials.getEmail() + "\n\nin VRE: " + serviceCredentials.getScope()
+ "\n\nhas requested to publish the algorithm: " + "\nLanguage: " + infoData.getLanguage()
@ -117,4 +118,10 @@ public class DeploySave {
return infoText;
}
public ItemDescription getCodeJarAdminCopy() {
return codeJarAdminCopy;
}
}

4
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/shared/Constants.java
View File

@ -35,8 +35,8 @@ public class Constants {
public static final String SAI_NAME = "SAIProfile";
public static final String SAI_CATEGORY = "SAI";
public static final String POOL_MANAGER_NAME = "dataminer-pool-manager";
public static final String POOL_MANAGER_CATEGORY = "DataAnalysis";
public static final String POOL_MANAGER_SERVICE_NAME = "dataminer-pool-manager";
public static final String POOL_MANAGER_SERVICE_CLASS = "DataAnalysis";
// Main Generator
public static final String PRE_INSTALLED = "Pre-Installed";