ref 11711: SAI - Add a simple text viewer so that log files created by Data Miner algorithms can be directly viewed in the browser

https://support.d4science.org/issues/11711

Added service info[ticket #12594]
Added support to show log information [ticket #12594]
Added support to show files html, json, pdf, txt [ticket #17106]
Updated information show to the user when a computation is submitted [ticket #17030]

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/data-miner-manager-cl@181866 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Giancarlo Panichi 2019-09-27 17:42:37 +00:00
parent ccf4656027
commit d6fe11322c
6 changed files with 226 additions and 72 deletions

View File

@ -1,45 +1,47 @@
<ReleaseNotes>
<Changeset component="${groupId}.${artifactId}.1-8-0"
date="2019-06-14">
<Changeset component="${groupId}.${artifactId}.1-8-0" date="2019-10-01">
<Change>Added service info [ticket #12594]</Change>
<Change>Added support to show log information [ticket #12594]</Change>
<Change>Added support to show files html, json, pdf, txt [ticket
#17106]</Change>
<Change>Updated information show to the user when a computation is
submitted [ticket #17030]</Change>
<Change>Added Item Id support [ticket #16503]</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-7-0"
date="2019-04-01">
<Change>Added location and zoom support [ticket #11708]</Change>
<Change>Added coordinates EPSG:4326 and EPSG:3857 support [ticket #11710]</Change>
<Change>Updated ComputationId to support token in process url [ticket
#17030]
</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-6-0"
date="2019-01-31">
<Changeset component="${groupId}.${artifactId}.1-7-0" date="2019-04-01">
<Change>Added location and zoom support [ticket #11708]</Change>
<Change>Added coordinates EPSG:4326 and EPSG:3857 support [ticket
#11710]
</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-6-0" date="2019-01-31">
<Change>Updated to support get operator by id with refresh</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-5-0"
date="2018-10-01">
<Changeset component="${groupId}.${artifactId}.1-5-0" date="2018-10-01">
<Change>Updated to support StorageHub[ticket #11720]</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-4-0"
date="2018-06-01">
<Changeset component="${groupId}.${artifactId}.1-4-0" date="2018-06-01">
<Change>Updated to support netcdf files</Change>
<Change>Added support for operators refresh[ticket #11741]</Change>
<Change>Added support for operators refresh[ticket #11741]</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-3-0"
date="2017-06-12">
<Changeset component="${groupId}.${artifactId}.1-3-0" date="2017-06-12">
<Change>Support Java 8 compatibility [ticket #8471]</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-2-0"
date="2017-05-01">
<Changeset component="${groupId}.${artifactId}.1-2-0" date="2017-05-01">
<Change>Updated to support DataMinerManagerWidget [ticket #6078]
</Change>
<Change>Fixed load balancing [ticket #7576]</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-1-0"
date="2017-03-20">
<Changeset component="${groupId}.${artifactId}.1-1-0" date="2017-03-20">
<Change>Updated Monitor interface</Change>
<Change>Added encoded parameters in equivalent http request [ticket
#7167]
</Change>
</Changeset>
<Changeset component="${groupId}.${artifactId}.1-0-0"
date="2017-02-14">
<Changeset component="${groupId}.${artifactId}.1-0-0" date="2017-02-14">
<Change>first release</Change>
</Changeset>
</ReleaseNotes>

View File

@ -11,6 +11,7 @@ import org.gcube.data.analysis.dataminermanagercl.shared.parameters.Parameter;
import org.gcube.data.analysis.dataminermanagercl.shared.process.ComputationStatus;
import org.gcube.data.analysis.dataminermanagercl.shared.process.Operator;
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorsClassification;
import org.gcube.data.analysis.dataminermanagercl.shared.service.ServiceInfo;
/**
*
@ -161,6 +162,15 @@ public abstract class SClient implements Serializable {
*/
public abstract ComputationId resubmitComputation(Map<String, String> computationProperties) throws Exception;
/**
* Retrieve service info
*
* @return service info
* @throws Exception
* exception
*/
public abstract ServiceInfo getServiceInfo() throws Exception;
@Override
public String toString() {
return "SClient";

View File

@ -46,6 +46,7 @@ import org.gcube.data.analysis.dataminermanagercl.shared.process.ComputationStat
import org.gcube.data.analysis.dataminermanagercl.shared.process.Operator;
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorCategory;
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorsClassification;
import org.gcube.data.analysis.dataminermanagercl.shared.service.ServiceInfo;
import org.n52.wps.client.ExecuteRequestBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -85,6 +86,10 @@ public class SClient4WPS extends SClient {
// private static final int OPERATOR_BRIEF_DESCRIPTION_MAX_LENGHT = 170;
private static final String OTHERS = "OTHERS";
private boolean discoveredByIS = false;
private ServiceCredentials serviceCredentials;
private String serviceAddress;
private String wpsToken;
private String wpsUser;
// private String wpsServiceURL;
@ -108,6 +113,8 @@ public class SClient4WPS extends SClient {
*/
public SClient4WPS(ServiceCredentials serviceCredentials) throws ServiceException {
super();
this.discoveredByIS = true;
this.serviceCredentials = serviceCredentials;
process = new HashMap<>();
runningProcess = new HashMap<>();
if (serviceCredentials == null) {
@ -129,33 +136,22 @@ public class SClient4WPS extends SClient {
} else {
wpsUser = userName;
}
List<String> serviceAddress;
try {
serviceAddress = InformationSystemUtils.retrieveServiceAddress(Constants.DATAMINER_SERVICE_CATEGORY,
Constants.DATA_MINER_SERVICE_NAME, serviceCredentials.getScope());
} catch (Exception e) {
logger.error("Error retrieving service address: " + e.getLocalizedMessage());
e.printStackTrace();
logger.error("Error retrieving service address: " + e.getLocalizedMessage(), e);
throw new ServiceException(e.getLocalizedMessage(), e);
}
logger.debug("Service Address retrieved:" + serviceAddress);
if (serviceAddress == null || serviceAddress.size() < 1) {
if (serviceAddress == null || serviceAddress.isEmpty()) {
logger.error("No DataMiner service address available!");
throw new ServiceException("No DataMiner service address available!");
} else {
logger.info("DataMiner service address found: " + serviceAddress.get(0));
wpsProcessingServlet = serviceAddress.get(0);
int wpsWebProcessingServiceIndex = wpsProcessingServlet.indexOf(Constants.WPSWebProcessingService);
if (wpsWebProcessingServiceIndex > 0) {
String wpsServiceUrl = wpsProcessingServlet.substring(0, wpsWebProcessingServiceIndex);
wpsCancelComputationServlet = wpsServiceUrl + Constants.WPSCancelComputationServlet;
logger.debug("Cancel computation servlet: " + wpsCancelComputationServlet);
} else {
logger.error("Cancel computation servlet not available!");
throw new ServiceException("Cancel computation servlet not available!");
}
logger.info("DataMiner service address found: " + serviceAddress);
wpsProcessingServlet = serviceAddress + Constants.WPSWebProcessingService;
wpsCancelComputationServlet = serviceAddress + Constants.WPSCancelComputationServlet;
}
@ -167,14 +163,17 @@ public class SClient4WPS extends SClient {
*
* @param serviceCredentials
* sevice credentials
* @param serviceAddressUrl
* @param serviceAddress
* valid url for example:
* <span>http://dataminer1-devnext.d4science.org/wps/</span>
* @throws ServiceException
* ServiceException
*/
public SClient4WPS(ServiceCredentials serviceCredentials, String serviceAddressUrl) throws ServiceException {
public SClient4WPS(ServiceCredentials serviceCredentials, String serviceAddress) throws ServiceException {
super();
discoveredByIS = false;
this.serviceCredentials = serviceCredentials;
this.serviceAddress = serviceAddress;
process = new HashMap<>();
runningProcess = new HashMap<>();
if (serviceCredentials == null) {
@ -197,15 +196,14 @@ public class SClient4WPS extends SClient {
wpsUser = userName;
}
logger.debug("Service Address:" + serviceAddressUrl);
if (serviceAddressUrl == null || serviceAddressUrl.isEmpty()) {
logger.error("Invalid DataMiner service address: " + serviceAddressUrl);
throw new ServiceException("Invalid DataMiner service address: " + serviceAddressUrl);
logger.debug("DataMiner Service Address:" + serviceAddress);
if (serviceAddress == null || serviceAddress.isEmpty()) {
logger.error("Invalid DataMiner service address: " + serviceAddress);
throw new ServiceException("Invalid DataMiner service address: " + serviceAddress);
} else {
logger.debug("DataMiner service address: " + serviceAddressUrl);
// http://dataminer1-devnext.d4science.org/wps/WebProcessingService
wpsProcessingServlet = serviceAddressUrl + Constants.WPSWebProcessingService;
wpsCancelComputationServlet = serviceAddressUrl + Constants.WPSCancelComputationServlet;
wpsProcessingServlet = serviceAddress + Constants.WPSWebProcessingService;
wpsCancelComputationServlet = serviceAddress + Constants.WPSCancelComputationServlet;
}
@ -213,6 +211,31 @@ public class SClient4WPS extends SClient {
}
@Override
public ServiceInfo getServiceInfo() throws ServiceException {
try {
LinkedHashMap<String, String> serviceProperties = new LinkedHashMap<>();
if (discoveredByIS) {
logger.info("DataMiner Service discovered by IS");
serviceProperties = InformationSystemUtils.retrieveServiceProperties(
Constants.DATAMINER_SERVICE_CATEGORY, Constants.DATA_MINER_SERVICE_NAME,
serviceCredentials.getScope());
} else {
logger.info("DataMiner Service not discovered by IS no properties available");
serviceProperties.put("Address", serviceAddress);
}
ServiceInfo serviceInfo=new ServiceInfo(serviceProperties);
logger.info("Service Info retrieved: " + serviceInfo);
return serviceInfo;
} catch (Exception e) {
logger.error("Error retrieving service properties: " + e.getLocalizedMessage(), e);
throw new ServiceException(e.getLocalizedMessage(), e);
}
}
private SClient4WPSSession createWPSClientSession() {
if (wpsClient == null) {
wpsClient = new SClient4WPSSession(wpsUser, wpsToken);
@ -860,6 +883,8 @@ public class SClient4WPS extends SClient {
throw new ServiceException("Invalid processLocation: " + processUrl);
}
processUrl = processUrl + "&gcube-token=" + wpsToken;
String equivalentRequest = extractEquivalentRequestForComputation(operator, equivalentRequestMap);
ComputationId computationId = new ComputationId(id, processUrl, operator.getId(), operator.getName(),
@ -978,8 +1003,13 @@ public class SClient4WPS extends SClient {
} else {
ComputationStatus computationStatus = null;
try {
String urlId = computationId.getUrlId();
int indexToken = urlId.indexOf("gcube-token");
if (indexToken > 0) {
urlId = urlId.substring(0, indexToken - 1);
}
responseObject = wpsClient.executeViaGET(computationId.getUrlId(), "");
responseObject = wpsClient.executeViaGET(urlId, "");
logger.debug("ComputationStatus ResponseObject: " + responseObject);
@ -1101,8 +1131,12 @@ public class SClient4WPS extends SClient {
outputParameters.put(outputParameter.getName(), outputParameter);
}
}
retrieveProcessOutput(computationId.getUrlId(), outputParameters, outputResource);
String urlId = computationId.getUrlId();
int indexToken = urlId.indexOf("gcube-token");
if (indexToken > 0) {
urlId = urlId.substring(0, indexToken - 1);
}
retrieveProcessOutput(urlId, outputParameters, outputResource);
return outputResource;
@ -1125,12 +1159,12 @@ public class SClient4WPS extends SClient {
logger.debug("Is Data");
readOutputOfTypeData(outputParameters, outputResource, out, outputID);
} else {
if(out.isSetReference()){
logger.debug("Is Reference: "+out.getReference());
if (out.isSetReference()) {
logger.debug("Is Reference: " + out.getReference());
logger.debug("This parameter is not recovered");
} else {
logger.debug("Unknow type, this parameter is not recovered");
}
}
}
@ -1147,8 +1181,8 @@ public class SClient4WPS extends SClient {
if (paramLiteral != null) {
paramLiteral.setValue(value);
logger.debug("Assigning value: " + value + " to output named: " + outputID);
Resource resource = new ObjectResource(outputID, paramLiteral.getName(),
paramLiteral.getDescription(), paramLiteral.getValue());
Resource resource = new ObjectResource(outputID, paramLiteral.getName(), paramLiteral.getDescription(),
paramLiteral.getValue());
outputResource.put(outputID, resource);
}
} else {
@ -1187,8 +1221,7 @@ public class SClient4WPS extends SClient {
case "image/jpeg":
case "image/png":
resource = new ImageResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType());
responseWPS.getDescription(), responseWPS.getData(), responseWPS.getMimeType());
outputResource.put(key, resource);
break;
case "text/csv":
@ -1349,6 +1382,8 @@ public class SClient4WPS extends SClient {
throw new ServiceException("Invalid processLocation: " + processUrl);
}
processUrl = processUrl + "&gcube-token=" + wpsToken;
String equivalentRequest = extractEquivalentRequestForResubmit(computationProperties, equivalentRequestMap);
ComputationId computationId = new ComputationId(id, processUrl, computationProperties.get("operator_id"),

View File

@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
public class SClient4WPSBuilder extends SClientBuilder {
private static Logger logger = LoggerFactory.getLogger(SClient4WPSBuilder.class);
private ServiceCredentials serviceCredendials;
private String serviceAddressUrl;
private String serviceAddress;
private boolean useUrl;
/**
@ -32,13 +32,13 @@ public class SClient4WPSBuilder extends SClientBuilder {
*
* @param serviceCredentials
* service credentials
* @param serviceAddressUrl
* @param serviceAddress
* valid url for example:
* <span>http://dataminer1-devnext.d4science.org/wps/</span>
*/
public SClient4WPSBuilder(ServiceCredentials serviceCredentials, String serviceAddressUrl) {
public SClient4WPSBuilder(ServiceCredentials serviceCredentials, String serviceAddress) {
this.serviceCredendials = serviceCredentials;
this.serviceAddressUrl = serviceAddressUrl;
this.serviceAddress = serviceAddress;
this.useUrl = true;
}
@ -50,8 +50,8 @@ public class SClient4WPSBuilder extends SClientBuilder {
SClient sClient;
if (useUrl) {
logger.debug("Use Url: " + useUrl);
logger.debug("DataMiner Service Address: " + serviceAddressUrl);
sClient = new SClient4WPS(serviceCredendials, serviceAddressUrl);
logger.debug("DataMiner Service Address: " + serviceAddress);
sClient = new SClient4WPS(serviceCredendials, serviceAddress);
} else {
logger.debug("Use Url: " + useUrl);
sClient = new SClient4WPS(serviceCredendials);

View File

@ -1,10 +1,14 @@
package org.gcube.data.analysis.dataminermanagercl.server.is;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data.analysis.dataminermanagercl.shared.Constants;
import org.gcube.data.analysis.dataminermanagercl.shared.exception.ServiceException;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
@ -21,11 +25,13 @@ public class InformationSystemUtils {
private static Logger logger = LoggerFactory.getLogger(InformationSystemUtils.class);
public static List<String> retrieveServiceAddress(String category, String name, String scope) throws Exception {
public static String retrieveServiceAddress(String category, String name, String scope) throws Exception {
try {
if (scope == null || scope.length() == 0)
return new ArrayList<String>();
logger.debug("Retrieve Service Address");
if (scope == null || scope.length() == 0) {
logger.error("No DataMiner service address retrieved, invalid scope requested: " + scope);
throw new ServiceException("No DataMiner service address retrieved, invalid scope requested: " + scope);
}
ScopeProvider.instance.set(scope);
@ -36,12 +42,74 @@ public class InformationSystemUtils {
DiscoveryClient<String> client = ICFactory.client();
List<String> addresses = client.submit(query);
return addresses;
logger.debug("Service Addresses retrieved:" + addresses);
if (addresses == null || addresses.isEmpty()) {
logger.error("No DataMiner service addresses available!");
throw new ServiceException("No DataMiner service address available!");
}
String address = addresses.get(0);
int wpsWebProcessingServiceIndex = address.indexOf(Constants.WPSWebProcessingService);
String serviceAddress=null;
if (wpsWebProcessingServiceIndex > 0) {
serviceAddress = address.substring(0, wpsWebProcessingServiceIndex);
} else {
logger.error("Invalid DataMiner service url retrieved: "+address);
throw new ServiceException("Invalid DataMiner service url retrieved: "+address);
}
logger.info("DataMiner service address found: " + serviceAddress);
return serviceAddress;
} catch (Throwable e) {
logger.error("Error in discovery DataMiner Service Endpoint in scope: " + scope);
logger.error("Error: " + e.getLocalizedMessage());
e.printStackTrace();
logger.error("Error: " + e.getLocalizedMessage(), e);
throw e;
}
}
public static LinkedHashMap<String, String> retrieveServiceProperties(String category, String name, String scope)
throws Exception {
try {
logger.debug("Retrieve Service Properties");
LinkedHashMap<String, String> serviceProperties = new LinkedHashMap<>();
if (scope == null || scope.length() == 0)
return serviceProperties;
ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + category + "'")
.addCondition("$resource/Profile/Name/text() eq '" + name + "'")
.setResult("$resource/Profile/AccessPoint");
DiscoveryClient<AccessPoint> client = ICFactory.clientFor(AccessPoint.class);
List<AccessPoint> accessPointList = client.submit(query);
if (accessPointList != null && !accessPointList.isEmpty()) {
AccessPoint accessPoint = accessPointList.get(0);
if(accessPoint.address()!=null&&!accessPoint.address().isEmpty()){
String accessPointAddress=accessPoint.address();
int wpsWebProcessingServiceIndex = accessPointAddress.indexOf(Constants.WPSWebProcessingService);
if (wpsWebProcessingServiceIndex > 0) {
String address = accessPointAddress.substring(0, wpsWebProcessingServiceIndex);
serviceProperties.put("Address", address);
}
}
if (accessPoint.properties() != null && !accessPoint.propertyMap().isEmpty()) {
for (String key : accessPoint.propertyMap().keySet()) {
Property property = accessPoint.propertyMap().get(key);
serviceProperties.put(property.name(), property.value());
}
}
}
logger.debug("Service Properties: " + serviceProperties);
return serviceProperties;
} catch (Throwable e) {
logger.error("Error in discovery DataMiner Service Endpoint Properties in scope: " + scope);
logger.error(e.getLocalizedMessage(), e);
throw e;
}
}

View File

@ -0,0 +1,39 @@
package org.gcube.data.analysis.dataminermanagercl.shared.service;
import java.io.Serializable;
import java.util.LinkedHashMap;
/**
*
* @author Giancarlo Panichi
*
*
*/
public class ServiceInfo implements Serializable {
private static final long serialVersionUID = 9046784925213335261L;
private LinkedHashMap<String, String> serviceProperties;
public ServiceInfo() {
super();
}
public ServiceInfo(LinkedHashMap<String, String> serviceProperties) {
super();
this.serviceProperties = serviceProperties;
}
public LinkedHashMap<String, String> getServiceProperties() {
return serviceProperties;
}
public void setServiceProperties(LinkedHashMap<String, String> serviceProperties) {
this.serviceProperties = serviceProperties;
}
@Override
public String toString() {
return "ServiceInfo [serviceProperties=" + serviceProperties + "]";
}
}