gCubeSystem
/
data-miner-manager-cl
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ref 11711: SAI - Add a simple text viewer so that log files created by Data Miner algorithms can be directly viewed in the browser 

https://support.d4science.org/issues/11711

Added service info[ticket #12594]
Added support to show log information [ticket #11711]
Added support to show files html, json, pdf, txt [ticket #17106]
Updated information show to the user when a computation is submitted [ticket #17030]

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/data-miner-manager-cl@181870 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Giancarlo Panichi 2019-09-30 16:40:34 +00:00
parent c9f3d35357
commit 99efe07a1c
7 changed files with 306 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
src/main/java/org/gcube/data/analysis/dataminermanagercl/server/dmservice/SClient4WPS.java
View File

@ -23,6 +23,7 @@ import org.gcube.data.analysis.dataminermanagercl.server.dmservice.wps.WPS2DM;
import org.gcube.data.analysis.dataminermanagercl.server.dmservice.wps.computationsvalue.ComputationValueBuilder;
import org.gcube.data.analysis.dataminermanagercl.server.is.InformationSystemUtils;
import org.gcube.data.analysis.dataminermanagercl.server.storage.StorageUtil;
import org.gcube.data.analysis.dataminermanagercl.server.uriresolver.UriResolverUtils;
import org.gcube.data.analysis.dataminermanagercl.server.util.ServiceCredentials;
import org.gcube.data.analysis.dataminermanagercl.shared.Constants;
import org.gcube.data.analysis.dataminermanagercl.shared.data.OutputData;
@ -47,6 +48,7 @@ import org.gcube.data.analysis.dataminermanagercl.shared.process.Operator;
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorCategory;
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorsClassification;
import org.gcube.data.analysis.dataminermanagercl.shared.service.ServiceInfo;
import org.gcube.data.analysis.dataminermanagercl.shared.service.ServiceInfoData;
import org.n52.wps.client.ExecuteRequestBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -214,7 +216,7 @@ public class SClient4WPS extends SClient {
@Override
public ServiceInfo getServiceInfo() throws ServiceException {
try {
LinkedHashMap<String, String> serviceProperties = new LinkedHashMap<>();
ArrayList<ServiceInfoData> serviceProperties = new ArrayList<>();
if (discoveredByIS) {
logger.info("DataMiner Service discovered by IS");
serviceProperties = InformationSystemUtils.retrieveServiceProperties(
@ -222,10 +224,11 @@ public class SClient4WPS extends SClient {
serviceCredentials.getScope());
} else {
logger.info("DataMiner Service not discovered by IS no properties available");
serviceProperties.put("Address", serviceAddress);
serviceProperties
.add(new ServiceInfoData("Address", serviceAddress, Constants.DEFAULT_SERVICE_INFO_CATEGORY));
}
ServiceInfo serviceInfo=new ServiceInfo(serviceProperties);
ServiceInfo serviceInfo = new ServiceInfo(serviceProperties);
logger.info("Service Info retrieved: " + serviceInfo);
return serviceInfo;
@ -772,7 +775,7 @@ public class SClient4WPS extends SClient {
*/
}
// TODO
//
@Override
public List<Parameter> getInputParameters(Operator operator) throws ServiceException {
try {
@ -1171,6 +1174,7 @@ public class SClient4WPS extends SClient {
}
}
// TODO
private void readOutputOfTypeData(Map<String, Parameter> outputParameters, Map<String, Resource> outputResource,
OutputDataType out, String outputID) {
String value = "";
@ -1198,10 +1202,10 @@ public class SClient4WPS extends SClient {
paramComplexData.getDescription(), paramComplexData.getValue());
outputResource.put(outputID, resource);
}
} else
} else {
// remove the element name, which is not useful
outputParameters.remove(outputID);
}
ComplexDataType cdt = out.getData().getComplexData();
LinkedHashMap<String, ResponseWPS> urls = retrieveURLsFromWPSResponse(cdt);
@ -1214,88 +1218,7 @@ public class SClient4WPS extends SClient {
logger.debug("ObjectParameter: " + objP);
outputParameters.put(key, objP);
if (responseWPS != null && responseWPS.getMimeType() != null) {
Resource resource;
switch (responseWPS.getMimeType()) {
case "image/bmp":
case "image/gif":
case "image/jpeg":
case "image/png":
resource = new ImageResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(), responseWPS.getMimeType());
outputResource.put(key, resource);
break;
case "text/csv":
if (responseWPS.getData() != null && !responseWPS.getData().isEmpty()) {
if (responseWPS.getData().startsWith("http:")
|| responseWPS.getData().startsWith("https:")
|| responseWPS.getData().startsWith("smp:")) {
Parameter complexParameter = outputParameters.get(outputID);
if (complexParameter instanceof FileParameter) {
FileParameter fileParameter = (FileParameter) complexParameter;
if (fileParameter.isNetcdf()) {
resource = new FileResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), true);
} else {
resource = new FileResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), false);
}
} else {
resource = new FileResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), false);
}
outputResource.put(key, resource);
} else {
resource = new ObjectResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData());
outputResource.put(key, resource);
}
} else {
resource = new ObjectResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData());
outputResource.put(key, resource);
}
break;
default:
if (responseWPS.getData() != null && !responseWPS.getData().isEmpty()) {
if (responseWPS.getData().startsWith("http:")
|| responseWPS.getData().startsWith("https:")
|| responseWPS.getData().startsWith("smp:")) {
Parameter complexParameter = outputParameters.get(outputID);
if (complexParameter instanceof FileParameter) {
FileParameter fileParameter = (FileParameter) complexParameter;
if (fileParameter.isNetcdf()) {
resource = new FileResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), true);
} else {
resource = new FileResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), false);
}
} else {
resource = new FileResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), false);
}
outputResource.put(key, resource);
} else {
resource = new ObjectResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData());
outputResource.put(key, resource);
}
} else {
resource = new ObjectResource(key, responseWPS.getDescription(),
responseWPS.getDescription(), responseWPS.getData());
outputResource.put(key, resource);
}
break;
}
extractFileResource(outputParameters, outputResource, outputID, key, responseWPS);
}
}
} else {
@ -1306,6 +1229,98 @@ public class SClient4WPS extends SClient {
}
}
private void extractFileResource(Map<String, Parameter> outputParameters, Map<String, Resource> outputResource,
String outputID, String key, ResponseWPS responseWPS) {
Resource resource;
UriResolverUtils uriResolverUtils = new UriResolverUtils();
switch (responseWPS.getMimeType()) {
case "image/bmp":
case "image/gif":
case "image/jpeg":
case "image/png":
resource = new ImageResource(key, responseWPS.getDescription(), responseWPS.getDescription(),
responseWPS.getData(), responseWPS.getMimeType());
outputResource.put(key, resource);
break;
case "text/csv":
if (responseWPS.getData() != null && !responseWPS.getData().isEmpty()) {
if (responseWPS.getData().startsWith("http:") || responseWPS.getData().startsWith("https:")
|| responseWPS.getData().startsWith("smp:")) {
String fileName = null;
try {
fileName = uriResolverUtils.getFileName(responseWPS.getData());
} catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
fileName = responseWPS.getDescription();
}
Parameter complexParameter = outputParameters.get(outputID);
if (complexParameter instanceof FileParameter) {
FileParameter fileParameter = (FileParameter) complexParameter;
if (fileParameter.isNetcdf()) {
resource = new FileResource(key, fileName, responseWPS.getDescription(),
responseWPS.getData(), responseWPS.getMimeType(), true);
} else {
resource = new FileResource(key, fileName, responseWPS.getDescription(),
responseWPS.getData(), responseWPS.getMimeType(), false);
}
} else {
resource = new FileResource(key, fileName, responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), false);
}
outputResource.put(key, resource);
} else {
resource = new ObjectResource(key, responseWPS.getDescription(), responseWPS.getDescription(),
responseWPS.getData());
outputResource.put(key, resource);
}
} else {
resource = new ObjectResource(key, responseWPS.getDescription(), responseWPS.getDescription(),
responseWPS.getData());
outputResource.put(key, resource);
}
break;
default:
if (responseWPS.getData() != null && !responseWPS.getData().isEmpty()) {
if (responseWPS.getData().startsWith("http:") || responseWPS.getData().startsWith("https:")
|| responseWPS.getData().startsWith("smp:")) {
String fileName = null;
try {
fileName = uriResolverUtils.getFileName(responseWPS.getData());
} catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
fileName = responseWPS.getDescription();
}
Parameter complexParameter = outputParameters.get(outputID);
if (complexParameter instanceof FileParameter) {
FileParameter fileParameter = (FileParameter) complexParameter;
if (fileParameter.isNetcdf()) {
resource = new FileResource(key, fileName, responseWPS.getDescription(),
responseWPS.getData(), responseWPS.getMimeType(), true);
} else {
resource = new FileResource(key, fileName, responseWPS.getDescription(),
responseWPS.getData(), responseWPS.getMimeType(), false);
}
} else {
resource = new FileResource(key, fileName, responseWPS.getDescription(), responseWPS.getData(),
responseWPS.getMimeType(), false);
}
outputResource.put(key, resource);
} else {
resource = new ObjectResource(key, responseWPS.getDescription(), responseWPS.getDescription(),
responseWPS.getData());
outputResource.put(key, resource);
}
} else {
resource = new ObjectResource(key, responseWPS.getDescription(), responseWPS.getDescription(),
responseWPS.getData());
outputResource.put(key, resource);
}
break;
}
}
@Override
public String cancelComputation(ComputationId computationId) throws ServiceException {
SClient4WPSSession wpsClient = null;

38
src/main/java/org/gcube/data/analysis/dataminermanagercl/server/is/InformationSystemUtils.java
View File

@ -1,6 +1,6 @@
package org.gcube.data.analysis.dataminermanagercl.server.is;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.List;
import org.gcube.common.resources.gcore.ServiceEndpoint;
@ -9,6 +9,7 @@ import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data.analysis.dataminermanagercl.shared.Constants;
import org.gcube.data.analysis.dataminermanagercl.shared.exception.ServiceException;
import org.gcube.data.analysis.dataminermanagercl.shared.service.ServiceInfoData;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
@ -23,6 +24,7 @@ import org.slf4j.LoggerFactory;
*/
public class InformationSystemUtils {
private static Logger logger = LoggerFactory.getLogger(InformationSystemUtils.class);
public static String retrieveServiceAddress(String category, String name, String scope) throws Exception {
@ -50,12 +52,12 @@ public class InformationSystemUtils {
String address = addresses.get(0);
int wpsWebProcessingServiceIndex = address.indexOf(Constants.WPSWebProcessingService);
String serviceAddress=null;
String serviceAddress = null;
if (wpsWebProcessingServiceIndex > 0) {
serviceAddress = address.substring(0, wpsWebProcessingServiceIndex);
} else {
logger.error("Invalid DataMiner service url retrieved: "+address);
throw new ServiceException("Invalid DataMiner service url retrieved: "+address);
logger.error("Invalid DataMiner service url retrieved: " + address);
throw new ServiceException("Invalid DataMiner service url retrieved: " + address);
}
logger.info("DataMiner service address found: " + serviceAddress);
return serviceAddress;
@ -67,11 +69,11 @@ public class InformationSystemUtils {
}
}
public static LinkedHashMap<String, String> retrieveServiceProperties(String category, String name, String scope)
public static ArrayList<ServiceInfoData> retrieveServiceProperties(String category, String name, String scope)
throws Exception {
try {
logger.debug("Retrieve Service Properties");
LinkedHashMap<String, String> serviceProperties = new LinkedHashMap<>();
ArrayList<ServiceInfoData> serviceProperties = new ArrayList<>();
if (scope == null || scope.length() == 0)
return serviceProperties;
@ -86,20 +88,32 @@ public class InformationSystemUtils {
if (accessPointList != null && !accessPointList.isEmpty()) {
AccessPoint accessPoint = accessPointList.get(0);
if(accessPoint.address()!=null&&!accessPoint.address().isEmpty()){
String accessPointAddress=accessPoint.address();
if (accessPoint.address() != null && !accessPoint.address().isEmpty()) {
String accessPointAddress = accessPoint.address();
int wpsWebProcessingServiceIndex = accessPointAddress.indexOf(Constants.WPSWebProcessingService);
if (wpsWebProcessingServiceIndex > 0) {
String address = accessPointAddress.substring(0, wpsWebProcessingServiceIndex);
serviceProperties.put("Address", address);
serviceProperties.add(new ServiceInfoData("Address", address, Constants.DEFAULT_SERVICE_INFO_CATEGORY));
}
}
if (accessPoint.properties() != null && !accessPoint.propertyMap().isEmpty()) {
for (String key : accessPoint.propertyMap().keySet()) {
Property property = accessPoint.propertyMap().get(key);
serviceProperties.put(property.name(), property.value());
if (property != null&&property.name() != null && !property.name().isEmpty()){
if(property.name().contains(":")) {
String[] propertyWithCategory=property.name().split(":");
if(propertyWithCategory.length>=2){
serviceProperties.add(new ServiceInfoData(propertyWithCategory[1],property.value(),propertyWithCategory[0]));
} else {
serviceProperties.add(new ServiceInfoData(property.name(), property.value(), Constants.DEFAULT_SERVICE_INFO_CATEGORY));
}
} else {
serviceProperties.add(new ServiceInfoData(property.name(), property.value(), Constants.DEFAULT_SERVICE_INFO_CATEGORY));
}
}
}
}

64
src/main/java/org/gcube/data/analysis/dataminermanagercl/server/uriresolver/UriResolverUtils.java Normal file
View File

@ -0,0 +1,64 @@
package org.gcube.data.analysis.dataminermanagercl.server.uriresolver;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
import org.gcube.data.analysis.dataminermanagercl.shared.exception.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Giancarlo Panichi
*
*
*/
public class UriResolverUtils {
private static Logger logger = LoggerFactory.getLogger(UriResolverUtils.class);
public String getFileName(String publicLink) throws ServiceException {
logger.info("Public Link: " + publicLink);
try {
URL urlObj = new URL(publicLink);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
connection.setRequestMethod("HEAD");
connection.setDoOutput(true);
logger.info("Response Code:" + connection.getResponseCode());
String fileName = null;
List<String> contentDisposition = connection.getHeaderFields().get("Content-Disposition");
logger.info("Content-Disposition: " + contentDisposition);
if (contentDisposition != null && !contentDisposition.isEmpty()) {
for (String value : contentDisposition) {
if (value != null && !value.isEmpty() && value.startsWith("attachment; filename=")) {
fileName = value.substring(21);
if (fileName != null && !fileName.isEmpty()) {
fileName = fileName.replace("\"", "");
logger.info("File name: " + fileName);
break;
}
}
}
}
if (fileName == null || fileName.isEmpty()) {
logger.error("Error retrieving filename from URI Resolver for public link: " + publicLink);
throw new ServiceException(
"Error retrieving filename from URI Resolver for public link: " + publicLink);
}
return fileName;
} catch (ServiceException e) {
logger.error(e.getLocalizedMessage(), e);
throw e;
} catch (Throwable e) {
logger.error("Error retrieving filename from URI Resolver for public link: " + publicLink);
logger.error(e.getLocalizedMessage(), e);
throw new ServiceException("Error retrieving filename from URI Resolver for url: " + publicLink, e);
}
}
}

8
src/main/java/org/gcube/data/analysis/dataminermanagercl/shared/Constants.java
View File

@ -14,9 +14,15 @@ public class Constants {
public final static String DEFAULT_SCOPE = "/gcube/devNext/NextNext";
public final static String DEFAULT_TOKEN = "";
public static final String DEFAULT_ROLE = "OrganizationMember";
//Data Miner Service Endpoint
public static final String DATAMINER_SERVICE_CATEGORY = "DataAnalysis";
public static final String DATA_MINER_SERVICE_NAME = "DataMiner";
//Default Service Info Category
public static final String DEFAULT_SERVICE_INFO_CATEGORY = "Generic";
// WPS Data Miner
public static final String WPSWebProcessingService = "WebProcessingService";
public static final String WPSCancelComputationServlet = "CancelComputationServlet";

10
src/main/java/org/gcube/data/analysis/dataminermanagercl/shared/service/ServiceInfo.java
View File

@ -1,7 +1,7 @@
package org.gcube.data.analysis.dataminermanagercl.shared.service;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.ArrayList;
/**
*
@ -12,22 +12,22 @@ import java.util.LinkedHashMap;
public class ServiceInfo implements Serializable {
private static final long serialVersionUID = 9046784925213335261L;
private LinkedHashMap<String, String> serviceProperties;
private ArrayList<ServiceInfoData> serviceProperties;
public ServiceInfo() {
super();
}
public ServiceInfo(LinkedHashMap<String, String> serviceProperties) {
public ServiceInfo(ArrayList<ServiceInfoData> serviceProperties) {
super();
this.serviceProperties = serviceProperties;
}
public LinkedHashMap<String, String> getServiceProperties() {
public ArrayList<ServiceInfoData> getServiceProperties() {
return serviceProperties;
}
public void setServiceProperties(LinkedHashMap<String, String> serviceProperties) {
public void setServiceProperties(ArrayList<ServiceInfoData> serviceProperties) {
this.serviceProperties = serviceProperties;
}

58
src/main/java/org/gcube/data/analysis/dataminermanagercl/shared/service/ServiceInfoData.java Normal file
View File

@ -0,0 +1,58 @@
package org.gcube.data.analysis.dataminermanagercl.shared.service;
import java.io.Serializable;
/**
*
* @author Giancarlo Panichi
*
*
*/
public class ServiceInfoData implements Serializable {
private static final long serialVersionUID = 4732143678328978038L;
private String key;
private String value;
private String category;
public ServiceInfoData() {
super();
}
public ServiceInfoData(String key, String value, String category) {
super();
this.key = key;
this.value = value;
this.category = category;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getCategory() {
return category;
}
public void setCategory(String category) {
this.category = category;
}
@Override
public String toString() {
return "ServiceInfoData [key=" + key + ", value=" + value + ", category=" + category + "]";
}
}

43
src/test/java/org/gcube/data/analysis/dataminermanagercl/test/URIResolverQueryTest.java Normal file
View File

@ -0,0 +1,43 @@
package org.gcube.data.analysis.dataminermanagercl.test;
import org.gcube.data.analysis.dataminermanagercl.server.uriresolver.UriResolverUtils;
import org.gcube.data.analysis.dataminermanagercl.shared.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import junit.framework.TestCase;
/**
*
* @author Giancarlo Panichi
*
*
*/
public class URIResolverQueryTest extends TestCase {
private static Logger logger = LoggerFactory.getLogger(URIResolverQueryTest.class);
public void testUriResolver() {
if (Constants.TEST_ENABLE) {
logger.info("Test URI Resolver");
try {
String publicLink = "https://data-dev.d4science.net/CyUd";
logger.debug("PublicLink: " + publicLink);
UriResolverUtils uriResolverUtils = new UriResolverUtils();
uriResolverUtils.getFileName(publicLink);
assertTrue("Success", true);
} catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
assertTrue("Error", false);
}
} else {
assertTrue("Success", true);
}
}
}