From 1b1117997ea7d26e9d97d3d764bc802e1139946f Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Wed, 20 Nov 2019 15:43:27 +0000 Subject: [PATCH] ref 18096: DataMiner - HTML file does not open in view widget but in a new browser window https://support.d4science.org/issues/18096 Fixed Content-Type in output files git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@182280 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 26 ++- pom.xml | 19 +- .../synchserver/mapping/OutputsManager.java | 217 +++++++++++------- 3 files changed, 159 insertions(+), 103 deletions(-) diff --git a/distro/changelog.xml b/distro/changelog.xml index 20ddd53..50c138e 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,24 +1,36 @@ - + + Fixed Content-Type support for files in the results of + computations[ticket #18096] + + Fixed https link for output parameter [ticket #17659] - + Updated https support [ticket #13024] - + Added https support [ticket #13024] - + added the right extension on output file lock file created on execution - + added accounting on algorithm execution - + service interface classes moved to wps project - + First Release \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2ba7d4d..b52a789 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.dataanalysis dataminer - 1.5.8-SNAPSHOT + 1.5.9-SNAPSHOT dataminer An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard. @@ -125,25 +125,14 @@ 4.11 test - + javassist javassist 3.12.1.GA - + + org.gcube.common storagehub-client-library diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/OutputsManager.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/OutputsManager.java index fcf109a..9fca807 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/OutputsManager.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/OutputsManager.java @@ -21,7 +21,7 @@ import org.slf4j.LoggerFactory; public class OutputsManager { private static Logger LOGGER = LoggerFactory.getLogger(OutputsManager.class); - + private AlgorithmConfiguration config; private List generatedFiles = new ArrayList(); @@ -29,7 +29,7 @@ public class OutputsManager { private IClient storageclient; private String computationsession; private List provenanceData = new ArrayList(); - + public List getProvenanceData() { return provenanceData; } @@ -37,7 +37,7 @@ public class OutputsManager { public List getGeneratedData() { return generatedFiles; } - + public List getGeneratedFiles() { return generatedFiles; } @@ -46,12 +46,13 @@ public class OutputsManager { return generatedTables; } - public OutputsManager(AlgorithmConfiguration config,String computationsession) { + public OutputsManager(AlgorithmConfiguration config, String computationsession) { this.config = config; - this.computationsession=computationsession; + this.computationsession = computationsession; } - public LinkedHashMap createOutput(StatisticalType prioroutput, StatisticalType posterioroutput) throws Exception { + public LinkedHashMap createOutput(StatisticalType prioroutput, StatisticalType posterioroutput) + throws Exception { LinkedHashMap outputs = new LinkedHashMap(); @@ -66,10 +67,10 @@ public class OutputsManager { StatisticalTypeToWPSType postconverter = new StatisticalTypeToWPSType(); postconverter.convert2WPSType(posterioroutput, false, config); generatedFiles.addAll(postconverter.getGeneratedFiles()); - LOGGER.debug("Generated Files "+generatedFiles); + LOGGER.debug("Generated Files " + generatedFiles); generatedTables.addAll(postconverter.getGeneratedTables()); - LOGGER.debug("Generated Tables "+generatedFiles); - + LOGGER.debug("Generated Tables " + generatedFiles); + LinkedHashMap postOutput = postconverter.outputSet; LinkedHashMap ndoutput = new LinkedHashMap(); @@ -102,21 +103,23 @@ public class OutputsManager { if (ConfigurationManager.useStorage()) { if (postInfo.getLocalMachineContent() != null) { // return the url from storage manager - String storageurl = uploadFileOnStorage(postInfo.getLocalMachineContent(), postInfo.getMimetype()); + String storageurl = uploadFileOnStorage(postInfo.getLocalMachineContent(), + postInfo.getMimetype()); postInfo.setContent(storageurl); } } /* - else if (postInfo.getLocalMachineContent() != null) { - String url = ""; - LOGGER.debug("Reference URL: " + url); - outputs.put(okey, url); - } - else*/ + * else if (postInfo.getLocalMachineContent() != null) { String + * url = ""; LOGGER.debug("Reference URL: " + + * url); outputs.put(okey, url); } else + */ if (info != null) { LOGGER.debug("Found a corresponding output: " + okey); outputs.put(okey, postInfo.getContent()); - //add link to the file also among the non deterministic output + // add link to the file also among the non deterministic + // output if (postInfo.getLocalMachineContent() != null) { ndoutput.put(okey, postInfo); } @@ -129,23 +132,23 @@ public class OutputsManager { System.gc(); } - + XmlObject ndxml = generateNonDeterministicOutput(ndoutput); outputs.put("non_deterministic_output", ndxml); - - //safety check for declared output, i.e. a priori output - for (String pkey:priorOutput.keySet()){ - if (outputs.get(pkey)==null){ - LOGGER.debug("Safety check: adding empty string for " + pkey+ " of type "+priorOutput.get(pkey).getClassname()); + + // safety check for declared output, i.e. a priori output + for (String pkey : priorOutput.keySet()) { + if (outputs.get(pkey) == null) { + LOGGER.debug("Safety check: adding empty string for " + pkey + " of type " + + priorOutput.get(pkey).getClassname()); outputs.put(pkey, ""); } } - LOGGER.debug("OutputsManager outputs "+outputs); + LOGGER.debug("OutputsManager outputs " + outputs); return outputs; } - - private void saveProvenanceData(IOWPSInformation info){ + private void saveProvenanceData(IOWPSInformation info) { String name = info.getName(); String id = info.getName(); DataProvenance provenance = DataProvenance.COMPUTED; @@ -153,59 +156,97 @@ public class OutputsManager { String operator = config.getAgent(); String computationId = computationsession; String type = info.getMimetype(); -/* if (info.getLocalMachineContent() != null) { - type = StoredType.DATA; - } - */ + /* + * if (info.getLocalMachineContent() != null) { type = StoredType.DATA; + * } + */ String payload = info.getContent(); - - StoredData data = new StoredData(name, info.getAbstractStr(),id, provenance, creationDate, operator, computationId, type,payload,config.getGcubeScope()); - + + StoredData data = new StoredData(name, info.getAbstractStr(), id, provenance, creationDate, operator, + computationId, type, payload, config.getGcubeScope()); + provenanceData.add(data); } - + private void prepareForStoring() { LOGGER.debug("Preparing storage client"); - //String scope = config.getGcubeScope(); - //ScopeProvider.instance.set(scope); + // String scope = config.getGcubeScope(); + // ScopeProvider.instance.set(scope); String serviceClass = "WPS"; String serviceName = "wps.synch"; String owner = config.getParam(ConfigurationManager.serviceUserNameParameterVariable); - storageclient = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, MemoryType.VOLATILE).getClient(); + storageclient = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, MemoryType.VOLATILE) + .getClient(); LOGGER.debug("Storage client ready"); } - - + private String uploadFileOnStorage(String localfile, String mimetype) throws Exception { LOGGER.debug("Storing->Start uploading on storage the following file: " + localfile); File localFile = new File(localfile); - String remotef = "/wps_synch_output/" +config.getAgent()+"/"+computationsession+"/"+ localFile.getName(); - storageclient.put(true).LFile(localfile).RFile(remotef); - + String remotef = "/wps_synch_output/" + config.getAgent() + "/" + computationsession + "/" + + localFile.getName(); + String contentType=retrieveContentType(localfile); + LOGGER.debug("Retrieved Content-Type: "+contentType); + if(contentType==null||contentType.isEmpty()){ + storageclient.put(true).LFile(localfile).RFile(remotef); + } else { + storageclient.put(true,contentType).LFile(localfile).RFile(remotef); + } String url = storageclient.getHttpsUrl().RFile(remotef); - + /* - if (config.getGcubeScope().startsWith("/gcube")) - url = "http://data-d.d4science.org/uri-resolver/smp?smp-uri=" + url + "&fileName=" + localFile.getName() + "&contentType=" + mimetype; - else - url = "http://data.d4science.org/uri-resolver/smp?smp-uri=" + url+ "&fileName=" + localFile.getName() + "&contentType=" + mimetype; - */ + * if (config.getGcubeScope().startsWith("/gcube")) url = + * "http://data-d.d4science.org/uri-resolver/smp?smp-uri=" + url + + * "&fileName=" + localFile.getName() + "&contentType=" + mimetype; else + * url = "http://data.d4science.org/uri-resolver/smp?smp-uri=" + url+ + * "&fileName=" + localFile.getName() + "&contentType=" + mimetype; + */ LOGGER.info("Storing->Uploading finished - URL: " + url); return url; } - + + private String retrieveContentType(String fileName) { + String contentType=null; + if (fileName != null && !fileName.isEmpty()) { + String fileNameLowerCase = fileName.toLowerCase(); + if (fileNameLowerCase.endsWith(".html") || fileNameLowerCase.endsWith(".htm")) { + contentType="text/html"; + } else { + if (fileNameLowerCase.endsWith(".pdf")) { + contentType="application/pdf"; + } else { + if (fileNameLowerCase.endsWith(".log") || fileNameLowerCase.endsWith(".txt")) { + contentType="text/plain"; + } else { + if (fileNameLowerCase.endsWith(".json")) { + contentType="application/json"; + } else { + } + } + } + } + } + return contentType; + } + public String cleanTagString(String tag) { return tag.replace(" ", "_").replaceAll("[\\]\\[!\"#$%&'()*+,\\./:;<=>?@\\^`{|}~-]", ""); } - public XmlObject generateNonDeterministicOutputPlain(LinkedHashMap ndoutput) throws Exception { - String XMLString = "\n" + " \n"; + public XmlObject generateNonDeterministicOutputPlain(LinkedHashMap ndoutput) + throws Exception { + String XMLString = "\n" + + " \n"; for (String key : ndoutput.keySet()) { IOWPSInformation info = ndoutput.get(key); String payload = info.getContent(); String mimetype = info.getMimetype(); - XMLString += " " + " \n" + " \n" + " " + mimetype + "\n" + " \n"; + XMLString += " " + " \n" + " \n" + + " " + mimetype + "\n" + " \n"; } XMLString += " \n" + "\n"; @@ -219,16 +260,21 @@ public class OutputsManager { return xmlData; } - public XmlObject generateNonDeterministicOutputCollection(LinkedHashMap ndoutput) throws Exception { - String XMLString = "" + - "\n\n" + " \n" + - " \n"; - + public XmlObject generateNonDeterministicOutputCollection(LinkedHashMap ndoutput) + throws Exception { + String XMLString = "" + + "\n\n" + " \n" + + " \n"; + for (String key : ndoutput.keySet()) { IOWPSInformation info = ndoutput.get(key); String payload = info.getContent(); String mimetype = info.getMimetype(); - XMLString += " " + " \n" + " \n" + " " + mimetype + "\n" + " \n"; + XMLString += " " + " \n" + " \n" + + " " + mimetype + "\n" + " \n"; } XMLString += " \n" + " \n\n"; @@ -241,37 +287,46 @@ public class OutputsManager { return xmlData; } - + public XmlObject generateNonDeterministicOutput(LinkedHashMap ndoutput) throws Exception { - - if (ndoutput.size()==0) + + if (ndoutput.size() == 0) return null; - - String XMLString = "" + - "\n\n"; + + String XMLString = "" + + "\n\n"; int count = 0; for (String key : ndoutput.keySet()) { IOWPSInformation info = ndoutput.get(key); String payload = info.getContent(); String mimetype = info.getMimetype(); String abstractStr = info.getAbstractStr(); - - LOGGER.debug("IOWPS Information [name="+info.getName()+", abstr="+info.getAbstractStr()+", content="+info.getContent()+", def="+info.getDefaultVal()+"]"); - - if ((abstractStr==null || abstractStr.trim().length()==0) && (payload!= null && payload.trim().length()>0)) + + LOGGER.debug("IOWPS Information [name=" + info.getName() + ", abstr=" + info.getAbstractStr() + ", content=" + + info.getContent() + ", def=" + info.getDefaultVal() + "]"); + + if ((abstractStr == null || abstractStr.trim().length() == 0) + && (payload != null && payload.trim().length() > 0)) abstractStr = info.getName(); else if (abstractStr == null) abstractStr = ""; - - //geospatialized -// XMLString += " " + "0,0"+ " \n" + " \n" + " " + mimetype + "\n" + " \n"; - XMLString += " " + " \n" + " \n" + " " + mimetype + "\n" + " \n"; + + // geospatialized + // XMLString += " " + + // "0,0"+ + // " \n" + // + " \n" + " " + + // mimetype + "\n" + " \n"; + XMLString += " " + " \n" + " \n" + " " + mimetype + + "\n" + " \n"; count++; } XMLString += " \n"; - - - + LOGGER.debug("Non deterministic output: " + XMLString); XmlObject xmlData = XmlObject.Factory.newInstance(); @@ -281,12 +336,12 @@ public class OutputsManager { return xmlData; } - - public void shutdown(){ - try{ - storageclient.close(); - }catch(Exception e){ - + + public void shutdown() { + try { + storageclient.close(); + } catch (Exception e) { + } } }