From 768ea5f2124d1c2b7e14261cbff98da905714af8 Mon Sep 17 00:00:00 2001 From: Lucio Lelii Date: Mon, 8 Jan 2018 11:53:50 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@161985 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 4 ++ pom.xml | 2 +- .../algorithms/AlgorithmClassification.java | 10 ---- .../InfrastructureDialoguer.java | 17 +++++++ .../AbstractEcologicalEngineMapper.java | 12 ++++- .../mapping/StatisticalTypeToWPSType.java | 28 +++++------ .../mapping/dataspace/DataspaceManager.java | 47 +++++++++++++++---- .../mapping/dataspace/StoredData.java | 11 +++++ .../synchserver/test/AlgorithmTest.java | 2 +- src/test/resources/AlgorithmTestURIs.txt | 2 +- 10 files changed, 98 insertions(+), 37 deletions(-) delete mode 100644 src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/algorithms/AlgorithmClassification.java diff --git a/distro/changelog.xml b/distro/changelog.xml index 882032c..67328df 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,4 +1,8 @@ + + added the right extension on output file + lock file created on execution + added accounting on algorithm execution diff --git a/pom.xml b/pom.xml index f0ae0ce..d9c29e7 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.dataanalysis dataminer - 1.5.1-SNAPSHOT + 1.5.2-SNAPSHOT dataminer An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard. diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/algorithms/AlgorithmClassification.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/algorithms/AlgorithmClassification.java deleted file mode 100644 index c1559cc..0000000 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/algorithms/AlgorithmClassification.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.algorithms; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -public @interface AlgorithmClassification { - - String value() default "Others"; -} diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java index 7cbe2ac..67699bd 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java @@ -74,6 +74,23 @@ public class InfrastructureDialoguer { LOGGER.debug("Found {} algorithms",resources.size()); return resources; + } + + public List getPrivateAlgorithmsInScope(String user) throws Exception{ + + LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader().getClass().getSimpleName()); + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/SecondaryType/string() eq 'StatisticalManagerAlgorithmPrivate' "); + query.addCondition(String.format("contains($resource/Profile/Body/privateusers/string(),'%s')",user)); + query.setResult("$resource/Profile/Name/text()"); + DiscoveryClient client = client(); + List resources = client.submit(query); + if (resources==null || resources.size()==0){ + throw new Exception("No resource named StatisticalManagerAlgorithmPrivate available in scope "+scope); + } + LOGGER.debug("Found {} algorithms",resources.size()); + + return resources; } diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java index a73860a..930dddd 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java @@ -3,6 +3,9 @@ package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping; import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -304,7 +307,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i observer.isStarted(this); LOGGER.info("classloader context in this thread is {}",Thread.currentThread().getContextClassLoader()); - + long startTimeLong = System.currentTimeMillis(); OperationResult operationResult = null; @@ -323,6 +326,11 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i InputsManager inputsManager = null; ConfigurationManager configManager = new ConfigurationManager(); // initializes parameters from file manageUserToken(); + Path dir = Paths.get(System.getProperty("java.io.tmpdir"), "dmlocks"); + if (!Files.exists(dir)) + dir = Files.createDirectory(dir); + Path lockFile = Files.createTempFile(dir, "dm", ".lck"); + LOGGER.info("lock file created {}",lockFile.toUri().toURL()); try { // wait for server resources to be available @@ -477,7 +485,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i cleanResources(); if (observer!=null) observer.isFinished(this); LOGGER.debug("All done - Computation Finished"); - + Files.deleteIfExists(lockFile); } } diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/StatisticalTypeToWPSType.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/StatisticalTypeToWPSType.java index 6dcba14..ab9ea86 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/StatisticalTypeToWPSType.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/StatisticalTypeToWPSType.java @@ -39,9 +39,9 @@ import org.slf4j.LoggerFactory; public class StatisticalTypeToWPSType { - + private static Logger logger = LoggerFactory.getLogger(StatisticalTypeToWPSType.class); - + public static Properties templates; static String ABSTRACT = "#ABSTRACT#"; static String TITLE = "#TITLE#"; @@ -51,9 +51,9 @@ public class StatisticalTypeToWPSType { static String ID = "#ID#"; static String IDMETHOD = "#IDMETHOD#"; static String METHOD_ORDER = "#ORDER_VALUE#"; - + private int orderValue = 0; - + public LinkedHashMap inputSet = new LinkedHashMap(); public LinkedHashMap outputSet = new LinkedHashMap(); public List generatedFiles = new ArrayList(); @@ -72,10 +72,9 @@ public class StatisticalTypeToWPSType { return; templates = new Properties(); - try(InputStream is = this.getClass().getClassLoader().getResourceAsStream("templates/classtemplate.properties")) - { - templates.load(is); - } + InputStream is = this.getClass().getClassLoader().getResourceAsStream("templates/classtemplate.properties"); + templates.load(is); + is.close(); } public StatisticalTypeToWPSType() throws Exception { @@ -179,9 +178,10 @@ public class StatisticalTypeToWPSType { logger.debug("The file exists! " + originalfile); filename = ((File) ptype.getContent()).getName(); String filenameDest = System.currentTimeMillis() + "_" + filename; + logger.debug("file destination for output is "+filenameDest); String destinationfile = new File(webpersistence, filenameDest).getAbsolutePath(); logger.debug("Copying file into a temporary file: " + destinationfile); - + FileManager.fileCopy(originalfile, destinationfile); content = config.getParam(ConfigurationManager.webpathVariable) + filenameDest; localcontent = destinationfile; @@ -320,11 +320,11 @@ public class StatisticalTypeToWPSType { return null; } else if (stype instanceof TablesList) { outputType = "csvFile"; - // outputType = "string"; +// outputType = "string"; String template = ((TablesList) stype).getTemplates().get(0).name(); abstractStr += " [a sequence of http links separated by | , each indicating a table in UTF-8 encoding following this template: " + mapper.linksMap.get(template) + "]"; mimeType = "text/csv"; - // mimeType = "text/plain"; +// mimeType = "text/plain"; } outputType += isinput ? "Input" : "Output"; @@ -369,9 +369,9 @@ public class StatisticalTypeToWPSType { fw.close(); } - - - + + + /* * enum a { A, B }; */ diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java index 595cdec..b89d5c3 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java @@ -6,9 +6,11 @@ import java.io.FileInputStream; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import org.gcube.common.homelibrary.home.Home; @@ -113,16 +115,12 @@ public class DataspaceManager implements Runnable { return uploadData(data, wsFolder, true); } public String uploadData(StoredData data, WorkspaceFolder wsFolder, boolean changename) throws Exception { - LOGGER.debug("Dataspace->Analysing " + data.name); + LOGGER.debug("Dataspace->Analysing " + data); // String filenameonwsString = WorkspaceUtil.getUniqueName(data.name, wsFolder); String filenameonwsString = data.name ; - if (changename){ - filenameonwsString = data.name + "_[" + data.computationId + "]";// ("_"+UUID.randomUUID()).replace("-", ""); - if (data.type.equals("text/csv")) - filenameonwsString+=".csv"; - else if (data.type.equals("image/png")) - filenameonwsString+=".png"; - } + if (changename) + filenameonwsString = data.name + "_[" + data.computationId + "]"+getExtension(data.payload, data.type);// ("_"+UUID.randomUUID()).replace("-", ""); + InputStream in = null; String url = ""; try { @@ -381,6 +379,7 @@ public class DataspaceManager implements Runnable { return payload; } + public void writeProvenance(ComputationData computation, List inputData, List outputData) throws Exception { LOGGER.debug("Dataspace->connecting to Workspace"); HomeManagerFactory factory = HomeLibrary.getHomeManagerFactory(); @@ -469,6 +468,7 @@ public class DataspaceManager implements Runnable { } public void deleteRunningComputationData() throws Exception { + LOGGER.debug("Dataspace->deleting computation item"); LOGGER.debug("Dataspace->connecting to Workspace"); HomeManagerFactory factory = HomeLibrary.getHomeManagerFactory(); @@ -503,4 +503,35 @@ public class DataspaceManager implements Runnable { LOGGER.debug("Dataspace->finished removing computation data - success "+!ws.exists(computation.id,computationsFolderWs.getId())); } + public static String getExtension(String payload, String type){ + String extension = ""; + if (type.toLowerCase().equals("text/plain")){} + else if (payload.toLowerCase().startsWith("http")){ + try { + URL obj= new URL(payload); + URLConnection conn = obj.openConnection(); + // get all headers + Map> map = conn.getHeaderFields(); + for (Map.Entry> entry : map.entrySet()) { + String value = entry.getValue().toString(); + if (value.toLowerCase().contains("filename=")){ + System.out.println("DataSpace->Searching in http header: found "+value); + extension = value.substring(value.lastIndexOf("."),value.lastIndexOf("\"")); + } + } + conn.getInputStream().close(); + } catch (Exception e) { + System.out.println("DataSpace->Error in the payload http link "+e.getMessage()); + } + } + else { + File paylFile = new File(payload); + if (paylFile.exists()){ + String paylname = paylFile.getName(); + extension = paylname.substring(paylname.lastIndexOf(".")); + } + } + return extension; + } + } diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/StoredData.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/StoredData.java index 38bb5f7..f5dabaf 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/StoredData.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/StoredData.java @@ -25,4 +25,15 @@ public class StoredData { String vre; String type; String payload; + + @Override + public String toString() { + return "StoredData [name=" + name + ", description=" + description + + ", id=" + id + ", provenance=" + provenance + + ", creationDate=" + creationDate + ", operator=" + operator + + ", computationId=" + computationId + ", vre=" + vre + + ", type=" + type + ", payload=" + payload + "]"; + } + + } diff --git a/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java b/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java index d761c81..fd21284 100644 --- a/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java +++ b/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java @@ -22,7 +22,7 @@ public class AlgorithmTest { @Test public void executeAlgorithmsFromFile() throws Exception{ - String env = "proto"; + String env = "dev"; Properties prop = new Properties(); prop.load(AlgorithmTest.class.getResourceAsStream("/test_params.properties")); diff --git a/src/test/resources/AlgorithmTestURIs.txt b/src/test/resources/AlgorithmTestURIs.txt index 0ce105b..ca3b895 100644 --- a/src/test/resources/AlgorithmTestURIs.txt +++ b/src/test/resources/AlgorithmTestURIs.txt @@ -173,7 +173,7 @@ {PROTOCOL}://{HOST}/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token={TOKEN}&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.WEB_APP_PUBLISHER&DataInputs=ZipFile={PROTOCOL}://goo.gl/dYQ089; #ECOPATH_WITH_ECOSIM {PROTOCOL}://{HOST}/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0&gcube-token={TOKEN}&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ECOPATH_WITH_ECOSIM -{PROTOCOL}://{HOST}/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token={TOKEN}&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ECOPATH_WITH_ECOSIM&DataInputs=Model File=http://data.d4science.org/eHFkNmhoSUwxMVpmcElhcUlmQUpWaWRGSjQzNkFXNElHbWJQNStIS0N6Yz0;Config File=http://data.d4science.org/ZGFWaGc4NjUrQmRmcElhcUlmQUpWbTNVQjhqdUV3OWdHbWJQNStIS0N6Yz0; +{PROTOCOL}://{HOST}/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token={TOKEN}&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ECOPATH_WITH_ECOSIM&DataInputs=Model File={PROTOCOL}://data.d4science.org/eHFkNmhoSUwxMVpmcElhcUlmQUpWaWRGSjQzNkFXNElHbWJQNStIS0N6Yz0;Config File={PROTOCOL}://data.d4science.org/ZGFWaGc4NjUrQmRmcElhcUlmQUpWbTNVQjhqdUV3OWdHbWJQNStIS0N6Yz0; #OCCURRENCES_MERGER {PROTOCOL}://{HOST}/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0&gcube-token={TOKEN}&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.generators.OCCURRENCES_MERGER {PROTOCOL}://{HOST}/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token={TOKEN}&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.generators.OCCURRENCES_MERGER&DataInputs=final_Table_Name=MergedOcc;leftTableName={PROTOCOL}://data.d4science.org/QkNIYVp1dm81bnhmcElhcUlmQUpWbzlqZWlDbXpmaytHbWJQNStIS0N6Yz0;rightTableName={PROTOCOL}://data.d4science.org/QkNIYVp1dm81bnhmcElhcUlmQUpWbzlqZWlDbXpmaytHbWJQNStIS0N6Yz0;longitudeColumn=decimalLongitude;latitudeColumn=decimalLatitude;recordedByColumn=recordedBy;scientificNameColumn=scientificName;eventDateColumn=eventDate;lastModificationColumn=modified;spatialTolerance=0.5;confidence=80;