gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Aligned Trunk to Branch 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@174213 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Giancarlo Panichi 2018-11-19 09:03:18 +00:00
parent 60ccac1784
commit 34d131b900
5 changed files with 270 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java
View File

@ -388,7 +388,9 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
time("Ecological Engine Algorithm selection"); time("Ecological Engine Algorithm selection");
// adding service parameters to the configuration // adding service parameters to the configuration
LOGGER.info("5 - Adding Service parameters to the configuration"); LOGGER.info("5 - Adding Service parameters to the configuration");
inputsManager.addInputServiceParameters(getInputParameters(algorithm), infrastructureDialoguer); List<StatisticalType> dataminerInputParameters = getInputParameters(algorithm);
LOGGER.debug("Dataminer Algo Default InputParameters: "+dataminerInputParameters);
inputsManager.addInputServiceParameters(dataminerInputParameters, infrastructureDialoguer);
time("Service parameters added to the algorithm"); time("Service parameters added to the algorithm");
// merging wps with ecological engine parameters - modifies the // merging wps with ecological engine parameters - modifies the
// config // config
@ -396,7 +398,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
LOGGER.debug("Operator class is " + this.getClass().getCanonicalName()); LOGGER.debug("Operator class is " + this.getClass().getCanonicalName());
// build computation Data // build computation Data
currentComputation = new ComputationData(config.getTaskID(), config.getAgent(), "", "", startTime, "-", "0", config.getTaskID(), configManager.getUsername(), config.getGcubeScope(), this.getClass().getCanonicalName()); currentComputation = new ComputationData(config.getTaskID(), config.getAgent(), "", "", startTime, "-", "0", config.getTaskID(), configManager.getUsername(), config.getGcubeScope(), this.getClass().getCanonicalName());
inputsManager.mergeWpsAndEcologicalInputs(supportDatabaseInfo); inputsManager.mergeWpsAndEcologicalInputs(supportDatabaseInfo, dataminerInputParameters);
generatedInputTables = inputsManager.getGeneratedTables(); generatedInputTables = inputsManager.getGeneratedTables();
generatedFiles = inputsManager.getGeneratedInputFiles(); generatedFiles = inputsManager.getGeneratedInputFiles();
time("Setup and download of input parameters with tables creation"); time("Setup and download of input parameters with tables creation");
@ -447,13 +449,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
time("Output preparation for WPS document (no storage manager)"); time("Output preparation for WPS document (no storage manager)");
outputmanager.shutdown(); outputmanager.shutdown();
// delete all temporary tables LOGGER.debug("12 - Final Computation Output");
LOGGER.info("12 - Deleting possible generated temporary tables"); LOGGER.debug("Outputs: "+ outputs);
LOGGER.debug("Final Computation Output: " + outputs);
endTime = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis()); endTime = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis());
if (!isCancelled()) { if (!isCancelled()) {
LOGGER.debug("Save Computation Data");
saveComputationOnWS(inputsManager.getProvenanceData(), outputmanager.getProvenanceData(), agent, generatedFiles); saveComputationOnWS(inputsManager.getProvenanceData(), outputmanager.getProvenanceData(), agent, generatedFiles);
} else { } else {
LOGGER.debug("Computation interrupted - no update"); LOGGER.debug("Computation interrupted - no update");
@ -542,8 +544,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
} }
private void saveComputationOnWS(List<StoredData> inputData, List<StoredData> outputData, ComputationalAgent agent, List<File> generatedFiles) { private void saveComputationOnWS(List<StoredData> inputData, List<StoredData> outputData, ComputationalAgent agent, List<File> generatedFiles) {
LOGGER.debug("Save Computation On WS");
LOGGER.debug("InputData: "+inputData);
LOGGER.debug("OutputData: "+outputData);
LOGGER.debug("Agent: "+agent);
LOGGER.debug("Generated files: "+generatedFiles);
LOGGER.debug("Provenance manager started for operator " + this.getClass().getCanonicalName()); LOGGER.debug("Provenance manager started for operator " + this.getClass().getCanonicalName());
ComputationData computation = new ComputationData(config.getTaskID(), config.getAgent(), agent.getDescription(), agent.getInfrastructure().name(), startTime, endTime, "100", config.getTaskID(), config.getParam(ConfigurationManager.serviceUserNameParameterVariable), config.getGcubeScope(), this.getClass().getCanonicalName()); ComputationData computation = new ComputationData(config.getTaskID(), config.getAgent(), agent.getDescription(), agent.getInfrastructure().name(), startTime, endTime, "100", config.getTaskID(), config.getParam(ConfigurationManager.serviceUserNameParameterVariable), config.getGcubeScope(), this.getClass().getCanonicalName());
// post on WS // post on WS
DataspaceManager manager = new DataspaceManager(config, computation, inputData, outputData, generatedFiles); DataspaceManager manager = new DataspaceManager(config, computation, inputData, outputData, generatedFiles);

189
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -41,7 +41,7 @@ import org.slf4j.LoggerFactory;
public class InputsManager { public class InputsManager {
private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class); private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class);
LinkedHashMap<String, Object> inputs; LinkedHashMap<String, Object> inputs;
List<String> generatedTables; List<String> generatedTables;
@ -51,11 +51,11 @@ public class InputsManager {
String computationId; String computationId;
List<StoredData> provenanceData = new ArrayList<StoredData>(); List<StoredData> provenanceData = new ArrayList<StoredData>();
public List<StoredData> getProvenanceData() { public List<StoredData> getProvenanceData() {
return provenanceData; return provenanceData;
} }
public static String inputsSeparator = "\\|"; public static String inputsSeparator = "\\|";
public AlgorithmConfiguration getConfig() { public AlgorithmConfiguration getConfig() {
@ -92,30 +92,35 @@ public class InputsManager {
config.setParam("DatabaseURL", supportDatabaseInfo.url); config.setParam("DatabaseURL", supportDatabaseInfo.url);
} }
public void mergeWpsAndEcologicalInputs(DatabaseInfo supportDatabaseInfo) throws Exception { public void mergeWpsAndEcologicalInputs(DatabaseInfo supportDatabaseInfo,
List<StatisticalType> dataminerInputParameters) throws Exception {
LOGGER.debug("Merge WPS And Ecological Inputs");
// browse input parameters from WPS // browse input parameters from WPS
for (String inputName : inputs.keySet()) { for (String inputName : inputs.keySet()) {
Object input = inputs.get(inputName); Object input = inputs.get(inputName);
LOGGER.debug("Managing Input Parameter with Name "+ inputName); LOGGER.debug("Managing Input Parameter with Name " + inputName);
// case of simple input // case of simple input
if (input instanceof String) { if (input instanceof String) {
LOGGER.debug("Simple Input: "+ input); LOGGER.debug("Simple Input: " + input);
// manage lists // manage lists
String inputAlgoOrig = ((String) input).trim(); String inputAlgoOrig = ((String) input).trim();
String inputAlgo = ((String) input).trim().replaceAll(inputsSeparator, AlgorithmConfiguration.listSeparator); String inputAlgo = ((String) input).trim().replaceAll(inputsSeparator,
AlgorithmConfiguration.listSeparator);
LOGGER.debug("Simple Input Transformed: " + inputAlgo); LOGGER.debug("Simple Input Transformed: " + inputAlgo);
config.setParam(inputName, inputAlgo); config.setParam(inputName, inputAlgo);
saveInputData(inputName,inputName,inputAlgoOrig); saveInputData(inputName, inputName, inputAlgoOrig);
} }
// case of Complex Input // case of Complex Input
else if (input instanceof GenericFileData) { else if (input instanceof GenericFileData) {
LOGGER.debug("Complex Input: " + input); LOGGER.debug("Complex Input");
// retrieve payload // retrieve payload
GenericFileData files = ((GenericFileData) input); GenericFileData files = ((GenericFileData) input);
LOGGER.debug("GenericFileData: [fileExtension=" + files.getFileExtension() + ", mimeType="
+ files.getMimeType() + "]");
List<File> localfiles = getLocalFiles(files,inputName); List<File> localfiles = getLocalFiles(files, inputName, dataminerInputParameters);
String inputtables = ""; String inputtables = "";
int nfiles = localfiles.size(); int nfiles = localfiles.size();
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
@ -128,10 +133,11 @@ public class InputsManager {
if (inputTableTemplates.get(inputName) != null) { if (inputTableTemplates.get(inputName) != null) {
LOGGER.debug("Creating table: " + tableName); LOGGER.debug("Creating table: " + tableName);
createTable(tableName, tableFile, config, supportDatabaseInfo, inputTableTemplates.get(inputName)); createTable(tableName, tableFile, config, supportDatabaseInfo,
inputTableTemplates.get(inputName));
generatedTables.add(tableName); generatedTables.add(tableName);
} }
//case of non-table input file, e.g. FFANN // case of non-table input file, e.g. FFANN
else else
tableName = tableFile.getAbsolutePath(); tableName = tableFile.getAbsolutePath();
if (i > 0) if (i > 0)
@ -140,13 +146,13 @@ public class InputsManager {
inputtables += tableName; inputtables += tableName;
saveInputData(tableFile.getName(), inputName, tableFile.getAbsolutePath()); saveInputData(tableFile.getName(), inputName, tableFile.getAbsolutePath());
if (i>0) if (i > 0)
sb.append("|"); sb.append("|");
sb.append(tableFile.getName()); sb.append(tableFile.getName());
} }
sb.append("|"); sb.append("|");
if (nfiles>0) if (nfiles > 0)
saveInputData(inputName, inputName, sb.toString()); saveInputData(inputName, inputName, sb.toString());
// the only possible complex input is a table - check the WPS // the only possible complex input is a table - check the WPS
@ -157,15 +163,15 @@ public class InputsManager {
} }
public boolean isXML(String fileContent){ public boolean isXML(String fileContent) {
if (fileContent.startsWith("&lt;")) if (fileContent.startsWith("&lt;"))
return true; return true;
else else
return false; return false;
} }
public String readOneLine(String filename){ public String readOneLine(String filename) {
try { try {
BufferedReader in = new BufferedReader(new FileReader(new File(filename))); BufferedReader in = new BufferedReader(new FileReader(new File(filename)));
@ -173,7 +179,7 @@ public class InputsManager {
String vud = ""; String vud = "";
while ((line = in.readLine()) != null) { while ((line = in.readLine()) != null) {
if (line.trim().length()>0){ if (line.trim().length() > 0) {
vud = line.trim(); vud = line.trim();
break; break;
} }
@ -183,47 +189,54 @@ public class InputsManager {
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
} }
} }
public String inputNameFromHttpHeader(String url) throws Exception{ public String inputNameFromHttpHeader(String url) throws Exception {
LOGGER.debug("Search filename in http header from: " + url);
URL obj = new URL(url); URL obj = new URL(url);
URLConnection conn = obj.openConnection(); URLConnection conn = obj.openConnection();
String filename=null; String filename = null;
// get all headers // get all headers
Map<String, List<String>> map = conn.getHeaderFields(); Map<String, List<String>> map = conn.getHeaderFields();
LOGGER.debug("Getting file name from http header");
for (Map.Entry<String, List<String>> entry : map.entrySet()) { for (Map.Entry<String, List<String>> entry : map.entrySet()) {
String value = entry.getValue().toString(); String value = entry.getValue().toString();
if (value.toLowerCase().contains("filename=")){ LOGGER.debug("Header value: " + value);
LOGGER.debug("Searching in http header: found file name in header value {}",value); if (value.toLowerCase().contains("filename")) {
filename=value.substring(value.indexOf("=")+1); LOGGER.debug("Searching in http header: found file name in header value {}", value);
filename=filename.replace("\"", "").replace("]", ""); filename = value.substring(value.indexOf("=") + 1);
LOGGER.debug("Searching in http header: retrieved file name {}",filename); filename = filename.replace("\"", "").replace("]", "");
LOGGER.debug("Searching in http header: retrieved file name {}", filename);
break; break;
} }
} }
LOGGER.debug("Filename retrieved from http header: " + filename);
return filename; return filename;
} }
public List<File> getLocalFiles(GenericFileData files,String inputName) throws Exception { public List<File> getLocalFiles(GenericFileData files, String inputName,
List<StatisticalType> dataminerInputParameters) throws Exception {
LOGGER.debug("GetLocalFiles: [files: " + files + ", inputName: " + inputName + "]");
// download input // download input
List<File> filesList = new ArrayList<File>(); List<File> filesList = new ArrayList<File>();
File f = files.getBaseFile(false); File f = files.getBaseFile(false);
LOGGER.debug("Retrieving file content as a URL link: " + f.getAbsolutePath()); LOGGER.debug("Retrieving local files: " + f.getAbsolutePath());
//TODO DO NOT READ FILE INTO MEMORY // TODO DO NOT READ FILE INTO MEMORY
// read file content // read file content
String fileLink = readOneLine(f.getAbsolutePath()); String fileLink = readOneLine(f.getAbsolutePath());
LOGGER.debug("File link: {} ...",fileLink.substring(0,Math.min(fileLink.length(),10))); LOGGER.debug("Check File is link: {} ...", fileLink.substring(0, Math.min(fileLink.length(), 10)));
String fileName = ""; String fileName = "";
// case of a http link // case of a http link
if (fileLink!=null && (fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:"))) { if (fileLink != null
&& (fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:"))) {
// manage the case of multiple files // manage the case of multiple files
LOGGER.debug("Complex Input payload is link");
LOGGER.debug("Retrieving files from url: " + fileLink);
String[] remotefiles = fileLink.split(inputsSeparator); String[] remotefiles = fileLink.split(inputsSeparator);
for (String subfilelink : remotefiles) { for (String subfilelink : remotefiles) {
subfilelink = subfilelink.trim(); subfilelink = subfilelink.trim();
LOGGER.debug("Managing link: {}",subfilelink); LOGGER.debug("Managing link: {}", subfilelink);
if (subfilelink.length() == 0) if (subfilelink.length() == 0)
continue; continue;
InputStream is = null; InputStream is = null;
@ -234,15 +247,18 @@ public class InputsManager {
// retrieve payload: for test purpose only // retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink); String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}",fileNameTemp); LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp==null) if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s",inputName, computationId, FilenameUtils.getExtension(inputName)); fileName = String.format("%s_(%s).%s", inputName, computationId,
else FilenameUtils.getExtension(fileNameTemp));
fileName = String.format("%s_(%s).%s",fileNameTemp, computationId, FilenameUtils.getExtension(fileNameTemp)); } else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
LOGGER.debug("the name of the generated file is {}",fileName); FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName); File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of); FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos); IOUtils.copy(is, fos);
@ -254,35 +270,54 @@ public class InputsManager {
LOGGER.debug("Created local file: {}", of.getAbsolutePath()); LOGGER.debug("Created local file: {}", of.getAbsolutePath());
} }
} else { } else {
LOGGER.debug("Complex Input payload is the filelink"); LOGGER.debug("Complex Input payload is file");
fileName = f.getName(); fileName = f.getName();
LOGGER.debug("Retriving local input from file: {}", fileName); LOGGER.debug("Retrieving local input from file: {}", fileName);
String fileExt=null; String fileExt = null;
if (isXML(fileLink)) if (isXML(fileLink)) {
{
String xmlFile = f.getAbsolutePath(); String xmlFile = f.getAbsolutePath();
String csvFile = xmlFile+".csv"; String csvFile = xmlFile + ".csv";
LOGGER.debug("Transforming XML file into a csv: {} ",csvFile); LOGGER.debug("Transforming XML file into a csv: {} ", csvFile);
GML2CSV.parseGML(xmlFile, csvFile); GML2CSV.parseGML(xmlFile, csvFile);
LOGGER.debug("GML Parsed: {} [..]",readOneLine(csvFile)); LOGGER.debug("GML Parsed: {} [..]", readOneLine(csvFile));
f = new File(csvFile); f = new File(csvFile);
fileExt = "csv"; fileExt = "csv";
} } else {
else{ LOGGER.debug("The file is a csv: {}", f.getAbsolutePath());
LOGGER.debug("The file is a csv: {}",f.getAbsolutePath());
fileExt = FilenameUtils.getExtension(fileName); fileExt = FilenameUtils.getExtension(fileName);
} }
String absFile = new File(f.getParent(), String.format("%s_(%s).%s",inputName,computationId, fileExt)).getAbsolutePath(); LOGGER.debug("Retrieve default extension");
LOGGER.debug("Renaming to: "+absFile); String fileDefaultValue = null;
for (StatisticalType defaultInputParameter : dataminerInputParameters) {
if (defaultInputParameter.getName().compareTo(inputName) == 0) {
fileDefaultValue = defaultInputParameter.getDefaultValue();
break;
}
}
LOGGER.debug("Parameter default value retrieved: " + fileDefaultValue);
if (fileDefaultValue != null && !fileDefaultValue.isEmpty()) {
int lastPointIndex = fileDefaultValue.lastIndexOf(".");
if (lastPointIndex > -1 && lastPointIndex < (fileDefaultValue.length() - 1)) {
fileExt = fileDefaultValue.substring(lastPointIndex + 1);
LOGGER.debug("Default Extension retrieved: " + fileExt);
}
}
LOGGER.debug("Use extension: " + fileExt);
String absFile = new File(f.getParent(), String.format("%s_(%s).%s", inputName, computationId, fileExt))
.getAbsolutePath();
LOGGER.debug("Renaming to: " + absFile);
System.gc(); System.gc();
boolean renamed = f.renameTo(new File(absFile)); boolean renamed = f.renameTo(new File(absFile));
if (renamed) if (renamed)
f = new File(absFile); f = new File(absFile);
LOGGER.debug("The file has been renamed as : {} - {}",f.getAbsolutePath(),renamed); LOGGER.debug("The file has been renamed as : {} - {}", f.getAbsolutePath(), renamed);
filesList.add(f); filesList.add(f);
} }
@ -290,7 +325,8 @@ public class InputsManager {
return filesList; return filesList;
} }
public void createTable(String tableName, File tableFile, AlgorithmConfiguration config, DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception { public void createTable(String tableName, File tableFile, AlgorithmConfiguration config,
DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception {
// creating table // creating table
LOGGER.debug("Complex Input size after download: " + tableFile.length()); LOGGER.debug("Complex Input size after download: " + tableFile.length());
@ -316,11 +352,13 @@ public class InputsManager {
TableTemplatesMapper mapper = new TableTemplatesMapper(); TableTemplatesMapper mapper = new TableTemplatesMapper();
String createstatement = mapper.generateCreateStatement(tableName, templatename, tableStructure); String createstatement = mapper.generateCreateStatement(tableName, templatename, tableStructure);
LOGGER.debug("Creating table: " + tableName); LOGGER.debug("Creating table: " + tableName);
DatabaseUtils.createBigTable(true, tableName, supportDatabaseInfo.driver, supportDatabaseInfo.username, supportDatabaseInfo.password, supportDatabaseInfo.url, createstatement, dbConnection); DatabaseUtils.createBigTable(true, tableName, supportDatabaseInfo.driver, supportDatabaseInfo.username,
DatabaseUtils.createRemoteTableFromFile(tableFile.getAbsolutePath(), tableName, ",", true, supportDatabaseInfo.username, supportDatabaseInfo.password, supportDatabaseInfo.url); supportDatabaseInfo.password, supportDatabaseInfo.url, createstatement, dbConnection);
DatabaseUtils.createRemoteTableFromFile(tableFile.getAbsolutePath(), tableName, ",", true,
supportDatabaseInfo.username, supportDatabaseInfo.password, supportDatabaseInfo.url);
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("Error in database transaction " ,e); LOGGER.error("Error in database transaction ", e);
throw new Exception("Error in creating the table for " + tableName + ": " + e.getLocalizedMessage()); throw new Exception("Error in creating the table for " + tableName + ": " + e.getLocalizedMessage());
} finally { } finally {
DatabaseUtils.closeDBConnection(dbConnection); DatabaseUtils.closeDBConnection(dbConnection);
@ -376,7 +414,8 @@ public class InputsManager {
return structure.toString(); return structure.toString();
} }
public void addInputServiceParameters(List<StatisticalType> agentInputs, InfrastructureDialoguer infrastructureDialoguer) throws Exception { public void addInputServiceParameters(List<StatisticalType> agentInputs,
InfrastructureDialoguer infrastructureDialoguer) throws Exception {
// check and fullfil additional parameters // check and fullfil additional parameters
DatabaseInfo dbinfo = null; DatabaseInfo dbinfo = null;
@ -384,10 +423,10 @@ public class InputsManager {
for (StatisticalType type : agentInputs) { for (StatisticalType type : agentInputs) {
if (type instanceof PrimitiveType) { if (type instanceof PrimitiveType) {
if (((PrimitiveType) type).getType()==PrimitiveTypes.CONSTANT){ if (((PrimitiveType) type).getType() == PrimitiveTypes.CONSTANT) {
String constant = ""+((PrimitiveType) type).getDefaultValue(); String constant = "" + ((PrimitiveType) type).getDefaultValue();
config.setParam(type.getName(), constant); config.setParam(type.getName(), constant);
LOGGER.debug("Constant parameter: "+constant); LOGGER.debug("Constant parameter: " + constant);
} }
} }
if (type instanceof ServiceType) { if (type instanceof ServiceType) {
@ -400,10 +439,10 @@ public class InputsManager {
String value = ""; String value = "";
if (sp == ServiceParameters.RANDOMSTRING) if (sp == ServiceParameters.RANDOMSTRING)
value = "stat" + UUID.randomUUID().toString().replace("-", ""); value = "stat" + UUID.randomUUID().toString().replace("-", "");
else if (sp == ServiceParameters.USERNAME){ else if (sp == ServiceParameters.USERNAME) {
value = (String) inputs.get(ConfigurationManager.usernameParameter); value = (String) inputs.get(ConfigurationManager.usernameParameter);
LOGGER.debug("User name used by the client: "+value); LOGGER.debug("User name used by the client: " + value);
} }
LOGGER.debug("ServiceType Adding: (" + name + "," + value + ")"); LOGGER.debug("ServiceType Adding: (" + name + "," + value + ")");
config.setParam(name, value); config.setParam(name, value);
@ -444,8 +483,7 @@ public class InputsManager {
} }
private void saveInputData(String name, String description, String payload) {
private void saveInputData(String name, String description, String payload){
String id = name; String id = name;
DataProvenance provenance = DataProvenance.IMPORTED; DataProvenance provenance = DataProvenance.IMPORTED;
String creationDate = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis()); String creationDate = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis());
@ -453,18 +491,17 @@ public class InputsManager {
String type = "text/plain"; String type = "text/plain";
if (payload != null && (new File (payload).exists())) { if (payload != null && (new File(payload).exists())) {
if (payload.toLowerCase().endsWith(".csv") || payload.toLowerCase().endsWith(".txt")) { if (payload.toLowerCase().endsWith(".csv") || payload.toLowerCase().endsWith(".txt")) {
type = "text/csv"; type = "text/csv";
} else } else
type = "application/d4science"; type = "application/d4science";
} }
StoredData data = new StoredData(name, description, id, provenance, creationDate, operator, computationId, type,payload,config.getGcubeScope()); StoredData data = new StoredData(name, description, id, provenance, creationDate, operator, computationId, type,
payload, config.getGcubeScope());
provenanceData.add(data); provenanceData.add(data);
} }
} }

7
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/OutputsManager.java
View File

@ -140,7 +140,7 @@ public class OutputsManager {
outputs.put(pkey, ""); outputs.put(pkey, "");
} }
} }
LOGGER.debug("OutputsManager outputs "+outputs);
return outputs; return outputs;
} }
@ -256,10 +256,7 @@ public class OutputsManager {
String mimetype = info.getMimetype(); String mimetype = info.getMimetype();
String abstractStr = info.getAbstractStr(); String abstractStr = info.getAbstractStr();
LOGGER.debug("IOWPS Information: " + "name "+info.getName()+"," LOGGER.debug("IOWPS Information [name="+info.getName()+", abstr="+info.getAbstractStr()+", content="+info.getContent()+", def="+info.getDefaultVal()+"]");
+"abstr "+info.getAbstractStr()+","
+"content "+info.getContent()+","
+"def "+info.getDefaultVal()+",");
if ((abstractStr==null || abstractStr.trim().length()==0) && (payload!= null && payload.trim().length()>0)) if ((abstractStr==null || abstractStr.trim().length()==0) && (payload!= null && payload.trim().length()>0))
abstractStr = info.getName(); abstractStr = info.getName();

262
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -62,12 +62,15 @@ public class DataspaceManager implements Runnable {
public static String operator = "operator_name"; public static String operator = "operator_name";
public static String payload = "payload"; public static String payload = "payload";
public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List<StoredData> inputData, List<StoredData> outputData, List<File> generatedFiles) { public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List<StoredData> inputData,
List<StoredData> outputData, List<File> generatedFiles) {
this.config = config; this.config = config;
this.computation = computation; this.computation = computation;
this.inputData = inputData; this.inputData = inputData;
this.outputData = outputData; this.outputData = outputData;
this.generatedFiles = generatedFiles; this.generatedFiles = generatedFiles;
LOGGER.debug("DataspaceManager [config=" + config + ", computation=" + computation + ", inputData=" + inputData
+ ", outputData=" + outputData + ", generatedFiles=" + generatedFiles + "]");
} }
public void run() { public void run() {
@ -92,20 +95,23 @@ public class DataspaceManager implements Runnable {
StorageHubClient shc = new StorageHubClient(); StorageHubClient shc = new StorageHubClient();
FolderContainer root =shc.getWSRoot(); FolderContainer root = shc.getWSRoot();
List<ItemContainer<? extends Item>> dataminerItems = root.findByName(dataminerFolder).getContainers(); List<ItemContainer<? extends Item>> dataminerItems = root.findByName(dataminerFolder).getContainers();
FolderContainer dataminerFolderContainer; FolderContainer dataminerFolderContainer;
// manage folders: create the folders network // manage folders: create the folders network
if (dataminerItems.isEmpty()) { if (dataminerItems.isEmpty()) {
LOGGER.debug("Dataspace->Creating DataMiner main folder"); LOGGER.debug("Dataspace->Creating DataMiner main folder");
dataminerFolderContainer = root.newFolder(dataminerFolder, "A folder collecting DataMiner experiments data and computation information"); dataminerFolderContainer = root.newFolder(dataminerFolder,
//((WorkspaceFolder) root.find(dataminerFolder)).setSystemFolder(true); "A folder collecting DataMiner experiments data and computation information");
} else if (dataminerItems.size()>1) throw new Exception("found more than one dataminer folder (impossible!!!)"); // ((WorkspaceFolder)
else dataminerFolderContainer = (FolderContainer) dataminerItems.get(0); // root.find(dataminerFolder)).setSystemFolder(true);
} else if (dataminerItems.size() > 1)
throw new Exception("found more than one dataminer folder (impossible!!!)");
else
dataminerFolderContainer = (FolderContainer) dataminerItems.get(0);
if (dataminerFolderContainer.findByName(importedDataFolder).getContainers().isEmpty()) { if (dataminerFolderContainer.findByName(importedDataFolder).getContainers().isEmpty()) {
LOGGER.debug("Dataspace->Creating DataMiner imported data folder"); LOGGER.debug("Dataspace->Creating DataMiner imported data folder");
@ -118,37 +124,41 @@ public class DataspaceManager implements Runnable {
} }
if (dataminerFolderContainer.findByName(computationsFolder).getContainers().isEmpty()) { if (dataminerFolderContainer.findByName(computationsFolder).getContainers().isEmpty()) {
LOGGER.debug("Dataspace->Creating DataMiner computations folder"); LOGGER.debug("Dataspace->Creating DataMiner computations folder");
dataminerFolderContainer.newFolder(computationsFolder, "A folder collecting DataMiner computations information"); dataminerFolderContainer.newFolder(computationsFolder,
"A folder collecting DataMiner computations information");
} }
return dataminerFolderContainer; return dataminerFolderContainer;
} }
public String uploadData(StoredData data, FolderContainer destinationFolder) throws Exception { public String uploadData(StoredData data, FolderContainer destinationFolder) throws Exception {
return uploadData(data, destinationFolder, true); return uploadData(data, destinationFolder, true);
} }
public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception { public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception {
LOGGER.debug("Dataspace->Analysing " + data); LOGGER.debug("Dataspace->Analysing " + data);
// String filenameonwsString = WorkspaceUtil.getUniqueName(data.name, wsFolder); // String filenameonwsString = WorkspaceUtil.getUniqueName(data.name,
String filenameonwsString = data.name ; // wsFolder);
String filenameonwsString = data.name;
if (changename) if (changename)
filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload, data.type)); filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId,
getExtension(data.payload));
InputStream in = null; InputStream in = null;
String url = ""; String url = "";
try { try {
long size = 0; //long size = 0;
if (data.type.equals("text/csv")||data.type.equals("application/d4science")||data.type.equals("image/png")) { if (data.type.equals("text/csv") || data.type.equals("application/d4science")
|| data.type.equals("image/png")) {
if (new File(data.payload).exists() || !data.payload.startsWith("http")) { if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
LOGGER.debug("Dataspace->Uploading file {}", data.payload); LOGGER.debug("Dataspace->Uploading file {}", data.payload);
in = new FileInputStream(new File(data.payload)); in = new FileInputStream(new File(data.payload));
size = new File(data.payload).length(); //size = new File(data.payload).length();
} else { } else {
LOGGER.debug("Dataspace->Uploading via URL {}", data.payload); LOGGER.debug("Dataspace->Uploading via URL {}", data.payload);
int tries = 10; int tries = 10;
for (int i=0;i<tries;i++){ for (int i = 0; i < tries; i++) {
try { try {
URL urlc = new URL(data.payload); URL urlc = new URL(data.payload);
url = urlc.toString(); url = urlc.toString();
@ -156,21 +166,22 @@ public class DataspaceManager implements Runnable {
urlConnection.setConnectTimeout(10000); urlConnection.setConnectTimeout(10000);
urlConnection.setReadTimeout(10000); urlConnection.setReadTimeout(10000);
in = new BufferedInputStream(urlConnection.getInputStream()); in = new BufferedInputStream(urlConnection.getInputStream());
}catch(Exception ee){ } catch (Exception ee) {
LOGGER.warn("Dataspace->Retrying connection to {} number {} ",data.payload,(i+1),ee); LOGGER.warn("Dataspace->Retrying connection to {} number {} ", data.payload, (i + 1), ee);
in =null; in = null;
} }
if (in!=null) if (in != null)
break; break;
else else
Thread.sleep(10000); Thread.sleep(10000);
} }
} }
if (in==null) if (in == null)
throw new Exception("Impossible to open stream from "+data.payload); throw new Exception("Impossible to open stream from " + data.payload);
// LOGGER.debug("Dataspace->final file name on ws " + data.name+" description "+data.description); // LOGGER.debug("Dataspace->final file name on ws " +
// data.name+" description "+data.description);
LOGGER.debug("Dataspace->WS OP saving the following file on the WS " + filenameonwsString); LOGGER.debug("Dataspace->WS OP saving the following file on the WS " + filenameonwsString);
Map<String, Object> properties = new LinkedHashMap<String, Object>(); Map<String, Object> properties = new LinkedHashMap<String, Object>();
@ -198,7 +209,7 @@ public class DataspaceManager implements Runnable {
in.close(); in.close();
} catch (Exception e) { } catch (Exception e) {
LOGGER.debug("Dataspace->Error creating file {}", e.getMessage()); LOGGER.debug("Dataspace->Error creating file {}", e.getMessage());
//LOGGER.debug(e); // LOGGER.debug(e);
} }
LOGGER.debug("Dataspace->File created {}", filenameonwsString); LOGGER.debug("Dataspace->File created {}", filenameonwsString);
} else { } else {
@ -206,8 +217,8 @@ public class DataspaceManager implements Runnable {
url = data.payload; url = data.payload;
} }
} catch (Throwable e) { } catch (Throwable e) {
LOGGER.error("Dataspace->Could not retrieve input payload {} ",data.payload,e); LOGGER.error("Dataspace->Could not retrieve input payload {} ", data.payload, e);
//LOGGER.debug(e); // LOGGER.debug(e);
url = "payload was not made available for this dataset"; url = "payload was not made available for this dataset";
data.payload = url; data.payload = url;
} }
@ -216,24 +227,25 @@ public class DataspaceManager implements Runnable {
public List<String> uploadInputData(List<StoredData> inputData, FolderContainer dataminerFolder) throws Exception { public List<String> uploadInputData(List<StoredData> inputData, FolderContainer dataminerFolder) throws Exception {
LOGGER.debug("Dataspace->uploading input data; Number of data: {}", inputData.size()); LOGGER.debug("Dataspace->uploading input data; Number of data: {}", inputData.size());
FolderContainer destinationFolder = (FolderContainer) dataminerFolder.findByName(importedDataFolder).getContainers().get(0); FolderContainer destinationFolder = (FolderContainer) dataminerFolder.findByName(importedDataFolder)
.getContainers().get(0);
List<String> urls = new ArrayList<String>(); List<String> urls = new ArrayList<String>();
for (StoredData input : inputData) { for (StoredData input : inputData) {
List<ItemContainer<? extends Item>> items = null; List<ItemContainer<? extends Item>> items = null;
if (input.type.equals("text/csv")||input.type.equals("application/d4science")||input.type.equals("image/png")) if (input.type.equals("text/csv") || input.type.equals("application/d4science")
|| input.type.equals("image/png"))
items = destinationFolder.findByName(input.name).getContainers(); items = destinationFolder.findByName(input.name).getContainers();
if (items==null || items.isEmpty()){ if (items == null || items.isEmpty()) {
String url = uploadData(input, destinationFolder,false); String url = uploadData(input, destinationFolder, false);
LOGGER.debug("Dataspace->returning property {}",url); LOGGER.debug("Dataspace->returning property {}", url);
urls.add(url); urls.add(url);
} } else {
else{
FileContainer item = (FileContainer) items.get(0); FileContainer item = (FileContainer) items.get(0);
LOGGER.debug("Dataspace->Input item {} is already available in the input folder",input.name); LOGGER.debug("Dataspace->Input item {} is already available in the input folder", input.name);
String url = item.getPublicLink().toString(); String url = item.getPublicLink().toString();
LOGGER.debug("Dataspace->returning WS url {}",url); LOGGER.debug("Dataspace->returning WS url {}", url);
urls.add(url); urls.add(url);
} }
} }
@ -242,9 +254,11 @@ public class DataspaceManager implements Runnable {
return urls; return urls;
} }
public List<String> uploadOutputData(List<StoredData> outputData,FolderContainer dataminerFolder) throws Exception { public List<String> uploadOutputData(List<StoredData> outputData, FolderContainer dataminerFolder)
throws Exception {
LOGGER.debug("Dataspace->uploading output data; Number of data: " + outputData.size()); LOGGER.debug("Dataspace->uploading output data; Number of data: " + outputData.size());
FolderContainer destinationFolder = (FolderContainer)dataminerFolder.findByName(computedDataFolder).getContainers().get(0); FolderContainer destinationFolder = (FolderContainer) dataminerFolder.findByName(computedDataFolder)
.getContainers().get(0);
List<String> urls = new ArrayList<String>(); List<String> urls = new ArrayList<String>();
for (StoredData output : outputData) { for (StoredData output : outputData) {
String url = uploadData(output, destinationFolder); String url = uploadData(output, destinationFolder);
@ -254,21 +268,23 @@ public class DataspaceManager implements Runnable {
return urls; return urls;
} }
public void uploadComputationData(ComputationData computation, List<StoredData> inputData, List<StoredData> outputData, FolderContainer dataminerFolder) throws Exception { public void uploadComputationData(ComputationData computation, List<StoredData> inputData,
List<StoredData> outputData, FolderContainer dataminerFolder) throws Exception {
LOGGER.debug("Dataspace->uploading computation data"); LOGGER.debug("Dataspace->uploading computation data");
FolderContainer computationContainer = (FolderContainer) dataminerFolder.findByName(computationsFolder).getContainers().get(0); FolderContainer computationContainer = (FolderContainer) dataminerFolder.findByName(computationsFolder)
.getContainers().get(0);
// create a folder in here // create a folder in here
LOGGER.debug("Dataspace->Creating computation folder " + computation.id); LOGGER.debug("Dataspace->Creating computation folder " + computation.id);
String cfoldername = computation.id; String cfoldername = computation.id;
FolderContainer newcomputationFolder = null; FolderContainer newcomputationFolder = null;
try{ try {
newcomputationFolder = computationContainer.newFolder(cfoldername, computation.operatorDescription); newcomputationFolder = computationContainer.newFolder(cfoldername, computation.operatorDescription);
}catch(java.lang.ClassCastException e){ } catch (java.lang.ClassCastException e) {
LOGGER.debug("Dataspace->concurrency exception - deleting remaining item"); LOGGER.debug("Dataspace->concurrency exception - deleting remaining item");
deleteRunningComputationData(); deleteRunningComputationData();
newcomputationFolder = computationContainer.newFolder(cfoldername, computation.operatorDescription); newcomputationFolder = computationContainer.newFolder(cfoldername, computation.operatorDescription);
} }
//String itemType = "COMPUTATION"; // String itemType = "COMPUTATION";
// create IO folders // create IO folders
LOGGER.debug("Dataspace->creating IO folders under " + cfoldername); LOGGER.debug("Dataspace->creating IO folders under " + cfoldername);
@ -313,86 +329,88 @@ public class DataspaceManager implements Runnable {
LOGGER.debug("Dataspace->Adding input properties for " + ninput + " inputs"); LOGGER.debug("Dataspace->Adding input properties for " + ninput + " inputs");
for (int i = 1; i <= ninput; i++) { for (int i = 1; i <= ninput; i++) {
StoredData input = inputData.get(i - 1); StoredData input = inputData.get(i - 1);
if (input.payload.contains("|")){ if (input.payload.contains("|")) {
String payload = input .payload; String payload = input.payload;
LOGGER.debug("Dataspace->Managing complex input {} : {}",input.name, payload); LOGGER.debug("Dataspace->Managing complex input {} : {}", input.name, payload);
//delete the names that are not useful // delete the names that are not useful
for (StoredData subinput:inputData){ for (StoredData subinput : inputData) {
if (input.description.equals(subinput.description)){ if (input.description.equals(subinput.description)) {
payload = payload.replace(subinput.name,subinput.payload); payload = payload.replace(subinput.name, subinput.payload);
subinput.name=null; subinput.name = null;
} }
} }
input.name = null; input.name = null;
//delete last pipe character // delete last pipe character
if (payload.endsWith("|")) if (payload.endsWith("|"))
payload = payload.substring(0,payload.length()-1); payload = payload.substring(0, payload.length() - 1);
LOGGER.debug("Dataspace->Complex input after processing "+payload); LOGGER.debug("Dataspace->Complex input after processing " + payload);
properties.put("input" + i + "_" + input.description, payload); properties.put("input" + i + "_" + input.description, payload);
input.payload=payload; input.payload = payload;
} }
} }
for (int i = 1; i <= ninput; i++) { for (int i = 1; i <= ninput; i++) {
StoredData input = inputData.get(i - 1); StoredData input = inputData.get(i - 1);
if (input.name!=null){ if (input.name != null) {
properties.put(String.format("input%d_%s",i, input.name), inputurls.get(i - 1)); properties.put(String.format("input%d_%s", i, input.name), inputurls.get(i - 1));
} }
} }
LOGGER.debug("Dataspace->Adding output properties for " + noutput + " outputs"); LOGGER.debug("Dataspace->Adding output properties for " + noutput + " outputs");
for (int i = 1; i <= noutput; i++) { for (int i = 1; i <= noutput; i++) {
properties.put(String.format("output%d_%s",i,outputData.get(i - 1).name), outputurls.get(i - 1)); properties.put(String.format("output%d_%s", i, outputData.get(i - 1).name), outputurls.get(i - 1));
} }
LOGGER.debug("Dataspace->Properties of the folder: {} ", properties ); LOGGER.debug("Dataspace->Properties of the folder: {} ", properties);
LOGGER.debug("Dataspace->Saving properties to ProvO XML file {} outputs",noutput); LOGGER.debug("Dataspace->Saving properties to ProvO XML file {} outputs", noutput);
/* /*
* XStream xstream = new XStream(); String xmlproperties = xstream.toXML(properties); * XStream xstream = new XStream(); String xmlproperties =
* xstream.toXML(properties);
*/ */
try { try {
String xmlproperties = ProvOGenerator.toProvO(computation, inputData, outputData); String xmlproperties = ProvOGenerator.toProvO(computation, inputData, outputData);
File xmltosave = new File(config.getPersistencePath(), "prov_o_" + UUID.randomUUID()); File xmltosave = new File(config.getPersistencePath(), "prov_o_" + UUID.randomUUID());
FileTools.saveString(xmltosave.getAbsolutePath(), xmlproperties, true, "UTF-8"); FileTools.saveString(xmltosave.getAbsolutePath(), xmlproperties, true, "UTF-8");
try(InputStream sis = new FileInputStream(xmltosave)){ try (InputStream sis = new FileInputStream(xmltosave)) {
newcomputationFolder.uploadFile(sis,computation.id + ".xml", computation.operatorDescription); newcomputationFolder.uploadFile(sis, computation.id + ".xml", computation.operatorDescription);
} }
xmltosave.delete(); xmltosave.delete();
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("Dataspace->Failed creating ProvO XML file ",e); LOGGER.error("Dataspace->Failed creating ProvO XML file ", e);
} }
/* /*
List<String> scopes = new ArrayList<String>(); * List<String> scopes = new ArrayList<String>();
scopes.add(config.getGcubeScope()); * scopes.add(config.getGcubeScope());
ws.createGcubeItem(computation.id, computation.operatorDescription, scopes, computation.user, itemType, properties, newcomputationFolder.getId()); * ws.createGcubeItem(computation.id, computation.operatorDescription,
* scopes, computation.user, itemType, properties,
* newcomputationFolder.getId());
*/ */
newcomputationFolder.setMetadata(new Metadata(properties)); newcomputationFolder.setMetadata(new Metadata(properties));
LOGGER.debug("Dataspace->finished uploading computation data"); LOGGER.debug("Dataspace->finished uploading computation data");
} }
public String buildCompositePayload(List<StoredData> inputData,String payload, String inputName){ public String buildCompositePayload(List<StoredData> inputData, String payload, String inputName) {
for (StoredData input:inputData){ for (StoredData input : inputData) {
if (inputName.equals(input.description)){ if (inputName.equals(input.description)) {
payload = payload.replace(input.name,input.payload); payload = payload.replace(input.name, input.payload);
} }
} }
return payload; return payload;
} }
public void writeProvenance(ComputationData computation, List<StoredData> inputData, List<StoredData> outputData)
public void writeProvenance(ComputationData computation, List<StoredData> inputData, List<StoredData> outputData) throws Exception { throws Exception {
LOGGER.debug("Dataspace->connecting to Workspace"); LOGGER.debug("Dataspace->connecting to Workspace");
LOGGER.debug("Dataspace->create folders network"); LOGGER.debug("Dataspace->create folders network");
FolderContainer dataminerFolder = createFoldersNetwork(); FolderContainer dataminerFolder = createFoldersNetwork();
@ -412,19 +430,18 @@ public class DataspaceManager implements Runnable {
try { try {
deleteRunningComputationData(); deleteRunningComputationData();
} catch (Exception e) { } catch (Exception e) {
LOGGER.debug("Dataspace->impossible to delete running computation : {} ",e.getMessage()); LOGGER.debug("Dataspace->impossible to delete running computation : {} ", e.getMessage());
} }
StorageHubClient shc = new StorageHubClient();
// LOGGER.debug("Dataspace->create folders network"); // LOGGER.debug("Dataspace->create folders network");
FolderContainer folderContainer = createFoldersNetwork(); FolderContainer folderContainer = createFoldersNetwork();
FolderContainer computationsContainer = (FolderContainer) folderContainer.findByName(computationsFolder).getContainers().get(0); FolderContainer computationsContainer = (FolderContainer) folderContainer.findByName(computationsFolder)
// LOGGER.debug("Dataspace->Creating computation item " + computation.id+" with status"+computation.status); .getContainers().get(0);
// LOGGER.debug("Dataspace->Creating computation item " +
// computation.id+" with status"+computation.status);
String itemType = "COMPUTATION"; String itemType = "COMPUTATION";
// write a computation item for the computation // write a computation item for the computation
Map<String, Object> properties = new LinkedHashMap<String, Object>(); Map<String, Object> properties = new LinkedHashMap<String, Object>();
properties.put(computation_id, computation.id); properties.put(computation_id, computation.id);
@ -452,23 +469,22 @@ public class DataspaceManager implements Runnable {
computationsContainer.newGcubeItem(gcubeItem); computationsContainer.newGcubeItem(gcubeItem);
LOGGER.debug("Dataspace->finished uploading computation data"); LOGGER.debug("Dataspace->finished uploading computation data");
} }
public String getStatus(String status){ public String getStatus(String status) {
double statusD = 0; double statusD = 0;
try{ try {
statusD = Double.parseDouble(status); statusD = Double.parseDouble(status);
}catch(Exception e){ } catch (Exception e) {
return status; return status;
} }
if (statusD==100) if (statusD == 100)
return "completed"; return "completed";
else if (statusD==-2) else if (statusD == -2)
return "error"; return "error";
else if (statusD==-1) else if (statusD == -1)
return "cancelled"; return "cancelled";
else else
return status; return status;
@ -479,65 +495,69 @@ public class DataspaceManager implements Runnable {
LOGGER.debug("Dataspace->deleting computation item"); LOGGER.debug("Dataspace->deleting computation item");
LOGGER.debug("Dataspace->connecting to Workspace"); LOGGER.debug("Dataspace->connecting to Workspace");
StorageHubClient shc = new StorageHubClient(); StorageHubClient shc = new StorageHubClient();
FolderContainer dataminerContainer = (FolderContainer) shc.getWSRoot().findByName(dataminerFolder).getContainers().get(0); FolderContainer dataminerContainer = (FolderContainer) shc.getWSRoot().findByName(dataminerFolder)
FolderContainer computationContainer = (FolderContainer) dataminerContainer.findByName(computationsFolder).getContainers().get(0); .getContainers().get(0);
FolderContainer computationContainer = (FolderContainer) dataminerContainer.findByName(computationsFolder)
.getContainers().get(0);
LOGGER.debug("Dataspace->removing computation data"); LOGGER.debug("Dataspace->removing computation data");
List<ItemContainer<? extends Item>> wi = computationContainer.findByName(computation.id).getContainers(); List<ItemContainer<? extends Item>> wi = computationContainer.findByName(computation.id).getContainers();
if (wi.isEmpty()){ if (wi.isEmpty()) {
for (ItemContainer<? extends Item> container : wi) for (ItemContainer<? extends Item> container : wi)
container.delete(); container.delete();
} } else
else LOGGER.debug("Dataspace->Warning Could not find {} under {}", computation.id,
LOGGER.debug("Dataspace->Warning Could not find {} under {}",computation.id, computationContainer.get().getName()); computationContainer.get().getName());
List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id)
.getContainers();
List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers(); if (fileComputations.size() > 0)
if (fileComputations.size()>0)
fileComputations.get(0).delete(); fileComputations.get(0).delete();
/*TODO: ASK GIANPAOLO /*
int maxtries = 3; * TODO: ASK GIANPAOLO int maxtries = 3; int i =1; while
int i =1; * (ws.exists(computation.id,computationsFolderWs.getId()) &&
while (ws.exists(computation.id,computationsFolderWs.getId()) && i<maxtries){ * i<maxtries){
LOGGER.debug("Dataspace->computation data still exist... retrying "+i); * LOGGER.debug("Dataspace->computation data still exist... retrying "+i
Thread.sleep(1000); * ); Thread.sleep(1000);
computationsFolderWs.find(computation.id).remove(); * computationsFolderWs.find(computation.id).remove(); i++; }
i++; */
}*/
LOGGER.debug("Dataspace->finished removing computation data "); LOGGER.debug("Dataspace->finished removing computation data ");
} }
public static String getExtension(String payload, String type){ // TODO
String extension = ""; public static String getExtension(String payload) {
if (type.toLowerCase().equals("text/plain")){} LOGGER.debug("DataSpace->Get Extension from: " + payload);
else if (payload.toLowerCase().startsWith("http")){ String extension="";
if (payload.toLowerCase().startsWith("http")) {
try { try {
URL obj= new URL(payload); URL obj = new URL(payload);
URLConnection conn = obj.openConnection(); URLConnection conn = obj.openConnection();
// get all headers // get all headers
Map<String, List<String>> map = conn.getHeaderFields(); Map<String, List<String>> map = conn.getHeaderFields();
for (Map.Entry<String, List<String>> entry : map.entrySet()) { for (Map.Entry<String, List<String>> entry : map.entrySet()) {
String value = entry.getValue().toString(); String value = entry.getValue().toString();
if (value.toLowerCase().contains("filename=")){ LOGGER.debug("Header value: " + value);
LOGGER.debug("DataSpace->Searching in http header: found "+value); if (value.toLowerCase().contains("filename")) {
extension = value.substring(value.lastIndexOf("."),value.lastIndexOf("\"")); LOGGER.debug("DataSpace->Searching in http header: found " + value);
extension = value.substring(value.lastIndexOf("."), value.lastIndexOf("\""));
break;
} }
} }
conn.getInputStream().close(); conn.getInputStream().close();
} catch (Exception e) { } catch (Exception e) {
LOGGER.warn("DataSpace->Error in the payload http link ",e); LOGGER.warn("DataSpace->Error in the payload http link ", e);
} }
} } else {
else {
File paylFile = new File(payload); File paylFile = new File(payload);
if (paylFile.exists()){ if (paylFile.exists()) {
String paylname = paylFile.getName(); String paylname = paylFile.getName();
extension = paylname.substring(paylname.lastIndexOf(".")); extension = paylname.substring(paylname.lastIndexOf("."));
} }
} }
LOGGER.debug("DataSpace->Extension retrieved: " + extension);
return extension; return extension;
} }

1
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/ProvOGenerator.java
View File

@ -2,7 +2,6 @@ package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping.datasp
import java.io.StringReader; import java.io.StringReader;
import java.io.StringWriter; import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilder;