gCubeSystem
/
dataminer
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Better management of Boolean types 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@129105 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2016-06-14 10:25:56 +00:00
parent fc73ba716c
commit 9ca4281883
3 changed files with 38 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/StatisticalTypeToWPSType.java
View File

@ -208,6 +208,7 @@ public class StatisticalTypeToWPSType {
case BOOLEAN:
outputType = "boolean";
mimeType = "text/plain";
defaultVal=defaultVal.toLowerCase();
break;
case IMAGES: {
// content = ptype.getContent();

43
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -71,6 +71,7 @@ public class DataspaceManager implements Runnable {
try {
deleteRunningComputationData();
} catch (Exception e) {
e.printStackTrace();
AnalysisLogger.getLogger().debug("Dataspace->No running computation available");
}
AnalysisLogger.getLogger().debug("Dataspace->Writing provenance information");
@ -91,9 +92,10 @@ public class DataspaceManager implements Runnable {
if (!ws.exists(dataminerFolder, root.getId())) {
AnalysisLogger.getLogger().debug("Dataspace->Creating DataMiner main folder");
root.createFolder(dataminerFolder, "A folder collecting DataMiner experiments data and computation information");
((WorkspaceFolder) root.find(dataminerFolder)).setSystemFolder(true);
}
WorkspaceFolder dataminerFolderWS = (WorkspaceFolder) root.find(dataminerFolder);
if (!ws.exists(importedDataFolder, dataminerFolderWS.getId())) {
AnalysisLogger.getLogger().debug("Dataspace->Creating DataMiner imported data folder");
dataminerFolderWS.createFolder(importedDataFolder, "A folder collecting DataMiner imported data");
@ -124,11 +126,13 @@ public class DataspaceManager implements Runnable {
InputStream in = null;
String url = "";
try {
long size = 0;
if (data.type.equals("text/csv")||data.type.equals("application/d4science")||data.type.equals("image/png")) {
if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
AnalysisLogger.getLogger().debug("Dataspace->Uploading file " + data.payload);
in = new FileInputStream(new File(data.payload));
size = new File(data.payload).length();
} else {
AnalysisLogger.getLogger().debug("Dataspace->Uploading via URL " + data.payload);
int tries = 10;
@ -168,7 +172,7 @@ public class DataspaceManager implements Runnable {
properties.put(data_type, data.type);
properties.put(payload, url);
FolderItem fileItem = WorkspaceUtil.createExternalFile(wsFolder, filenameonwsString, data.description, in,properties,data.type);
FolderItem fileItem = WorkspaceUtil.createExternalFile(wsFolder, filenameonwsString, data.description, in,properties,data.type,size);
//fileItem.getProperties().addProperties(properties);
AnalysisLogger.getLogger().debug("Dataspace->WS OP file saved on the WS " + filenameonwsString);
@ -252,7 +256,14 @@ public class DataspaceManager implements Runnable {
AnalysisLogger.getLogger().debug("Dataspace->Creating computation folder " + computation.id);
WorkspaceFolder cfolder = ((WorkspaceFolder) folderItem);
String cfoldername = computation.id;
WorkspaceFolder newcomputationFolder = cfolder.createFolder(cfoldername, computation.operatorDescription);
WorkspaceFolder newcomputationFolder = null;
try{
newcomputationFolder = cfolder.createFolder(cfoldername, computation.operatorDescription);
}catch(java.lang.ClassCastException e){
AnalysisLogger.getLogger().debug("Dataspace->concurrency exception - deleting remaining item");
deleteRunningComputationData();
newcomputationFolder = cfolder.createFolder(cfoldername, computation.operatorDescription);
}
String itemType = "COMPUTATION";
// create IO folders
@ -347,7 +358,7 @@ public class DataspaceManager implements Runnable {
File xmltosave = new File(config.getPersistencePath(), "prov_o_" + UUID.randomUUID());
FileTools.saveString(xmltosave.getAbsolutePath(), xmlproperties, true, "UTF-8");
InputStream sis = new FileInputStream(xmltosave);
WorkspaceUtil.createExternalFile(newcomputationFolder, computation.id + ".xml", computation.operatorDescription, "text/xml", sis);
WorkspaceUtil.createExternalFile(newcomputationFolder, computation.id + ".xml", computation.operatorDescription, sis,null,"text/xml",xmltosave.length());
sis.close();
xmltosave.delete();
} catch (Exception e) {
@ -403,6 +414,7 @@ public class DataspaceManager implements Runnable {
try {
deleteRunningComputationData();
} catch (Exception e) {
e.printStackTrace();
AnalysisLogger.getLogger().debug("Dataspace->impossible to delete running computation");
}
// AnalysisLogger.getLogger().debug("Dataspace->updating computation status");
@ -475,8 +487,25 @@ public class DataspaceManager implements Runnable {
WorkspaceFolder dataminerFolderWS = (WorkspaceFolder) root.find(dataminerFolder);
WorkspaceItem computationsFolderItem = dataminerFolderWS.find(computationsFolder);
AnalysisLogger.getLogger().debug("Dataspace->removing computation data");
((WorkspaceFolder) computationsFolderItem).find(computation.id).remove();
AnalysisLogger.getLogger().debug("Dataspace->finished removing computation data");
WorkspaceFolder computationsFolderWs = ((WorkspaceFolder) computationsFolderItem);
WorkspaceItem wi = computationsFolderWs.find(computation.id);
if (wi!=null){
AnalysisLogger.getLogger().debug("Dataspace->Found "+computation.id+" under "+computationsFolderWs.getName()+" - removing");
wi.remove();
}
else
AnalysisLogger.getLogger().debug("Dataspace->Warning Could not find "+computation.id+" under "+computationsFolderWs.getName());
int maxtries = 3;
int i =1;
while (ws.exists(computation.id,computationsFolderWs.getId()) && i<maxtries){
AnalysisLogger.getLogger().debug("Dataspace->computation data still exist... retrying "+i);
Thread.sleep(1000);
computationsFolderWs.find(computation.id).remove();
i++;
}
AnalysisLogger.getLogger().debug("Dataspace->finished removing computation data - success "+!ws.exists(computation.id,computationsFolderWs.getId()));
}
}

2
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/MultiThreadingCalls.java
View File

@ -29,7 +29,7 @@ public class MultiThreadingCalls {
// final URL urlToCall = new URL("http://"+host+"/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=4ccc2c35-60c9-4c9b-9800-616538d5d48b");
int nthreads = 50;
int nthreads = 100;
for (int i = 0; i < nthreads; i++) {
final int index = i+1;