gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@174113 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2018-11-12 17:16:15 +00:00
parent 26ad1e8cc9
commit 60ccac1784
2 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -218,7 +218,7 @@ public class InputsManager {
LOGGER.debug("File link: {} ...",fileLink.substring(0,Math.min(fileLink.length(),10)));
String fileName = "";
// case of a http link
if (fileLink!=null && fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:")) {
if (fileLink!=null && (fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:"))) {
// manage the case of multiple files
String[] remotefiles = fileLink.split(inputsSeparator);
for (String subfilelink : remotefiles) {
@ -256,8 +256,11 @@ public class InputsManager {
} else {
LOGGER.debug("Complex Input payload is the filelink");
fileName = f.getName();
LOGGER.debug("Retriving local input from file: {}", fileName);
String fileExt=null;
if (isXML(fileLink))
{
String xmlFile = f.getAbsolutePath();
@ -266,11 +269,14 @@ public class InputsManager {
GML2CSV.parseGML(xmlFile, csvFile);
LOGGER.debug("GML Parsed: {} [..]",readOneLine(csvFile));
f = new File(csvFile);
fileExt = "csv";
}
else{
LOGGER.debug("The file is a csv: {}",f.getAbsolutePath());
fileExt = FilenameUtils.getExtension(fileName);
}
String absFile = new File(String.format("%s_(%s).csv",f.getParent(),inputName,computationId)).getAbsolutePath();
String absFile = new File(f.getParent(), String.format("%s_(%s).%s",inputName,computationId, fileExt)).getAbsolutePath();
LOGGER.debug("Renaming to: "+absFile);
System.gc();
boolean renamed = f.renameTo(new File(absFile));

30
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -142,11 +142,11 @@ public class DataspaceManager implements Runnable {
if (data.type.equals("text/csv")||data.type.equals("application/d4science")||data.type.equals("image/png")) {
if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
LOGGER.debug("Dataspace->Uploading file " + data.payload);
LOGGER.debug("Dataspace->Uploading file {}", data.payload);
in = new FileInputStream(new File(data.payload));
size = new File(data.payload).length();
} else {
LOGGER.debug("Dataspace->Uploading via URL " + data.payload);
LOGGER.debug("Dataspace->Uploading via URL {}", data.payload);
int tries = 10;
for (int i=0;i<tries;i++){
try {
@ -197,12 +197,12 @@ public class DataspaceManager implements Runnable {
try {
in.close();
} catch (Exception e) {
LOGGER.debug("Dataspace->Error creating file " + e.getMessage());
LOGGER.debug("Dataspace->Error creating file {}", e.getMessage());
//LOGGER.debug(e);
}
LOGGER.debug("Dataspace->File created " + filenameonwsString);
LOGGER.debug("Dataspace->File created {}", filenameonwsString);
} else {
LOGGER.debug("Dataspace->String parameter " + data.payload);
LOGGER.debug("Dataspace->String parameter {}", data.payload);
url = data.payload;
}
} catch (Throwable e) {
@ -215,7 +215,7 @@ public class DataspaceManager implements Runnable {
}
public List<String> uploadInputData(List<StoredData> inputData, FolderContainer dataminerFolder) throws Exception {
LOGGER.debug("Dataspace->uploading input data; Number of data: " + inputData.size());
LOGGER.debug("Dataspace->uploading input data; Number of data: {}", inputData.size());
FolderContainer destinationFolder = (FolderContainer) dataminerFolder.findByName(importedDataFolder).getContainers().get(0);
List<String> urls = new ArrayList<String>();
for (StoredData input : inputData) {
@ -226,14 +226,14 @@ public class DataspaceManager implements Runnable {
if (items==null || items.isEmpty()){
String url = uploadData(input, destinationFolder,false);
LOGGER.debug("Dataspace->returning property "+url);
LOGGER.debug("Dataspace->returning property {}",url);
urls.add(url);
}
else{
FileContainer item = (FileContainer) items.get(0);
LOGGER.debug("Dataspace->Input item "+input.name+" is already available in the input folder");
LOGGER.debug("Dataspace->Input item {} is already available in the input folder",input.name);
String url = item.getPublicLink().toString();
LOGGER.debug("Dataspace->returning WS url "+url);
LOGGER.debug("Dataspace->returning WS url {}",url);
urls.add(url);
}
}
@ -315,7 +315,7 @@ public class DataspaceManager implements Runnable {
StoredData input = inputData.get(i - 1);
if (input.payload.contains("|")){
String payload = input .payload;
LOGGER.debug("Dataspace->Managing complex input "+input.name+" : "+payload);
LOGGER.debug("Dataspace->Managing complex input {} : {}",input.name, payload);
//delete the names that are not useful
for (StoredData subinput:inputData){
@ -351,9 +351,9 @@ public class DataspaceManager implements Runnable {
}
LOGGER.debug("Dataspace->Properties of the folder: " + properties);
LOGGER.debug("Dataspace->Properties of the folder: {} ", properties );
LOGGER.debug("Dataspace->Saving properties to ProvO XML file " + noutput + " outputs");
LOGGER.debug("Dataspace->Saving properties to ProvO XML file {} outputs",noutput);
/*
* XStream xstream = new XStream(); String xmlproperties = xstream.toXML(properties);
@ -489,7 +489,7 @@ public class DataspaceManager implements Runnable {
container.delete();
}
else
LOGGER.debug("Dataspace->Warning Could not find "+computation.id+" under "+computationContainer.get().getName());
LOGGER.debug("Dataspace->Warning Could not find {} under {}",computation.id, computationContainer.get().getName());
List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers();
@ -522,13 +522,13 @@ public class DataspaceManager implements Runnable {
for (Map.Entry<String, List<String>> entry : map.entrySet()) {
String value = entry.getValue().toString();
if (value.toLowerCase().contains("filename=")){
System.out.println("DataSpace->Searching in http header: found "+value);
LOGGER.debug("DataSpace->Searching in http header: found "+value);
extension = value.substring(value.lastIndexOf("."),value.lastIndexOf("\""));
}
}
conn.getInputStream().close();
} catch (Exception e) {
System.out.println("DataSpace->Error in the payload http link "+e.getMessage());
LOGGER.warn("DataSpace->Error in the payload http link ",e);
}
}
else {