gCubeSystem
/
dataminer
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@174113 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2018-11-12 17:16:15 +00:00
parent 26ad1e8cc9
commit 60ccac1784
2 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -218,7 +218,7 @@ public class InputsManager {
LOGGER.debug("File link: {} ...",fileLink.substring(0,Math.min(fileLink.length(),10))); LOGGER.debug("File link: {} ...",fileLink.substring(0,Math.min(fileLink.length(),10)));
String fileName = ""; String fileName = "";
// case of a http link // case of a http link
if (fileLink!=null && fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:")) { if (fileLink!=null && (fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:"))) {
// manage the case of multiple files // manage the case of multiple files
String[] remotefiles = fileLink.split(inputsSeparator); String[] remotefiles = fileLink.split(inputsSeparator);
for (String subfilelink : remotefiles) { for (String subfilelink : remotefiles) {
@ -256,8 +256,11 @@ public class InputsManager {
} else { } else {
LOGGER.debug("Complex Input payload is the filelink"); LOGGER.debug("Complex Input payload is the filelink");
fileName = f.getName(); fileName = f.getName();
LOGGER.debug("Retriving local input from file: {}", fileName); LOGGER.debug("Retriving local input from file: {}", fileName);
String fileExt=null;
if (isXML(fileLink)) if (isXML(fileLink))
{ {
String xmlFile = f.getAbsolutePath(); String xmlFile = f.getAbsolutePath();
@ -266,11 +269,14 @@ public class InputsManager {
GML2CSV.parseGML(xmlFile, csvFile); GML2CSV.parseGML(xmlFile, csvFile);
LOGGER.debug("GML Parsed: {} [..]",readOneLine(csvFile)); LOGGER.debug("GML Parsed: {} [..]",readOneLine(csvFile));
f = new File(csvFile); f = new File(csvFile);
fileExt = "csv";
} }
else{ else{
LOGGER.debug("The file is a csv: {}",f.getAbsolutePath()); LOGGER.debug("The file is a csv: {}",f.getAbsolutePath());
fileExt = FilenameUtils.getExtension(fileName);
} }
String absFile = new File(String.format("%s_(%s).csv",f.getParent(),inputName,computationId)).getAbsolutePath();
String absFile = new File(f.getParent(), String.format("%s_(%s).%s",inputName,computationId, fileExt)).getAbsolutePath();
LOGGER.debug("Renaming to: "+absFile); LOGGER.debug("Renaming to: "+absFile);
System.gc(); System.gc();
boolean renamed = f.renameTo(new File(absFile)); boolean renamed = f.renameTo(new File(absFile));

30
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -142,11 +142,11 @@ public class DataspaceManager implements Runnable {
if (data.type.equals("text/csv")||data.type.equals("application/d4science")||data.type.equals("image/png")) { if (data.type.equals("text/csv")||data.type.equals("application/d4science")||data.type.equals("image/png")) {
if (new File(data.payload).exists() || !data.payload.startsWith("http")) { if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
LOGGER.debug("Dataspace->Uploading file " + data.payload); LOGGER.debug("Dataspace->Uploading file {}", data.payload);
in = new FileInputStream(new File(data.payload)); in = new FileInputStream(new File(data.payload));
size = new File(data.payload).length(); size = new File(data.payload).length();
} else { } else {
LOGGER.debug("Dataspace->Uploading via URL " + data.payload); LOGGER.debug("Dataspace->Uploading via URL {}", data.payload);
int tries = 10; int tries = 10;
for (int i=0;i<tries;i++){ for (int i=0;i<tries;i++){
try { try {
@ -197,12 +197,12 @@ public class DataspaceManager implements Runnable {
try { try {
in.close(); in.close();
} catch (Exception e) { } catch (Exception e) {
LOGGER.debug("Dataspace->Error creating file " + e.getMessage()); LOGGER.debug("Dataspace->Error creating file {}", e.getMessage());
//LOGGER.debug(e); //LOGGER.debug(e);
} }
LOGGER.debug("Dataspace->File created " + filenameonwsString); LOGGER.debug("Dataspace->File created {}", filenameonwsString);
} else { } else {
LOGGER.debug("Dataspace->String parameter " + data.payload); LOGGER.debug("Dataspace->String parameter {}", data.payload);
url = data.payload; url = data.payload;
} }
} catch (Throwable e) { } catch (Throwable e) {
@ -215,7 +215,7 @@ public class DataspaceManager implements Runnable {
} }
public List<String> uploadInputData(List<StoredData> inputData, FolderContainer dataminerFolder) throws Exception { public List<String> uploadInputData(List<StoredData> inputData, FolderContainer dataminerFolder) throws Exception {
LOGGER.debug("Dataspace->uploading input data; Number of data: " + inputData.size()); LOGGER.debug("Dataspace->uploading input data; Number of data: {}", inputData.size());
FolderContainer destinationFolder = (FolderContainer) dataminerFolder.findByName(importedDataFolder).getContainers().get(0); FolderContainer destinationFolder = (FolderContainer) dataminerFolder.findByName(importedDataFolder).getContainers().get(0);
List<String> urls = new ArrayList<String>(); List<String> urls = new ArrayList<String>();
for (StoredData input : inputData) { for (StoredData input : inputData) {
@ -226,14 +226,14 @@ public class DataspaceManager implements Runnable {
if (items==null || items.isEmpty()){ if (items==null || items.isEmpty()){
String url = uploadData(input, destinationFolder,false); String url = uploadData(input, destinationFolder,false);
LOGGER.debug("Dataspace->returning property "+url); LOGGER.debug("Dataspace->returning property {}",url);
urls.add(url); urls.add(url);
} }
else{ else{
FileContainer item = (FileContainer) items.get(0); FileContainer item = (FileContainer) items.get(0);
LOGGER.debug("Dataspace->Input item "+input.name+" is already available in the input folder"); LOGGER.debug("Dataspace->Input item {} is already available in the input folder",input.name);
String url = item.getPublicLink().toString(); String url = item.getPublicLink().toString();
LOGGER.debug("Dataspace->returning WS url "+url); LOGGER.debug("Dataspace->returning WS url {}",url);
urls.add(url); urls.add(url);
} }
} }
@ -315,7 +315,7 @@ public class DataspaceManager implements Runnable {
StoredData input = inputData.get(i - 1); StoredData input = inputData.get(i - 1);
if (input.payload.contains("|")){ if (input.payload.contains("|")){
String payload = input .payload; String payload = input .payload;
LOGGER.debug("Dataspace->Managing complex input "+input.name+" : "+payload); LOGGER.debug("Dataspace->Managing complex input {} : {}",input.name, payload);
//delete the names that are not useful //delete the names that are not useful
for (StoredData subinput:inputData){ for (StoredData subinput:inputData){
@ -351,9 +351,9 @@ public class DataspaceManager implements Runnable {
} }
LOGGER.debug("Dataspace->Properties of the folder: " + properties); LOGGER.debug("Dataspace->Properties of the folder: {} ", properties );
LOGGER.debug("Dataspace->Saving properties to ProvO XML file " + noutput + " outputs"); LOGGER.debug("Dataspace->Saving properties to ProvO XML file {} outputs",noutput);
/* /*
* XStream xstream = new XStream(); String xmlproperties = xstream.toXML(properties); * XStream xstream = new XStream(); String xmlproperties = xstream.toXML(properties);
@ -489,7 +489,7 @@ public class DataspaceManager implements Runnable {
container.delete(); container.delete();
} }
else else
LOGGER.debug("Dataspace->Warning Could not find "+computation.id+" under "+computationContainer.get().getName()); LOGGER.debug("Dataspace->Warning Could not find {} under {}",computation.id, computationContainer.get().getName());
List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers(); List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers();
@ -522,13 +522,13 @@ public class DataspaceManager implements Runnable {
for (Map.Entry<String, List<String>> entry : map.entrySet()) { for (Map.Entry<String, List<String>> entry : map.entrySet()) {
String value = entry.getValue().toString(); String value = entry.getValue().toString();
if (value.toLowerCase().contains("filename=")){ if (value.toLowerCase().contains("filename=")){
System.out.println("DataSpace->Searching in http header: found "+value); LOGGER.debug("DataSpace->Searching in http header: found "+value);
extension = value.substring(value.lastIndexOf("."),value.lastIndexOf("\"")); extension = value.substring(value.lastIndexOf("."),value.lastIndexOf("\""));
} }
} }
conn.getInputStream().close(); conn.getInputStream().close();
} catch (Exception e) { } catch (Exception e) {
System.out.println("DataSpace->Error in the payload http link "+e.getMessage()); LOGGER.warn("DataSpace->Error in the payload http link ",e);
} }
} }
else { else {