gCubeSystem
/
dataminer
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@173518 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2018-10-16 13:27:01 +00:00
parent 67fbb1f724
commit 9013721e12
2 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -215,7 +215,7 @@ public class InputsManager {
//TODO DO NOT READ FILE INTO MEMORY //TODO DO NOT READ FILE INTO MEMORY
// read file content // read file content
String fileLink = readOneLine(f.getAbsolutePath()); String fileLink = readOneLine(f.getAbsolutePath());
LOGGER.debug("File link: " + fileLink.substring(0,Math.min(fileLink.length(),10)) + "..."); LOGGER.debug("File link: {} ...",fileLink.substring(0,Math.min(fileLink.length(),10)));
String fileName = ""; String fileName = "";
// case of a http link // case of a http link
if (fileLink!=null && fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:")) { if (fileLink!=null && fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:")) {
@ -223,7 +223,7 @@ public class InputsManager {
String[] remotefiles = fileLink.split(inputsSeparator); String[] remotefiles = fileLink.split(inputsSeparator);
for (String subfilelink : remotefiles) { for (String subfilelink : remotefiles) {
subfilelink = subfilelink.trim(); subfilelink = subfilelink.trim();
LOGGER.debug("Managing link: " + subfilelink); LOGGER.debug("Managing link: {}",subfilelink);
if (subfilelink.length() == 0) if (subfilelink.length() == 0)
continue; continue;
InputStream is = null; InputStream is = null;
@ -234,12 +234,15 @@ public class InputsManager {
// retrieve payload: for test purpose only // retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink); String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}",fileNameTemp);
if (fileNameTemp==null) if (fileNameTemp==null)
fileName = String.format("%s_(%s)",inputName, computationId); fileName = String.format("%s_(%s).%s",inputName, computationId, FilenameUtils.getExtension(inputName));
else else
fileName = String.format("%s_(%s).%s",fileNameTemp, computationId, FilenameUtils.getExtension(fileNameTemp)); fileName = String.format("%s_(%s).%s",fileNameTemp, computationId, FilenameUtils.getExtension(fileNameTemp));
LOGGER.debug("Retrieving remote input in file: {}", fileName);
LOGGER.debug("Creating local temp file: {}", fileName); LOGGER.debug("the name of the generated file is {}",fileName);
File of = new File(config.getPersistencePath(), fileName); File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of); FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos); IOUtils.copy(is, fos);
@ -253,27 +256,27 @@ public class InputsManager {
} else { } else {
LOGGER.debug("Complex Input payload is the filelink"); LOGGER.debug("Complex Input payload is the filelink");
fileName = f.getName(); fileName = f.getName();
LOGGER.debug("Retriving local input from file: " + fileName); LOGGER.debug("Retriving local input from file: {}", fileName);
if (isXML(fileLink)) if (isXML(fileLink))
{ {
String xmlFile = f.getAbsolutePath(); String xmlFile = f.getAbsolutePath();
String csvFile = xmlFile+".csv"; String csvFile = xmlFile+".csv";
LOGGER.debug("Transforming XML file into a csv: " + csvFile); LOGGER.debug("Transforming XML file into a csv: {} ",csvFile);
GML2CSV.parseGML(xmlFile, csvFile); GML2CSV.parseGML(xmlFile, csvFile);
LOGGER.debug("GML Parsed: " + readOneLine(csvFile)+"[..]"); LOGGER.debug("GML Parsed: {} [..]",readOneLine(csvFile));
f = new File(csvFile); f = new File(csvFile);
} }
else{ else{
LOGGER.debug("The file is a csv: " + f.getAbsolutePath()); LOGGER.debug("The file is a csv: {}",f.getAbsolutePath());
} }
String absFile = new File(f.getParent(),inputName+ "_[" + computationId + "].csv").getAbsolutePath(); String absFile = new File(String.format("%s_(%s).csv",f.getParent(),inputName,computationId)).getAbsolutePath();
LOGGER.debug("Renaming to: "+absFile); LOGGER.debug("Renaming to: "+absFile);
System.gc(); System.gc();
boolean renamed = f.renameTo(new File(absFile)); boolean renamed = f.renameTo(new File(absFile));
if (renamed) if (renamed)
f = new File(absFile); f = new File(absFile);
LOGGER.debug("The file has been renamed as : " + f.getAbsolutePath()+" - "+renamed); LOGGER.debug("The file has been renamed as : {} - {}",f.getAbsolutePath(),renamed);
filesList.add(f); filesList.add(f);
} }

16
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -134,7 +134,7 @@ public class DataspaceManager implements Runnable {
String filenameonwsString = data.name ; String filenameonwsString = data.name ;
if (changename) if (changename)
filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload, data.type)); filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload, data.type));
InputStream in = null; InputStream in = null;
String url = ""; String url = "";
try { try {
@ -186,7 +186,7 @@ public class DataspaceManager implements Runnable {
FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description); FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description);
fileContainer.setMetadata(new Metadata(properties)); fileContainer.setMetadata(new Metadata(properties));
LOGGER.debug("Dataspace->WS OP file saved on the WS " + filenameonwsString); LOGGER.debug("Dataspace->WS OP file saved on the WS " + filenameonwsString);
url = fileContainer.getPublicLink().toString(); url = fileContainer.getPublicLink().toString();
@ -246,8 +246,8 @@ public class DataspaceManager implements Runnable {
FolderContainer destinationFolder = (FolderContainer)dataminerFolder.findByName(computedDataFolder).getContainers().get(0); FolderContainer destinationFolder = (FolderContainer)dataminerFolder.findByName(computedDataFolder).getContainers().get(0);
List<String> urls = new ArrayList<String>(); List<String> urls = new ArrayList<String>();
for (StoredData output : outputData) { for (StoredData output : outputData) {
String url = uploadData(output, destinationFolder); String url = uploadData(output, destinationFolder);
urls.add(url); urls.add(url);
} }
LOGGER.debug("Dataspace->finished uploading output data"); LOGGER.debug("Dataspace->finished uploading output data");
return urls; return urls;
@ -373,7 +373,7 @@ public class DataspaceManager implements Runnable {
List<String> scopes = new ArrayList<String>(); List<String> scopes = new ArrayList<String>();
scopes.add(config.getGcubeScope()); scopes.add(config.getGcubeScope());
ws.createGcubeItem(computation.id, computation.operatorDescription, scopes, computation.user, itemType, properties, newcomputationFolder.getId()); ws.createGcubeItem(computation.id, computation.operatorDescription, scopes, computation.user, itemType, properties, newcomputationFolder.getId());
*/ */
newcomputationFolder.setMetadata(new Metadata(properties)); newcomputationFolder.setMetadata(new Metadata(properties));
@ -448,7 +448,7 @@ public class DataspaceManager implements Runnable {
gcubeItem.setItemType(itemType); gcubeItem.setItemType(itemType);
gcubeItem.setPropertyMap(new Metadata(properties)); gcubeItem.setPropertyMap(new Metadata(properties));
gcubeItem.setCreator(AuthorizationProvider.instance.get().getClient().getId()); gcubeItem.setCreator(AuthorizationProvider.instance.get().getClient().getId());
computationsContainer.newGcubeItem(gcubeItem); computationsContainer.newGcubeItem(gcubeItem);
@ -490,9 +490,9 @@ public class DataspaceManager implements Runnable {
else else
LOGGER.debug("Dataspace->Warning Could not find "+computation.id+" under "+computationContainer.get().getName()); LOGGER.debug("Dataspace->Warning Could not find "+computation.id+" under "+computationContainer.get().getName());
List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers(); List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers();
if (fileComputations.size()>0) if (fileComputations.size()>0)
fileComputations.get(0).delete(); fileComputations.get(0).delete();