gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@173518 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2018-10-16 13:27:01 +00:00
parent 2b2d42d120
commit 053e90458f
2 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -215,7 +215,7 @@ public class InputsManager {
//TODO DO NOT READ FILE INTO MEMORY
// read file content
String fileLink = readOneLine(f.getAbsolutePath());
LOGGER.debug("File link: " + fileLink.substring(0,Math.min(fileLink.length(),10)) + "...");
LOGGER.debug("File link: {} ...",fileLink.substring(0,Math.min(fileLink.length(),10)));
String fileName = "";
// case of a http link
if (fileLink!=null && fileLink.toLowerCase().startsWith("http:") || fileLink.toLowerCase().startsWith("https:")) {
@ -223,7 +223,7 @@ public class InputsManager {
String[] remotefiles = fileLink.split(inputsSeparator);
for (String subfilelink : remotefiles) {
subfilelink = subfilelink.trim();
LOGGER.debug("Managing link: " + subfilelink);
LOGGER.debug("Managing link: {}",subfilelink);
if (subfilelink.length() == 0)
continue;
InputStream is = null;
@ -234,12 +234,15 @@ public class InputsManager {
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}",fileNameTemp);
if (fileNameTemp==null)
fileName = String.format("%s_(%s)",inputName, computationId);
fileName = String.format("%s_(%s).%s",inputName, computationId, FilenameUtils.getExtension(inputName));
else
fileName = String.format("%s_(%s).%s",fileNameTemp, computationId, FilenameUtils.getExtension(fileNameTemp));
LOGGER.debug("Retrieving remote input in file: {}", fileName);
LOGGER.debug("Creating local temp file: {}", fileName);
LOGGER.debug("the name of the generated file is {}",fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
@ -253,27 +256,27 @@ public class InputsManager {
} else {
LOGGER.debug("Complex Input payload is the filelink");
fileName = f.getName();
LOGGER.debug("Retriving local input from file: " + fileName);
LOGGER.debug("Retriving local input from file: {}", fileName);
if (isXML(fileLink))
{
String xmlFile = f.getAbsolutePath();
String csvFile = xmlFile+".csv";
LOGGER.debug("Transforming XML file into a csv: " + csvFile);
LOGGER.debug("Transforming XML file into a csv: {} ",csvFile);
GML2CSV.parseGML(xmlFile, csvFile);
LOGGER.debug("GML Parsed: " + readOneLine(csvFile)+"[..]");
LOGGER.debug("GML Parsed: {} [..]",readOneLine(csvFile));
f = new File(csvFile);
}
else{
LOGGER.debug("The file is a csv: " + f.getAbsolutePath());
LOGGER.debug("The file is a csv: {}",f.getAbsolutePath());
}
String absFile = new File(f.getParent(),inputName+ "_[" + computationId + "].csv").getAbsolutePath();
String absFile = new File(String.format("%s_(%s).csv",f.getParent(),inputName,computationId)).getAbsolutePath();
LOGGER.debug("Renaming to: "+absFile);
System.gc();
boolean renamed = f.renameTo(new File(absFile));
if (renamed)
f = new File(absFile);
LOGGER.debug("The file has been renamed as : " + f.getAbsolutePath()+" - "+renamed);
LOGGER.debug("The file has been renamed as : {} - {}",f.getAbsolutePath(),renamed);
filesList.add(f);
}

16
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -134,7 +134,7 @@ public class DataspaceManager implements Runnable {
String filenameonwsString = data.name ;
if (changename)
filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload, data.type));
InputStream in = null;
String url = "";
try {
@ -186,7 +186,7 @@ public class DataspaceManager implements Runnable {
FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description);
fileContainer.setMetadata(new Metadata(properties));
LOGGER.debug("Dataspace->WS OP file saved on the WS " + filenameonwsString);
url = fileContainer.getPublicLink().toString();
@ -246,8 +246,8 @@ public class DataspaceManager implements Runnable {
FolderContainer destinationFolder = (FolderContainer)dataminerFolder.findByName(computedDataFolder).getContainers().get(0);
List<String> urls = new ArrayList<String>();
for (StoredData output : outputData) {
String url = uploadData(output, destinationFolder);
urls.add(url);
String url = uploadData(output, destinationFolder);
urls.add(url);
}
LOGGER.debug("Dataspace->finished uploading output data");
return urls;
@ -373,7 +373,7 @@ public class DataspaceManager implements Runnable {
List<String> scopes = new ArrayList<String>();
scopes.add(config.getGcubeScope());
ws.createGcubeItem(computation.id, computation.operatorDescription, scopes, computation.user, itemType, properties, newcomputationFolder.getId());
*/
*/
newcomputationFolder.setMetadata(new Metadata(properties));
@ -448,7 +448,7 @@ public class DataspaceManager implements Runnable {
gcubeItem.setItemType(itemType);
gcubeItem.setPropertyMap(new Metadata(properties));
gcubeItem.setCreator(AuthorizationProvider.instance.get().getClient().getId());
computationsContainer.newGcubeItem(gcubeItem);
@ -490,9 +490,9 @@ public class DataspaceManager implements Runnable {
else
LOGGER.debug("Dataspace->Warning Could not find "+computation.id+" under "+computationContainer.get().getName());
List<ItemContainer<? extends Item>> fileComputations = computationContainer.findByName(computation.id).getContainers();
if (fileComputations.size()>0)
fileComputations.get(0).delete();