gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Changed DatastapaceManager logs

This commit is contained in:
Giancarlo Panichi 2022-03-31 16:18:26 +02:00
parent e093239c7b
commit d2c45d0c8b
1 changed files with 27 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -64,7 +64,7 @@ public class DataspaceManager implements Runnable {
public static String payload = "payload"; public static String payload = "payload";
private String statusComputationName; private String statusComputationName;
private static final String STATUS_POSTFIX="-STATUS"; private static final String STATUS_POSTFIX = "-STATUS";
public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List<StoredData> inputData, public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List<StoredData> inputData,
List<StoredData> outputData, List<File> generatedFiles) { List<StoredData> outputData, List<File> generatedFiles) {
@ -73,7 +73,7 @@ public class DataspaceManager implements Runnable {
this.inputData = inputData; this.inputData = inputData;
this.outputData = outputData; this.outputData = outputData;
this.generatedFiles = generatedFiles; this.generatedFiles = generatedFiles;
this.statusComputationName = this.computation.id+STATUS_POSTFIX; this.statusComputationName = this.computation.id + STATUS_POSTFIX;
LOGGER.debug("DataspaceManager [config=" + config + ", computation=" + computation + ", inputData=" + inputData LOGGER.debug("DataspaceManager [config=" + config + ", computation=" + computation + ", inputData=" + inputData
+ ", outputData=" + outputData + ", generatedFiles=" + generatedFiles + "]"); + ", outputData=" + outputData + ", generatedFiles=" + generatedFiles + "]");
@ -142,25 +142,26 @@ public class DataspaceManager implements Runnable {
} }
public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception { public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception {
LOGGER.debug("Dataspace->Analysing " + data); LOGGER.debug("Dataspace->Uploaddata:"
+ " [data={},destinationFolder={}, changename={}] ", data, destinationFolder,
changename);
// String filenameonwsString = WorkspaceUtil.getUniqueName(data.name, // String filenameonwsString = WorkspaceUtil.getUniqueName(data.name,
// wsFolder); // wsFolder);
String filenameonwsString = data.name; String filenameonwsString = data.name;
if (changename) if (changename)
filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload));
getExtension(data.payload));
InputStream in = null; InputStream in = null;
String url = ""; String url = "";
try { try {
//long size = 0; // long size = 0;
if (data.type.equals("text/csv") || data.type.equals("application/d4science") if (data.type.equals("text/csv") || data.type.equals("application/d4science")
|| data.type.equals("image/png")) { || data.type.equals("image/png")) {
if (new File(data.payload).exists() || !data.payload.startsWith("http")) { if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
LOGGER.debug("Dataspace->Uploading file {}", data.payload); LOGGER.debug("Dataspace->Uploading file {}", data.payload);
in = new FileInputStream(new File(data.payload)); in = new FileInputStream(new File(data.payload));
//size = new File(data.payload).length(); // size = new File(data.payload).length();
} else { } else {
LOGGER.debug("Dataspace->Uploading via URL {}", data.payload); LOGGER.debug("Dataspace->Uploading via URL {}", data.payload);
int tries = 10; int tries = 10;
@ -188,7 +189,7 @@ public class DataspaceManager implements Runnable {
// LOGGER.debug("Dataspace->final file name on ws " + // LOGGER.debug("Dataspace->final file name on ws " +
// data.name+" description "+data.description); // data.name+" description "+data.description);
LOGGER.debug("Dataspace->WS OP saving the following file on the WS " + filenameonwsString); LOGGER.debug("Dataspace->WS OP saving the following file on the WS: " + filenameonwsString);
Map<String, Object> properties = new LinkedHashMap<String, Object>(); Map<String, Object> properties = new LinkedHashMap<String, Object>();
properties.put(computation_id, data.computationId); properties.put(computation_id, data.computationId);
@ -203,9 +204,11 @@ public class DataspaceManager implements Runnable {
properties.put(payload, url); properties.put(payload, url);
FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description); FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description);
fileContainer.setMetadata(new Metadata(properties)); LOGGER.debug("Dataspace->WS OP file uploaded on WS: " + filenameonwsString);
Metadata metadata = new Metadata(properties);
fileContainer.setMetadata(metadata);
LOGGER.debug("Dataspace->WS OP file saved on the WS " + filenameonwsString); LOGGER.debug("Dataspace->WS OP file set metadata: " + metadata);
url = fileContainer.getPublicLink().toString(); url = fileContainer.getPublicLink().toString();
LOGGER.debug("Dataspace->WS OP url produced for the file " + url); LOGGER.debug("Dataspace->WS OP url produced for the file " + url);
@ -395,10 +398,9 @@ public class DataspaceManager implements Runnable {
} }
/* /*
* List<String> scopes = new ArrayList<String>(); * List<String> scopes = new ArrayList<String>();
* scopes.add(config.getGcubeScope()); * scopes.add(config.getGcubeScope()); ws.createGcubeItem(computation.id,
* ws.createGcubeItem(computation.id, computation.operatorDescription, * computation.operatorDescription, scopes, computation.user, itemType,
* scopes, computation.user, itemType, properties, * properties, newcomputationFolder.getId());
* newcomputationFolder.getId());
*/ */
newcomputationFolder.setMetadata(new Metadata(properties)); newcomputationFolder.setMetadata(new Metadata(properties));
@ -465,8 +467,8 @@ public class DataspaceManager implements Runnable {
List<String> scopes = new ArrayList<String>(); List<String> scopes = new ArrayList<String>();
scopes.add(config.getGcubeScope()); scopes.add(config.getGcubeScope());
//TODO: update gcubeItem not recreate it... // TODO: update gcubeItem not recreate it...
GCubeItem gcubeItem = new GCubeItem(); GCubeItem gcubeItem = new GCubeItem();
gcubeItem.setName(this.statusComputationName); gcubeItem.setName(this.statusComputationName);
gcubeItem.setDescription(computation.operatorDescription); gcubeItem.setDescription(computation.operatorDescription);
@ -509,7 +511,8 @@ public class DataspaceManager implements Runnable {
.getContainers().get(0); .getContainers().get(0);
LOGGER.debug("Dataspace->removing computation data"); LOGGER.debug("Dataspace->removing computation data");
List<ItemContainer<? extends Item>> wi = computationContainer.findByName(this.statusComputationName).getContainers(); List<ItemContainer<? extends Item>> wi = computationContainer.findByName(this.statusComputationName)
.getContainers();
if (!wi.isEmpty()) { if (!wi.isEmpty()) {
for (ItemContainer<? extends Item> container : wi) { for (ItemContainer<? extends Item> container : wi) {
boolean retry = false; boolean retry = false;
@ -517,25 +520,23 @@ public class DataspaceManager implements Runnable {
try { try {
container.forceDelete(); container.forceDelete();
retry = false; retry = false;
}catch (ItemLockedException e) { } catch (ItemLockedException e) {
LOGGER.warn("item locked, retrying"); LOGGER.warn("item locked, retrying");
Thread.sleep(1000); Thread.sleep(1000);
retry = true; retry = true;
} }
}while (retry); } while (retry);
} }
} else } else
LOGGER.debug("Dataspace->Warning Could not find {} under {}", this.statusComputationName, LOGGER.debug("Dataspace->Warning Could not find {} under {}", this.statusComputationName,
computationContainer.get().getName()); computationContainer.get().getName());
/* /*
* TODO: ASK GIANPAOLO int maxtries = 3; int i =1; while * TODO: ASK GIANPAOLO int maxtries = 3; int i =1; while
* (ws.exists(computation.id,computationsFolderWs.getId()) && * (ws.exists(computation.id,computationsFolderWs.getId()) && i<maxtries){
* i<maxtries){ * LOGGER.debug("Dataspace->computation data still exist... retrying "+i );
* LOGGER.debug("Dataspace->computation data still exist... retrying "+i * Thread.sleep(1000); computationsFolderWs.find(computation.id).remove(); i++;
* ); Thread.sleep(1000); * }
* computationsFolderWs.find(computation.id).remove(); i++; }
*/ */
LOGGER.debug("Dataspace->finished removing computation data "); LOGGER.debug("Dataspace->finished removing computation data ");
@ -544,7 +545,7 @@ public class DataspaceManager implements Runnable {
// TODO // TODO
public static String getExtension(String payload) { public static String getExtension(String payload) {
LOGGER.debug("DataSpace->Get Extension from: " + payload); LOGGER.debug("DataSpace->Get Extension from: " + payload);
String extension=""; String extension = "";
if (payload.toLowerCase().startsWith("http")) { if (payload.toLowerCase().startsWith("http")) {
try { try {
URL obj = new URL(payload); URL obj = new URL(payload);