From e093239c7b7be24fdd638a837db408654489e17c Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Fri, 25 Mar 2022 12:47:26 +0100 Subject: [PATCH 1/3] Updated write exclusion behavior --- .../AbstractEcologicalEngineMapper.java | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java index f8b08ac..a8e27d1 100755 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java @@ -53,12 +53,11 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i private static final int COMPUTATION_WAIT_FOR_RUN_REQUEST = 20000; /** - * Deploying procedure: 1 - modify configuration files 2 - modify resource - * file: resources/templates/setup.cfg 3 - generate classes with - * ClassGenerator 4 - add new classes in the wps_config.xml on the wps web - * app config folder 5 - produce the Jar file of this project 6 - copy the - * jar file in the lib folder of the wps web app change the server - * parameters in the wps_config.xml file + * Deploying procedure: 1 - modify configuration files 2 - modify resource file: + * resources/templates/setup.cfg 3 - generate classes with ClassGenerator 4 - + * add new classes in the wps_config.xml on the wps web app config folder 5 - + * produce the Jar file of this project 6 - copy the jar file in the lib folder + * of the wps web app change the server parameters in the wps_config.xml file */ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class); @@ -538,15 +537,25 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i } private boolean checkWriteAuthorization(String username) { - if (env != null && env.getShubUsersExcluded() != null) { - if (env.getShubUsersExcluded().isEmpty()) { - return false; - } - if (env.getShubUsersExcluded().contains(username)) { + if (env != null) { + if (env.getShubUsersExcluded() != null) { + if (env.getShubUsersExcluded().isEmpty()) { + // all users write + return true; + } + if (env.getShubUsersExcluded().contains(username)) { + return false; + } else { + // username write + return true; + } + } else { + //This is the * case, no users write. return false; } + } else { + return false; } - return true; } private void accountAlgorithmExecution(long start, long end, OperationResult result) { From d2c45d0c8b6fde74f1df7804e13b00d812600cd6 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Thu, 31 Mar 2022 16:18:26 +0200 Subject: [PATCH 2/3] Changed DatastapaceManager logs --- .../mapping/dataspace/DataspaceManager.java | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java index bfd0467..5669956 100755 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java @@ -64,7 +64,7 @@ public class DataspaceManager implements Runnable { public static String payload = "payload"; private String statusComputationName; - private static final String STATUS_POSTFIX="-STATUS"; + private static final String STATUS_POSTFIX = "-STATUS"; public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List inputData, List outputData, List generatedFiles) { @@ -73,7 +73,7 @@ public class DataspaceManager implements Runnable { this.inputData = inputData; this.outputData = outputData; this.generatedFiles = generatedFiles; - this.statusComputationName = this.computation.id+STATUS_POSTFIX; + this.statusComputationName = this.computation.id + STATUS_POSTFIX; LOGGER.debug("DataspaceManager [config=" + config + ", computation=" + computation + ", inputData=" + inputData + ", outputData=" + outputData + ", generatedFiles=" + generatedFiles + "]"); @@ -142,25 +142,26 @@ public class DataspaceManager implements Runnable { } public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception { - LOGGER.debug("Dataspace->Analysing " + data); + LOGGER.debug("Dataspace->Uploaddata:" + + " [data={},destinationFolder={}, changename={}] ", data, destinationFolder, + changename); // String filenameonwsString = WorkspaceUtil.getUniqueName(data.name, // wsFolder); String filenameonwsString = data.name; if (changename) - filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, - getExtension(data.payload)); + filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload)); InputStream in = null; String url = ""; try { - //long size = 0; + // long size = 0; if (data.type.equals("text/csv") || data.type.equals("application/d4science") || data.type.equals("image/png")) { if (new File(data.payload).exists() || !data.payload.startsWith("http")) { LOGGER.debug("Dataspace->Uploading file {}", data.payload); in = new FileInputStream(new File(data.payload)); - //size = new File(data.payload).length(); + // size = new File(data.payload).length(); } else { LOGGER.debug("Dataspace->Uploading via URL {}", data.payload); int tries = 10; @@ -188,7 +189,7 @@ public class DataspaceManager implements Runnable { // LOGGER.debug("Dataspace->final file name on ws " + // data.name+" description "+data.description); - LOGGER.debug("Dataspace->WS OP saving the following file on the WS " + filenameonwsString); + LOGGER.debug("Dataspace->WS OP saving the following file on the WS: " + filenameonwsString); Map properties = new LinkedHashMap(); properties.put(computation_id, data.computationId); @@ -203,9 +204,11 @@ public class DataspaceManager implements Runnable { properties.put(payload, url); FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description); - fileContainer.setMetadata(new Metadata(properties)); + LOGGER.debug("Dataspace->WS OP file uploaded on WS: " + filenameonwsString); + Metadata metadata = new Metadata(properties); + fileContainer.setMetadata(metadata); - LOGGER.debug("Dataspace->WS OP file saved on the WS " + filenameonwsString); + LOGGER.debug("Dataspace->WS OP file set metadata: " + metadata); url = fileContainer.getPublicLink().toString(); LOGGER.debug("Dataspace->WS OP url produced for the file " + url); @@ -395,10 +398,9 @@ public class DataspaceManager implements Runnable { } /* * List scopes = new ArrayList(); - * scopes.add(config.getGcubeScope()); - * ws.createGcubeItem(computation.id, computation.operatorDescription, - * scopes, computation.user, itemType, properties, - * newcomputationFolder.getId()); + * scopes.add(config.getGcubeScope()); ws.createGcubeItem(computation.id, + * computation.operatorDescription, scopes, computation.user, itemType, + * properties, newcomputationFolder.getId()); */ newcomputationFolder.setMetadata(new Metadata(properties)); @@ -465,8 +467,8 @@ public class DataspaceManager implements Runnable { List scopes = new ArrayList(); scopes.add(config.getGcubeScope()); - - //TODO: update gcubeItem not recreate it... + + // TODO: update gcubeItem not recreate it... GCubeItem gcubeItem = new GCubeItem(); gcubeItem.setName(this.statusComputationName); gcubeItem.setDescription(computation.operatorDescription); @@ -509,7 +511,8 @@ public class DataspaceManager implements Runnable { .getContainers().get(0); LOGGER.debug("Dataspace->removing computation data"); - List> wi = computationContainer.findByName(this.statusComputationName).getContainers(); + List> wi = computationContainer.findByName(this.statusComputationName) + .getContainers(); if (!wi.isEmpty()) { for (ItemContainer container : wi) { boolean retry = false; @@ -517,25 +520,23 @@ public class DataspaceManager implements Runnable { try { container.forceDelete(); retry = false; - }catch (ItemLockedException e) { + } catch (ItemLockedException e) { LOGGER.warn("item locked, retrying"); Thread.sleep(1000); retry = true; } - }while (retry); + } while (retry); } } else LOGGER.debug("Dataspace->Warning Could not find {} under {}", this.statusComputationName, computationContainer.get().getName()); - /* * TODO: ASK GIANPAOLO int maxtries = 3; int i =1; while - * (ws.exists(computation.id,computationsFolderWs.getId()) && - * icomputation data still exist... retrying "+i - * ); Thread.sleep(1000); - * computationsFolderWs.find(computation.id).remove(); i++; } + * (ws.exists(computation.id,computationsFolderWs.getId()) && icomputation data still exist... retrying "+i ); + * Thread.sleep(1000); computationsFolderWs.find(computation.id).remove(); i++; + * } */ LOGGER.debug("Dataspace->finished removing computation data "); @@ -544,7 +545,7 @@ public class DataspaceManager implements Runnable { // TODO public static String getExtension(String payload) { LOGGER.debug("DataSpace->Get Extension from: " + payload); - String extension=""; + String extension = ""; if (payload.toLowerCase().startsWith("http")) { try { URL obj = new URL(payload); From 7899ababdd6efd43c9dd5762885b0a271a6cdc59 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Fri, 1 Apr 2022 10:25:36 +0200 Subject: [PATCH 3/3] Updated Changelog --- CHANGELOG.md | 35 +++++------------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 813c0a7..279ff59 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,92 +1,67 @@ -This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - # Changelog for "dataminer" ## [v1.8.1-SNAPSHOT] - 2022-03-21 -### Fixes - +- Update wps service to support not writing of the computation status to the user's workspace [#23054] - Fixed protocol parameter when persistence is disabled ## [v1.8.0] - 2022-01-24 -### Fixes - - Fixed max computations support [#22700] ## [v1.7.1] - 2021-05-24 -### Fixes - - Fixed obsolete short urls [#20971] ## [v1.7.0] - 2020-11-20 -### Fixes - - import range modified to resolve old repositories invalid url -## [v1.6.0] - 2020-05-12 -### Fixes +## [v1.6.0] - 2020-05-12 - Added storagehub retry in InputsManager class, getLocalFile method [#19253] - ## [v1.5.9] - 2019-11-20 -### Fixes - - Fixed Content-Type support for files in the results of computations [#18096] - ## [v1.5.8] - 2019-10-01 -### Fixes - - Fixed https link for output parameter [#17659] - ## [v1.5.7] - 2019-03-01 -### Features - - Updated https support [#13024] ## [v1.5.2] - 2017-12-13 -### Fixes - - added the right extension on output file - lock file created on execution - ## [v1.5.1] - 2017-09-14 -### Features - - added accounting on algorithm execution - ## [v1.5.0] - 2017-07-31 -### Features - - service interface classes moved to wps project - ## [v1.1.0] - 2016-10-03 - First Release + +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +