feature/23054 #2
35
CHANGELOG.md
35
CHANGELOG.md
|
@ -1,92 +1,67 @@
|
||||||
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
||||||
|
|
||||||
# Changelog for "dataminer"
|
# Changelog for "dataminer"
|
||||||
|
|
||||||
## [v1.8.1-SNAPSHOT] - 2022-03-21
|
## [v1.8.1-SNAPSHOT] - 2022-03-21
|
||||||
|
|
||||||
### Fixes
|
- Update wps service to support not writing of the computation status to the user's workspace [#23054]
|
||||||
|
|
||||||
- Fixed protocol parameter when persistence is disabled
|
- Fixed protocol parameter when persistence is disabled
|
||||||
|
|
||||||
|
|
||||||
## [v1.8.0] - 2022-01-24
|
## [v1.8.0] - 2022-01-24
|
||||||
|
|
||||||
### Fixes
|
|
||||||
|
|
||||||
- Fixed max computations support [#22700]
|
- Fixed max computations support [#22700]
|
||||||
|
|
||||||
|
|
||||||
## [v1.7.1] - 2021-05-24
|
## [v1.7.1] - 2021-05-24
|
||||||
|
|
||||||
### Fixes
|
|
||||||
|
|
||||||
- Fixed obsolete short urls [#20971]
|
- Fixed obsolete short urls [#20971]
|
||||||
|
|
||||||
|
|
||||||
## [v1.7.0] - 2020-11-20
|
## [v1.7.0] - 2020-11-20
|
||||||
|
|
||||||
### Fixes
|
|
||||||
|
|
||||||
- import range modified to resolve old repositories invalid url
|
- import range modified to resolve old repositories invalid url
|
||||||
|
|
||||||
## [v1.6.0] - 2020-05-12
|
|
||||||
|
|
||||||
### Fixes
|
## [v1.6.0] - 2020-05-12
|
||||||
|
|
||||||
- Added storagehub retry in InputsManager class, getLocalFile method [#19253]
|
- Added storagehub retry in InputsManager class, getLocalFile method [#19253]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.5.9] - 2019-11-20
|
## [v1.5.9] - 2019-11-20
|
||||||
|
|
||||||
### Fixes
|
|
||||||
|
|
||||||
- Fixed Content-Type support for files in the results of computations [#18096]
|
- Fixed Content-Type support for files in the results of computations [#18096]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.5.8] - 2019-10-01
|
## [v1.5.8] - 2019-10-01
|
||||||
|
|
||||||
### Fixes
|
|
||||||
|
|
||||||
- Fixed https link for output parameter [#17659]
|
- Fixed https link for output parameter [#17659]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.5.7] - 2019-03-01
|
## [v1.5.7] - 2019-03-01
|
||||||
|
|
||||||
### Features
|
|
||||||
|
|
||||||
- Updated https support [#13024]
|
- Updated https support [#13024]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.5.2] - 2017-12-13
|
## [v1.5.2] - 2017-12-13
|
||||||
|
|
||||||
### Fixes
|
|
||||||
|
|
||||||
- added the right extension on output file
|
- added the right extension on output file
|
||||||
- lock file created on execution
|
- lock file created on execution
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.5.1] - 2017-09-14
|
## [v1.5.1] - 2017-09-14
|
||||||
|
|
||||||
### Features
|
|
||||||
|
|
||||||
- added accounting on algorithm execution
|
- added accounting on algorithm execution
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.5.0] - 2017-07-31
|
## [v1.5.0] - 2017-07-31
|
||||||
|
|
||||||
### Features
|
|
||||||
|
|
||||||
- service interface classes moved to wps project
|
- service interface classes moved to wps project
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [v1.1.0] - 2016-10-03
|
## [v1.1.0] - 2016-10-03
|
||||||
|
|
||||||
- First Release
|
- First Release
|
||||||
|
|
||||||
|
|
||||||
|
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
|
|
@ -53,12 +53,11 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
|
||||||
private static final int COMPUTATION_WAIT_FOR_RUN_REQUEST = 20000;
|
private static final int COMPUTATION_WAIT_FOR_RUN_REQUEST = 20000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Deploying procedure: 1 - modify configuration files 2 - modify resource
|
* Deploying procedure: 1 - modify configuration files 2 - modify resource file:
|
||||||
* file: resources/templates/setup.cfg 3 - generate classes with
|
* resources/templates/setup.cfg 3 - generate classes with ClassGenerator 4 -
|
||||||
* ClassGenerator 4 - add new classes in the wps_config.xml on the wps web
|
* add new classes in the wps_config.xml on the wps web app config folder 5 -
|
||||||
* app config folder 5 - produce the Jar file of this project 6 - copy the
|
* produce the Jar file of this project 6 - copy the jar file in the lib folder
|
||||||
* jar file in the lib folder of the wps web app change the server
|
* of the wps web app change the server parameters in the wps_config.xml file
|
||||||
* parameters in the wps_config.xml file
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class);
|
||||||
|
@ -538,15 +537,25 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean checkWriteAuthorization(String username) {
|
private boolean checkWriteAuthorization(String username) {
|
||||||
if (env != null && env.getShubUsersExcluded() != null) {
|
if (env != null) {
|
||||||
if (env.getShubUsersExcluded().isEmpty()) {
|
if (env.getShubUsersExcluded() != null) {
|
||||||
return false;
|
if (env.getShubUsersExcluded().isEmpty()) {
|
||||||
}
|
// all users write
|
||||||
if (env.getShubUsersExcluded().contains(username)) {
|
return true;
|
||||||
|
}
|
||||||
|
if (env.getShubUsersExcluded().contains(username)) {
|
||||||
|
return false;
|
||||||
|
} else {
|
||||||
|
// username write
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
//This is the * case, no users write.
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void accountAlgorithmExecution(long start, long end, OperationResult result) {
|
private void accountAlgorithmExecution(long start, long end, OperationResult result) {
|
||||||
|
|
|
@ -64,7 +64,7 @@ public class DataspaceManager implements Runnable {
|
||||||
public static String payload = "payload";
|
public static String payload = "payload";
|
||||||
|
|
||||||
private String statusComputationName;
|
private String statusComputationName;
|
||||||
private static final String STATUS_POSTFIX="-STATUS";
|
private static final String STATUS_POSTFIX = "-STATUS";
|
||||||
|
|
||||||
public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List<StoredData> inputData,
|
public DataspaceManager(AlgorithmConfiguration config, ComputationData computation, List<StoredData> inputData,
|
||||||
List<StoredData> outputData, List<File> generatedFiles) {
|
List<StoredData> outputData, List<File> generatedFiles) {
|
||||||
|
@ -73,7 +73,7 @@ public class DataspaceManager implements Runnable {
|
||||||
this.inputData = inputData;
|
this.inputData = inputData;
|
||||||
this.outputData = outputData;
|
this.outputData = outputData;
|
||||||
this.generatedFiles = generatedFiles;
|
this.generatedFiles = generatedFiles;
|
||||||
this.statusComputationName = this.computation.id+STATUS_POSTFIX;
|
this.statusComputationName = this.computation.id + STATUS_POSTFIX;
|
||||||
|
|
||||||
LOGGER.debug("DataspaceManager [config=" + config + ", computation=" + computation + ", inputData=" + inputData
|
LOGGER.debug("DataspaceManager [config=" + config + ", computation=" + computation + ", inputData=" + inputData
|
||||||
+ ", outputData=" + outputData + ", generatedFiles=" + generatedFiles + "]");
|
+ ", outputData=" + outputData + ", generatedFiles=" + generatedFiles + "]");
|
||||||
|
@ -142,25 +142,26 @@ public class DataspaceManager implements Runnable {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception {
|
public String uploadData(StoredData data, FolderContainer destinationFolder, boolean changename) throws Exception {
|
||||||
LOGGER.debug("Dataspace->Analysing " + data);
|
LOGGER.debug("Dataspace->Uploaddata:"
|
||||||
|
+ " [data={},destinationFolder={}, changename={}] ", data, destinationFolder,
|
||||||
|
changename);
|
||||||
// String filenameonwsString = WorkspaceUtil.getUniqueName(data.name,
|
// String filenameonwsString = WorkspaceUtil.getUniqueName(data.name,
|
||||||
// wsFolder);
|
// wsFolder);
|
||||||
String filenameonwsString = data.name;
|
String filenameonwsString = data.name;
|
||||||
if (changename)
|
if (changename)
|
||||||
filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId,
|
filenameonwsString = String.format("%s_(%s)%s", data.name, data.computationId, getExtension(data.payload));
|
||||||
getExtension(data.payload));
|
|
||||||
|
|
||||||
InputStream in = null;
|
InputStream in = null;
|
||||||
String url = "";
|
String url = "";
|
||||||
try {
|
try {
|
||||||
//long size = 0;
|
// long size = 0;
|
||||||
if (data.type.equals("text/csv") || data.type.equals("application/d4science")
|
if (data.type.equals("text/csv") || data.type.equals("application/d4science")
|
||||||
|| data.type.equals("image/png")) {
|
|| data.type.equals("image/png")) {
|
||||||
|
|
||||||
if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
|
if (new File(data.payload).exists() || !data.payload.startsWith("http")) {
|
||||||
LOGGER.debug("Dataspace->Uploading file {}", data.payload);
|
LOGGER.debug("Dataspace->Uploading file {}", data.payload);
|
||||||
in = new FileInputStream(new File(data.payload));
|
in = new FileInputStream(new File(data.payload));
|
||||||
//size = new File(data.payload).length();
|
// size = new File(data.payload).length();
|
||||||
} else {
|
} else {
|
||||||
LOGGER.debug("Dataspace->Uploading via URL {}", data.payload);
|
LOGGER.debug("Dataspace->Uploading via URL {}", data.payload);
|
||||||
int tries = 10;
|
int tries = 10;
|
||||||
|
@ -188,7 +189,7 @@ public class DataspaceManager implements Runnable {
|
||||||
|
|
||||||
// LOGGER.debug("Dataspace->final file name on ws " +
|
// LOGGER.debug("Dataspace->final file name on ws " +
|
||||||
// data.name+" description "+data.description);
|
// data.name+" description "+data.description);
|
||||||
LOGGER.debug("Dataspace->WS OP saving the following file on the WS " + filenameonwsString);
|
LOGGER.debug("Dataspace->WS OP saving the following file on the WS: " + filenameonwsString);
|
||||||
Map<String, Object> properties = new LinkedHashMap<String, Object>();
|
Map<String, Object> properties = new LinkedHashMap<String, Object>();
|
||||||
|
|
||||||
properties.put(computation_id, data.computationId);
|
properties.put(computation_id, data.computationId);
|
||||||
|
@ -203,9 +204,11 @@ public class DataspaceManager implements Runnable {
|
||||||
properties.put(payload, url);
|
properties.put(payload, url);
|
||||||
|
|
||||||
FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description);
|
FileContainer fileContainer = destinationFolder.uploadFile(in, filenameonwsString, data.description);
|
||||||
fileContainer.setMetadata(new Metadata(properties));
|
LOGGER.debug("Dataspace->WS OP file uploaded on WS: " + filenameonwsString);
|
||||||
|
Metadata metadata = new Metadata(properties);
|
||||||
|
fileContainer.setMetadata(metadata);
|
||||||
|
|
||||||
LOGGER.debug("Dataspace->WS OP file saved on the WS " + filenameonwsString);
|
LOGGER.debug("Dataspace->WS OP file set metadata: " + metadata);
|
||||||
|
|
||||||
url = fileContainer.getPublicLink().toString();
|
url = fileContainer.getPublicLink().toString();
|
||||||
LOGGER.debug("Dataspace->WS OP url produced for the file " + url);
|
LOGGER.debug("Dataspace->WS OP url produced for the file " + url);
|
||||||
|
@ -395,10 +398,9 @@ public class DataspaceManager implements Runnable {
|
||||||
}
|
}
|
||||||
/*
|
/*
|
||||||
* List<String> scopes = new ArrayList<String>();
|
* List<String> scopes = new ArrayList<String>();
|
||||||
* scopes.add(config.getGcubeScope());
|
* scopes.add(config.getGcubeScope()); ws.createGcubeItem(computation.id,
|
||||||
* ws.createGcubeItem(computation.id, computation.operatorDescription,
|
* computation.operatorDescription, scopes, computation.user, itemType,
|
||||||
* scopes, computation.user, itemType, properties,
|
* properties, newcomputationFolder.getId());
|
||||||
* newcomputationFolder.getId());
|
|
||||||
*/
|
*/
|
||||||
newcomputationFolder.setMetadata(new Metadata(properties));
|
newcomputationFolder.setMetadata(new Metadata(properties));
|
||||||
|
|
||||||
|
@ -465,8 +467,8 @@ public class DataspaceManager implements Runnable {
|
||||||
|
|
||||||
List<String> scopes = new ArrayList<String>();
|
List<String> scopes = new ArrayList<String>();
|
||||||
scopes.add(config.getGcubeScope());
|
scopes.add(config.getGcubeScope());
|
||||||
|
|
||||||
//TODO: update gcubeItem not recreate it...
|
// TODO: update gcubeItem not recreate it...
|
||||||
GCubeItem gcubeItem = new GCubeItem();
|
GCubeItem gcubeItem = new GCubeItem();
|
||||||
gcubeItem.setName(this.statusComputationName);
|
gcubeItem.setName(this.statusComputationName);
|
||||||
gcubeItem.setDescription(computation.operatorDescription);
|
gcubeItem.setDescription(computation.operatorDescription);
|
||||||
|
@ -509,7 +511,8 @@ public class DataspaceManager implements Runnable {
|
||||||
.getContainers().get(0);
|
.getContainers().get(0);
|
||||||
LOGGER.debug("Dataspace->removing computation data");
|
LOGGER.debug("Dataspace->removing computation data");
|
||||||
|
|
||||||
List<ItemContainer<? extends Item>> wi = computationContainer.findByName(this.statusComputationName).getContainers();
|
List<ItemContainer<? extends Item>> wi = computationContainer.findByName(this.statusComputationName)
|
||||||
|
.getContainers();
|
||||||
if (!wi.isEmpty()) {
|
if (!wi.isEmpty()) {
|
||||||
for (ItemContainer<? extends Item> container : wi) {
|
for (ItemContainer<? extends Item> container : wi) {
|
||||||
boolean retry = false;
|
boolean retry = false;
|
||||||
|
@ -517,25 +520,23 @@ public class DataspaceManager implements Runnable {
|
||||||
try {
|
try {
|
||||||
container.forceDelete();
|
container.forceDelete();
|
||||||
retry = false;
|
retry = false;
|
||||||
}catch (ItemLockedException e) {
|
} catch (ItemLockedException e) {
|
||||||
LOGGER.warn("item locked, retrying");
|
LOGGER.warn("item locked, retrying");
|
||||||
Thread.sleep(1000);
|
Thread.sleep(1000);
|
||||||
retry = true;
|
retry = true;
|
||||||
}
|
}
|
||||||
}while (retry);
|
} while (retry);
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
LOGGER.debug("Dataspace->Warning Could not find {} under {}", this.statusComputationName,
|
LOGGER.debug("Dataspace->Warning Could not find {} under {}", this.statusComputationName,
|
||||||
computationContainer.get().getName());
|
computationContainer.get().getName());
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* TODO: ASK GIANPAOLO int maxtries = 3; int i =1; while
|
* TODO: ASK GIANPAOLO int maxtries = 3; int i =1; while
|
||||||
* (ws.exists(computation.id,computationsFolderWs.getId()) &&
|
* (ws.exists(computation.id,computationsFolderWs.getId()) && i<maxtries){
|
||||||
* i<maxtries){
|
* LOGGER.debug("Dataspace->computation data still exist... retrying "+i );
|
||||||
* LOGGER.debug("Dataspace->computation data still exist... retrying "+i
|
* Thread.sleep(1000); computationsFolderWs.find(computation.id).remove(); i++;
|
||||||
* ); Thread.sleep(1000);
|
* }
|
||||||
* computationsFolderWs.find(computation.id).remove(); i++; }
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
LOGGER.debug("Dataspace->finished removing computation data ");
|
LOGGER.debug("Dataspace->finished removing computation data ");
|
||||||
|
@ -544,7 +545,7 @@ public class DataspaceManager implements Runnable {
|
||||||
// TODO
|
// TODO
|
||||||
public static String getExtension(String payload) {
|
public static String getExtension(String payload) {
|
||||||
LOGGER.debug("DataSpace->Get Extension from: " + payload);
|
LOGGER.debug("DataSpace->Get Extension from: " + payload);
|
||||||
String extension="";
|
String extension = "";
|
||||||
if (payload.toLowerCase().startsWith("http")) {
|
if (payload.toLowerCase().startsWith("http")) {
|
||||||
try {
|
try {
|
||||||
URL obj = new URL(payload);
|
URL obj = new URL(payload);
|
||||||
|
|
Loading…
Reference in New Issue