gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

reverting retry in order to push another retry feature 

Revert "Added Retry in Input Parameter URL retrieve for fix StorageHub sync."

This reverts commit 4d6318df92.
This commit is contained in:
roberto cirillo 2020-05-12 15:18:23 +02:00
parent 4d6318df92
commit fdffbd6063
2 changed files with 32 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@ -9,7 +9,7 @@
</parent>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer</artifactId>
<version>1.5.10-SNAPSHOT</version>
<version>1.5.9-SNAPSHOT</version>
<name>dataminer</name>
<description>An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard.</description>
@ -74,12 +74,12 @@
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>52n-wps-algorithm-gcube</artifactId>
<version>[3.6.1,3.7.0-SNAPSHOT)</version>
<version>[3.6.1,3.7.0)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>52n-wps-server-gcube</artifactId>
<version>[3.6.1, 3.7.0-SNAPSHOT)</version>
<version>[3.6.1, 3.7.0)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>

83
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -5,10 +5,8 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
@ -43,8 +41,6 @@ import org.slf4j.LoggerFactory;
public class InputsManager {
private static final int RETRIEVE_INPUT_MAX_RETRY = 5;
private static final int RETRY_INTERVAL_TIME = 2000;
private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class);
LinkedHashMap<String, Object> inputs;
@ -243,9 +239,35 @@ public class InputsManager {
LOGGER.debug("Managing link: {}", subfilelink);
if (subfilelink.length() == 0)
continue;
// TODO
fileName = retrieveInputFromUrl(subfilelink, inputName, filesList);
InputStream is = null;
HttpURLConnection urlConnection = null;
URL url = new URL(subfilelink);
urlConnection = (HttpURLConnection) url.openConnection();
is = new BufferedInputStream(urlConnection.getInputStream());
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(fileNameTemp));
} else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
is.close();
fos.flush();
fos.close();
urlConnection.disconnect();
filesList.add(of);
LOGGER.debug("Created local file: {}", of.getAbsolutePath());
}
} else {
LOGGER.debug("Complex Input payload is file");
@ -303,53 +325,6 @@ public class InputsManager {
return filesList;
}
private String retrieveInputFromUrl(String subfilelink, String inputName, List<File> filesList) throws Exception {
String fileName = "";
for (int i = 0; i < RETRIEVE_INPUT_MAX_RETRY; i++) {
try {
InputStream is = null;
HttpURLConnection urlConnection = null;
URL url = new URL(subfilelink);
urlConnection = (HttpURLConnection) url.openConnection();
is = new BufferedInputStream(urlConnection.getInputStream());
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(fileNameTemp));
} else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
is.close();
fos.flush();
fos.close();
urlConnection.disconnect();
filesList.add(of);
LOGGER.debug("Created local file: {}", of.getAbsolutePath());
return fileName;
} catch (Exception e) {
LOGGER.error("ERROR retrieving Input URL: " + subfilelink, e);
try {
Thread.sleep(RETRY_INTERVAL_TIME);
} catch (InterruptedException e1) {
}
}
}
throw new Exception("Retry limit exceeded. Error retrieving Input URL : " + subfilelink);
}
public void createTable(String tableName, File tableFile, AlgorithmConfiguration config,
DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception {
@ -509,7 +484,7 @@ public class InputsManager {
}
private void saveInputData(String name, String description, String payload) {
LOGGER.debug("SaveInputData [name=" + name + ", description=" + description + ", payload=" + payload + "]");
LOGGER.debug("SaveInputData [name="+name+", description="+description+", payload="+payload+"]");
String id = name;
DataProvenance provenance = DataProvenance.IMPORTED;
String creationDate = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis());