gCubeSystem
/
dataminer
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Added Retry in Input Parameter URL retrieve for fix StorageHub sync.

This commit is contained in:
Giancarlo Panichi 2020-05-12 11:24:31 +02:00
parent 1dcab757f7
commit 690126d6c0
2 changed files with 57 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@ -9,7 +9,7 @@
</parent>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer</artifactId>
<version>1.5.9-SNAPSHOT</version>
<version>1.5.10-SNAPSHOT</version>
<name>dataminer</name>
<description>An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard.</description>
@ -74,12 +74,12 @@
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>52n-wps-algorithm-gcube</artifactId>
<version>[3.6.1,3.7.0)</version>
<version>[3.6.1,3.7.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>52n-wps-server-gcube</artifactId>
<version>[3.6.1, 3.7.0)</version>
<version>[3.6.1, 3.7.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>

83
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -5,8 +5,10 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
@ -41,6 +43,8 @@ import org.slf4j.LoggerFactory;
public class InputsManager {
private static final int RETRIEVE_INPUT_MAX_RETRY = 5;
private static final int RETRY_INTERVAL_TIME = 2000;
private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class);
LinkedHashMap<String, Object> inputs;
@ -239,35 +243,9 @@ public class InputsManager {
LOGGER.debug("Managing link: {}", subfilelink);
if (subfilelink.length() == 0)
continue;
InputStream is = null;
HttpURLConnection urlConnection = null;
URL url = new URL(subfilelink);
urlConnection = (HttpURLConnection) url.openConnection();
is = new BufferedInputStream(urlConnection.getInputStream());
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
// TODO
fileName = retrieveInputFromUrl(subfilelink, inputName, filesList);
LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(fileNameTemp));
} else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
is.close();
fos.flush();
fos.close();
urlConnection.disconnect();
filesList.add(of);
LOGGER.debug("Created local file: {}", of.getAbsolutePath());
}
} else {
LOGGER.debug("Complex Input payload is file");
@ -325,6 +303,53 @@ public class InputsManager {
return filesList;
}
private String retrieveInputFromUrl(String subfilelink, String inputName, List<File> filesList) throws Exception {
String fileName = "";
for (int i = 0; i < RETRIEVE_INPUT_MAX_RETRY; i++) {
try {
InputStream is = null;
HttpURLConnection urlConnection = null;
URL url = new URL(subfilelink);
urlConnection = (HttpURLConnection) url.openConnection();
is = new BufferedInputStream(urlConnection.getInputStream());
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(fileNameTemp));
} else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
is.close();
fos.flush();
fos.close();
urlConnection.disconnect();
filesList.add(of);
LOGGER.debug("Created local file: {}", of.getAbsolutePath());
return fileName;
} catch (Exception e) {
LOGGER.error("ERROR retrieving Input URL: " + subfilelink, e);
try {
Thread.sleep(RETRY_INTERVAL_TIME);
} catch (InterruptedException e1) {
}
}
}
throw new Exception("Retry limit exceeded. Error retrieving Input URL : " + subfilelink);
}
public void createTable(String tableName, File tableFile, AlgorithmConfiguration config,
DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception {
@ -484,7 +509,7 @@ public class InputsManager {
}
private void saveInputData(String name, String description, String payload) {
LOGGER.debug("SaveInputData [name="+name+", description="+description+", payload="+payload+"]");
LOGGER.debug("SaveInputData [name=" + name + ", description=" + description + ", payload=" + payload + "]");
String id = name;
DataProvenance provenance = DataProvenance.IMPORTED;
String creationDate = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis());