gCubeSystem
/
dataminer
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Added Retry in Input Parameter URL retrieve for fix StorageHub sync.

This commit is contained in:
Giancarlo Panichi 2020-05-12 11:24:31 +02:00
parent 34083f1357
commit 4d6318df92
2 changed files with 57 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@ -9,7 +9,7 @@
</parent> </parent>
<groupId>org.gcube.dataanalysis</groupId> <groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer</artifactId> <artifactId>dataminer</artifactId>
<version>1.5.9-SNAPSHOT</version> <version>1.5.10-SNAPSHOT</version>
<name>dataminer</name> <name>dataminer</name>
<description>An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard.</description> <description>An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard.</description>
@ -74,12 +74,12 @@
<dependency> <dependency>
<groupId>org.gcube.dataanalysis</groupId> <groupId>org.gcube.dataanalysis</groupId>
<artifactId>52n-wps-algorithm-gcube</artifactId> <artifactId>52n-wps-algorithm-gcube</artifactId>
<version>[3.6.1,3.7.0)</version> <version>[3.6.1,3.7.0-SNAPSHOT)</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.gcube.dataanalysis</groupId> <groupId>org.gcube.dataanalysis</groupId>
<artifactId>52n-wps-server-gcube</artifactId> <artifactId>52n-wps-server-gcube</artifactId>
<version>[3.6.1, 3.7.0)</version> <version>[3.6.1, 3.7.0-SNAPSHOT)</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.gcube.dataanalysis</groupId> <groupId>org.gcube.dataanalysis</groupId>

81
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java
View File

@ -5,8 +5,10 @@ import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileReader; import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
@ -41,6 +43,8 @@ import org.slf4j.LoggerFactory;
public class InputsManager { public class InputsManager {
private static final int RETRIEVE_INPUT_MAX_RETRY = 5;
private static final int RETRY_INTERVAL_TIME = 2000;
private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class); private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class);
LinkedHashMap<String, Object> inputs; LinkedHashMap<String, Object> inputs;
@ -239,35 +243,9 @@ public class InputsManager {
LOGGER.debug("Managing link: {}", subfilelink); LOGGER.debug("Managing link: {}", subfilelink);
if (subfilelink.length() == 0) if (subfilelink.length() == 0)
continue; continue;
InputStream is = null; // TODO
HttpURLConnection urlConnection = null; fileName = retrieveInputFromUrl(subfilelink, inputName, filesList);
URL url = new URL(subfilelink);
urlConnection = (HttpURLConnection) url.openConnection();
is = new BufferedInputStream(urlConnection.getInputStream());
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(fileNameTemp));
} else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
is.close();
fos.flush();
fos.close();
urlConnection.disconnect();
filesList.add(of);
LOGGER.debug("Created local file: {}", of.getAbsolutePath());
} }
} else { } else {
LOGGER.debug("Complex Input payload is file"); LOGGER.debug("Complex Input payload is file");
@ -325,6 +303,53 @@ public class InputsManager {
return filesList; return filesList;
} }
private String retrieveInputFromUrl(String subfilelink, String inputName, List<File> filesList) throws Exception {
String fileName = "";
for (int i = 0; i < RETRIEVE_INPUT_MAX_RETRY; i++) {
try {
InputStream is = null;
HttpURLConnection urlConnection = null;
URL url = new URL(subfilelink);
urlConnection = (HttpURLConnection) url.openConnection();
is = new BufferedInputStream(urlConnection.getInputStream());
// retrieve payload: for test purpose only
String fileNameTemp = inputNameFromHttpHeader(subfilelink);
LOGGER.debug("the fileNameTemp is {}", fileNameTemp);
if (fileNameTemp != null && !fileNameTemp.isEmpty()) {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(fileNameTemp));
} else {
fileName = String.format("%s_(%s).%s", inputName, computationId,
FilenameUtils.getExtension(inputName));
}
LOGGER.debug("the name of the generated file is {}", fileName);
File of = new File(config.getPersistencePath(), fileName);
FileOutputStream fos = new FileOutputStream(of);
IOUtils.copy(is, fos);
is.close();
fos.flush();
fos.close();
urlConnection.disconnect();
filesList.add(of);
LOGGER.debug("Created local file: {}", of.getAbsolutePath());
return fileName;
} catch (Exception e) {
LOGGER.error("ERROR retrieving Input URL: " + subfilelink, e);
try {
Thread.sleep(RETRY_INTERVAL_TIME);
} catch (InterruptedException e1) {
}
}
}
throw new Exception("Retry limit exceeded. Error retrieving Input URL : " + subfilelink);
}
public void createTable(String tableName, File tableFile, AlgorithmConfiguration config, public void createTable(String tableName, File tableFile, AlgorithmConfiguration config,
DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception { DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception {