From 690126d6c03539b139ec60be2c9547fc1aa8fadb Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Tue, 12 May 2020 11:24:31 +0200 Subject: [PATCH] Added Retry in Input Parameter URL retrieve for fix StorageHub sync. --- pom.xml | 6 +- .../synchserver/mapping/InputsManager.java | 83 ++++++++++++------- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index b1fa05b..8e87dfa 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.dataanalysis dataminer - 1.5.9-SNAPSHOT + 1.5.10-SNAPSHOT dataminer An e-Infrastructure service providing state-of-the art DataMining algorithms and ecological modelling approaches under the Web Processing Service (WPS) standard. @@ -74,12 +74,12 @@ org.gcube.dataanalysis 52n-wps-algorithm-gcube - [3.6.1,3.7.0) + [3.6.1,3.7.0-SNAPSHOT) org.gcube.dataanalysis 52n-wps-server-gcube - [3.6.1, 3.7.0) + [3.6.1, 3.7.0-SNAPSHOT) org.gcube.dataanalysis diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java index 074b72c..de4af51 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/InputsManager.java @@ -5,8 +5,10 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; +import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -41,6 +43,8 @@ import org.slf4j.LoggerFactory; public class InputsManager { + private static final int RETRIEVE_INPUT_MAX_RETRY = 5; + private static final int RETRY_INTERVAL_TIME = 2000; private static final Logger LOGGER = LoggerFactory.getLogger(InputsManager.class); LinkedHashMap inputs; @@ -239,35 +243,9 @@ public class InputsManager { LOGGER.debug("Managing link: {}", subfilelink); if (subfilelink.length() == 0) continue; - InputStream is = null; - HttpURLConnection urlConnection = null; - URL url = new URL(subfilelink); - urlConnection = (HttpURLConnection) url.openConnection(); - is = new BufferedInputStream(urlConnection.getInputStream()); - // retrieve payload: for test purpose only - String fileNameTemp = inputNameFromHttpHeader(subfilelink); + // TODO + fileName = retrieveInputFromUrl(subfilelink, inputName, filesList); - LOGGER.debug("the fileNameTemp is {}", fileNameTemp); - - if (fileNameTemp != null && !fileNameTemp.isEmpty()) { - fileName = String.format("%s_(%s).%s", inputName, computationId, - FilenameUtils.getExtension(fileNameTemp)); - } else { - fileName = String.format("%s_(%s).%s", inputName, computationId, - FilenameUtils.getExtension(inputName)); - - } - LOGGER.debug("the name of the generated file is {}", fileName); - - File of = new File(config.getPersistencePath(), fileName); - FileOutputStream fos = new FileOutputStream(of); - IOUtils.copy(is, fos); - is.close(); - fos.flush(); - fos.close(); - urlConnection.disconnect(); - filesList.add(of); - LOGGER.debug("Created local file: {}", of.getAbsolutePath()); } } else { LOGGER.debug("Complex Input payload is file"); @@ -325,6 +303,53 @@ public class InputsManager { return filesList; } + private String retrieveInputFromUrl(String subfilelink, String inputName, List filesList) throws Exception { + String fileName = ""; + for (int i = 0; i < RETRIEVE_INPUT_MAX_RETRY; i++) { + try { + InputStream is = null; + HttpURLConnection urlConnection = null; + URL url = new URL(subfilelink); + urlConnection = (HttpURLConnection) url.openConnection(); + is = new BufferedInputStream(urlConnection.getInputStream()); + // retrieve payload: for test purpose only + String fileNameTemp = inputNameFromHttpHeader(subfilelink); + + LOGGER.debug("the fileNameTemp is {}", fileNameTemp); + + if (fileNameTemp != null && !fileNameTemp.isEmpty()) { + fileName = String.format("%s_(%s).%s", inputName, computationId, + FilenameUtils.getExtension(fileNameTemp)); + } else { + fileName = String.format("%s_(%s).%s", inputName, computationId, + FilenameUtils.getExtension(inputName)); + + } + LOGGER.debug("the name of the generated file is {}", fileName); + + File of = new File(config.getPersistencePath(), fileName); + FileOutputStream fos = new FileOutputStream(of); + IOUtils.copy(is, fos); + is.close(); + fos.flush(); + fos.close(); + urlConnection.disconnect(); + filesList.add(of); + LOGGER.debug("Created local file: {}", of.getAbsolutePath()); + return fileName; + } catch (Exception e) { + LOGGER.error("ERROR retrieving Input URL: " + subfilelink, e); + try { + Thread.sleep(RETRY_INTERVAL_TIME); + } catch (InterruptedException e1) { + + } + } + } + throw new Exception("Retry limit exceeded. Error retrieving Input URL : " + subfilelink); + + } + public void createTable(String tableName, File tableFile, AlgorithmConfiguration config, DatabaseInfo supportDatabaseInfo, String inputTableTemplate) throws Exception { @@ -484,7 +509,7 @@ public class InputsManager { } private void saveInputData(String name, String description, String payload) { - LOGGER.debug("SaveInputData [name="+name+", description="+description+", payload="+payload+"]"); + LOGGER.debug("SaveInputData [name=" + name + ", description=" + description + ", payload=" + payload + "]"); String id = name; DataProvenance provenance = DataProvenance.IMPORTED; String creationDate = new java.text.SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format(System.currentTimeMillis());