add retry with exponential backof and delay between the calls

dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java

@@ -3,10 +3,12 @@ package eu.dnetlib.dhp.oa.graph.dump;
 
 import java.io.Serializable;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.*;
+import org.apache.http.HttpStatus;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
@@ -18,6 +20,9 @@ public class SendToZenodoHDFS implements Serializable {
 	private static final String NEW = "new"; // to be used for a brand new deposition in zenodo
 	private static final String VERSION = "version"; // to be used to upload a new version of a published deposition
 	private static final String UPDATE = "update"; // to upload content to an open deposition not published
+	private static final Integer NUMBER_OF_RETRIES = 5;
+	private static final Integer DELAY = 10;
+	private static final Integer MULTIPLIER = 5;
 
 	public static void main(final String[] args) throws Exception, MissingConceptDoiException {
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -79,20 +84,33 @@ public class SendToZenodoHDFS implements Serializable {
 
 			Path p = fileStatus.getPath();
 			String pString = p.toString();
+			boolean retry = true;
+			int numberOfRetries = 0;
+
 			if (!pString.endsWith("_SUCCESS")) {
 				String name = pString.substring(pString.lastIndexOf("/") + 1);
 				FSDataInputStream inputStream = fileSystem.open(p);
-				zenodoApiClient.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
-			}
+				while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
+					int response_code = zenodoApiClient
+						.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
+					if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
+						retry = false;
+					} else {
+						numberOfRetries += 1;
+						TimeUnit.SECONDS.sleep(DELAY * MULTIPLIER ^ numberOfRetries);
+					}
+				}
+				if (numberOfRetries == NUMBER_OF_RETRIES) {
+					throw new RuntimeException("reached the maximun number or retries to upload on Zenodo");
+				}
 
+			}
+			TimeUnit.SECONDS.sleep(DELAY);
 		}
 		if (!metadata.equals("")) {
 			zenodoApiClient.sendMretadata(metadata);
 		}
 
-//		if (Boolean.TRUE.equals(publish)) {
-//			zenodoApiClient.publish();
-//		}
 	}
 
 }

dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java

@@ -21,15 +21,19 @@ import org.apache.http.impl.client.CloseableHttpClient;
 // import org.apache.http.impl.client.HttpClients;
 import org.apache.http.util.EntityUtils;
 import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.gson.Gson;
 
+import eu.dnetlib.dhp.oa.graph.dump.SparkCopyGraph;
 import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
 import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
+import kotlin.Pair;
 import okhttp3.*;
 
 public class ZenodoAPIClient implements Serializable {
-
+	private static final Logger log = LoggerFactory.getLogger(ZenodoAPIClient.class);
 	String urlString;
 	String bucket;
 
@@ -242,8 +246,8 @@ public class ZenodoAPIClient implements Serializable {
 			.build();
 
 		try (Response response = httpClient.newCall(request).execute()) {
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
+//			if (!response.isSuccessful())
+//				throw new IOException("Unexpected code " + response + response.body().string());
 			return response.code();
 		}
 	}
@@ -347,8 +351,10 @@ public class ZenodoAPIClient implements Serializable {
 
 		try (Response response = httpClient.newCall(request).execute()) {
 
-			if (!response.isSuccessful())
+			if (!response.isSuccessful()) {
+				//log.info("response headers " + response.headers().toString());
 				throw new IOException("Unexpected code " + response + response.body().string());
+			}
 
 			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
 			String latest_draft = zenodoModel.getLinks().getLatest_draft();
@@ -396,7 +402,12 @@ public class ZenodoAPIClient implements Serializable {
 	}
 
 	private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException {
-
+		String urlString = "http://checkip.amazonaws.com/";
+		URL url = new URL(urlString);
+		try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
+			log.info(br.readLine());
+		}
+//log.info();
 		ZenodoModelList zenodoModelList = new Gson()
 			.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
 
@@ -433,21 +444,28 @@ public class ZenodoAPIClient implements Serializable {
 
 		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
 
-		HttpUrl.Builder urlBuilder = HttpUrl.parse(urlString).newBuilder();
-		urlBuilder.addQueryParameter("page", page);
+		HttpUrl.Builder urlBuilder = HttpUrl
+			.parse(urlString)// + "?access_token=" + access_token + "&page=" + page)
+			.newBuilder();
+		 urlBuilder.addQueryParameter("page", page);
 		String url = urlBuilder.build().toString();
 
 		Request request = new Request.Builder()
 			.url(url)
-			.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
-			.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
+			 .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
+			 .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
 			.get()
 			.build();
 
+		log.info("URL: " + request.url().toString());
+		log.info("Headers: " + request.headers().toString());
+
 		try (Response response = httpClient.newCall(request).execute()) {
 
-			if (!response.isSuccessful())
+			if (!response.isSuccessful()) {
+				log.info("response headers: " + response.headers().toString());
 				throw new IOException("Unexpected code " + response + response.body().string());
+			}
 
 			return response.body().string();
 

dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/projectsubset/oozie_app/workflow.xml

@@ -89,7 +89,7 @@
             </property>
         </configuration>
     </global>
-    <start to="dump_project"/>
+    <start to="send_zenodo"/>
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>

dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java

@@ -146,7 +146,7 @@ public class ZenodoUploadTest {
 		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
 			ACCESS_TOKEN);
 
-		client.newVersion("1210237");
+		client.newVersion("4559725");
 
 		File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar");
 //		File file = new File(getClass()