add retry with exponential backof and delay between the calls
This commit is contained in:
parent
32d64dd7a1
commit
5529bbe3cc
|
@ -3,10 +3,12 @@ package eu.dnetlib.dhp.oa.graph.dump;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.*;
|
import org.apache.hadoop.fs.*;
|
||||||
|
import org.apache.http.HttpStatus;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
|
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
|
||||||
|
@ -18,6 +20,9 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
private static final String NEW = "new"; // to be used for a brand new deposition in zenodo
|
private static final String NEW = "new"; // to be used for a brand new deposition in zenodo
|
||||||
private static final String VERSION = "version"; // to be used to upload a new version of a published deposition
|
private static final String VERSION = "version"; // to be used to upload a new version of a published deposition
|
||||||
private static final String UPDATE = "update"; // to upload content to an open deposition not published
|
private static final String UPDATE = "update"; // to upload content to an open deposition not published
|
||||||
|
private static final Integer NUMBER_OF_RETRIES = 5;
|
||||||
|
private static final Integer DELAY = 10;
|
||||||
|
private static final Integer MULTIPLIER = 5;
|
||||||
|
|
||||||
public static void main(final String[] args) throws Exception, MissingConceptDoiException {
|
public static void main(final String[] args) throws Exception, MissingConceptDoiException {
|
||||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||||
|
@ -79,20 +84,33 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
|
|
||||||
Path p = fileStatus.getPath();
|
Path p = fileStatus.getPath();
|
||||||
String pString = p.toString();
|
String pString = p.toString();
|
||||||
|
boolean retry = true;
|
||||||
|
int numberOfRetries = 0;
|
||||||
|
|
||||||
if (!pString.endsWith("_SUCCESS")) {
|
if (!pString.endsWith("_SUCCESS")) {
|
||||||
String name = pString.substring(pString.lastIndexOf("/") + 1);
|
String name = pString.substring(pString.lastIndexOf("/") + 1);
|
||||||
FSDataInputStream inputStream = fileSystem.open(p);
|
FSDataInputStream inputStream = fileSystem.open(p);
|
||||||
zenodoApiClient.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
|
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
|
||||||
}
|
int response_code = zenodoApiClient
|
||||||
|
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
|
||||||
|
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
|
||||||
|
retry = false;
|
||||||
|
} else {
|
||||||
|
numberOfRetries += 1;
|
||||||
|
TimeUnit.SECONDS.sleep(DELAY * MULTIPLIER ^ numberOfRetries);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (numberOfRetries == NUMBER_OF_RETRIES) {
|
||||||
|
throw new RuntimeException("reached the maximun number or retries to upload on Zenodo");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
TimeUnit.SECONDS.sleep(DELAY);
|
||||||
}
|
}
|
||||||
if (!metadata.equals("")) {
|
if (!metadata.equals("")) {
|
||||||
zenodoApiClient.sendMretadata(metadata);
|
zenodoApiClient.sendMretadata(metadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
// if (Boolean.TRUE.equals(publish)) {
|
|
||||||
// zenodoApiClient.publish();
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,15 +21,19 @@ import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
// import org.apache.http.impl.client.HttpClients;
|
// import org.apache.http.impl.client.HttpClients;
|
||||||
import org.apache.http.util.EntityUtils;
|
import org.apache.http.util.EntityUtils;
|
||||||
import org.jetbrains.annotations.NotNull;
|
import org.jetbrains.annotations.NotNull;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.oa.graph.dump.SparkCopyGraph;
|
||||||
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
|
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
|
||||||
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
|
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
|
||||||
|
import kotlin.Pair;
|
||||||
import okhttp3.*;
|
import okhttp3.*;
|
||||||
|
|
||||||
public class ZenodoAPIClient implements Serializable {
|
public class ZenodoAPIClient implements Serializable {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(ZenodoAPIClient.class);
|
||||||
String urlString;
|
String urlString;
|
||||||
String bucket;
|
String bucket;
|
||||||
|
|
||||||
|
@ -242,8 +246,8 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
if (!response.isSuccessful())
|
// if (!response.isSuccessful())
|
||||||
throw new IOException("Unexpected code " + response + response.body().string());
|
// throw new IOException("Unexpected code " + response + response.body().string());
|
||||||
return response.code();
|
return response.code();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -347,8 +351,10 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
if (!response.isSuccessful())
|
if (!response.isSuccessful()) {
|
||||||
|
//log.info("response headers " + response.headers().toString());
|
||||||
throw new IOException("Unexpected code " + response + response.body().string());
|
throw new IOException("Unexpected code " + response + response.body().string());
|
||||||
|
}
|
||||||
|
|
||||||
ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
|
ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
|
||||||
String latest_draft = zenodoModel.getLinks().getLatest_draft();
|
String latest_draft = zenodoModel.getLinks().getLatest_draft();
|
||||||
|
@ -396,7 +402,12 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException {
|
private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException {
|
||||||
|
String urlString = "http://checkip.amazonaws.com/";
|
||||||
|
URL url = new URL(urlString);
|
||||||
|
try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
|
||||||
|
log.info(br.readLine());
|
||||||
|
}
|
||||||
|
//log.info();
|
||||||
ZenodoModelList zenodoModelList = new Gson()
|
ZenodoModelList zenodoModelList = new Gson()
|
||||||
.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
|
.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
|
||||||
|
|
||||||
|
@ -433,21 +444,28 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
HttpUrl.Builder urlBuilder = HttpUrl.parse(urlString).newBuilder();
|
HttpUrl.Builder urlBuilder = HttpUrl
|
||||||
urlBuilder.addQueryParameter("page", page);
|
.parse(urlString)// + "?access_token=" + access_token + "&page=" + page)
|
||||||
|
.newBuilder();
|
||||||
|
urlBuilder.addQueryParameter("page", page);
|
||||||
String url = urlBuilder.build().toString();
|
String url = urlBuilder.build().toString();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(url)
|
.url(url)
|
||||||
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.get()
|
.get()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
log.info("URL: " + request.url().toString());
|
||||||
|
log.info("Headers: " + request.headers().toString());
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
if (!response.isSuccessful())
|
if (!response.isSuccessful()) {
|
||||||
|
log.info("response headers: " + response.headers().toString());
|
||||||
throw new IOException("Unexpected code " + response + response.body().string());
|
throw new IOException("Unexpected code " + response + response.body().string());
|
||||||
|
}
|
||||||
|
|
||||||
return response.body().string();
|
return response.body().string();
|
||||||
|
|
||||||
|
|
|
@ -89,7 +89,7 @@
|
||||||
</property>
|
</property>
|
||||||
</configuration>
|
</configuration>
|
||||||
</global>
|
</global>
|
||||||
<start to="dump_project"/>
|
<start to="send_zenodo"/>
|
||||||
<kill name="Kill">
|
<kill name="Kill">
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||||
</kill>
|
</kill>
|
||||||
|
|
|
@ -146,7 +146,7 @@ public class ZenodoUploadTest {
|
||||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
ACCESS_TOKEN);
|
ACCESS_TOKEN);
|
||||||
|
|
||||||
client.newVersion("1210237");
|
client.newVersion("4559725");
|
||||||
|
|
||||||
File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar");
|
File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar");
|
||||||
// File file = new File(getClass()
|
// File file = new File(getClass()
|
||||||
|
|
Loading…
Reference in New Issue