diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index bf90722..49aa440 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -3,10 +3,12 @@ package eu.dnetlib.dhp.oa.graph.dump; import java.io.Serializable; import java.util.Optional; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; +import org.apache.http.HttpStatus; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException; @@ -18,6 +20,9 @@ public class SendToZenodoHDFS implements Serializable { private static final String NEW = "new"; // to be used for a brand new deposition in zenodo private static final String VERSION = "version"; // to be used to upload a new version of a published deposition private static final String UPDATE = "update"; // to upload content to an open deposition not published + private static final Integer NUMBER_OF_RETRIES = 5; + private static final Integer DELAY = 10; + private static final Integer MULTIPLIER = 5; public static void main(final String[] args) throws Exception, MissingConceptDoiException { final ArgumentApplicationParser parser = new ArgumentApplicationParser( @@ -79,20 +84,33 @@ public class SendToZenodoHDFS implements Serializable { Path p = fileStatus.getPath(); String pString = p.toString(); + boolean retry = true; + int numberOfRetries = 0; + if (!pString.endsWith("_SUCCESS")) { String name = pString.substring(pString.lastIndexOf("/") + 1); FSDataInputStream inputStream = fileSystem.open(p); - zenodoApiClient.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); - } + while (retry && numberOfRetries < NUMBER_OF_RETRIES) { + int response_code = zenodoApiClient + .uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); + if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) { + retry = false; + } else { + numberOfRetries += 1; + TimeUnit.SECONDS.sleep(DELAY * MULTIPLIER ^ numberOfRetries); + } + } + if (numberOfRetries == NUMBER_OF_RETRIES) { + throw new RuntimeException("reached the maximun number or retries to upload on Zenodo"); + } + } + TimeUnit.SECONDS.sleep(DELAY); } if (!metadata.equals("")) { zenodoApiClient.sendMretadata(metadata); } -// if (Boolean.TRUE.equals(publish)) { -// zenodoApiClient.publish(); -// } } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java index 59f6055..721930d 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java @@ -21,15 +21,19 @@ import org.apache.http.impl.client.CloseableHttpClient; // import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.gson.Gson; +import eu.dnetlib.dhp.oa.graph.dump.SparkCopyGraph; import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel; import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList; +import kotlin.Pair; import okhttp3.*; public class ZenodoAPIClient implements Serializable { - + private static final Logger log = LoggerFactory.getLogger(ZenodoAPIClient.class); String urlString; String bucket; @@ -242,8 +246,8 @@ public class ZenodoAPIClient implements Serializable { .build(); try (Response response = httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) - throw new IOException("Unexpected code " + response + response.body().string()); +// if (!response.isSuccessful()) +// throw new IOException("Unexpected code " + response + response.body().string()); return response.code(); } } @@ -347,8 +351,10 @@ public class ZenodoAPIClient implements Serializable { try (Response response = httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) + if (!response.isSuccessful()) { + //log.info("response headers " + response.headers().toString()); throw new IOException("Unexpected code " + response + response.body().string()); + } ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class); String latest_draft = zenodoModel.getLinks().getLatest_draft(); @@ -396,7 +402,12 @@ public class ZenodoAPIClient implements Serializable { } private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException { - + String urlString = "http://checkip.amazonaws.com/"; + URL url = new URL(urlString); + try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) { + log.info(br.readLine()); + } +//log.info(); ZenodoModelList zenodoModelList = new Gson() .fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class); @@ -433,21 +444,28 @@ public class ZenodoAPIClient implements Serializable { OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); - HttpUrl.Builder urlBuilder = HttpUrl.parse(urlString).newBuilder(); - urlBuilder.addQueryParameter("page", page); + HttpUrl.Builder urlBuilder = HttpUrl + .parse(urlString)// + "?access_token=" + access_token + "&page=" + page) + .newBuilder(); + urlBuilder.addQueryParameter("page", page); String url = urlBuilder.build().toString(); Request request = new Request.Builder() .url(url) - .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) .get() .build(); + log.info("URL: " + request.url().toString()); + log.info("Headers: " + request.headers().toString()); + try (Response response = httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) + if (!response.isSuccessful()) { + log.info("response headers: " + response.headers().toString()); throw new IOException("Unexpected code " + response + response.body().string()); + } return response.body().string(); diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/projectsubset/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/projectsubset/oozie_app/workflow.xml index 620f761..54ec370 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/projectsubset/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/projectsubset/oozie_app/workflow.xml @@ -89,7 +89,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java index 25aec29..410a452 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java @@ -146,7 +146,7 @@ public class ZenodoUploadTest { ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, ACCESS_TOKEN); - client.newVersion("1210237"); + client.newVersion("4559725"); File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar"); // File file = new File(getClass()