From f2b890f8a8f8b8ab5a6958c0c852c89b5b63e375 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 24 Oct 2023 08:55:14 +0200 Subject: [PATCH] - --- .../dhp/oa/graph/dump/SendToZenodoHDFS.java | 9 + .../dhp/oa/zenodoapi/ZenodoAPIClient.java | 172 ++++-------------- 2 files changed, 48 insertions(+), 133 deletions(-) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 49aa440..72801ee 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -9,6 +9,9 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.http.HttpStatus; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException; @@ -24,6 +27,8 @@ public class SendToZenodoHDFS implements Serializable { private static final Integer DELAY = 10; private static final Integer MULTIPLIER = 5; + private static final Logger log = LoggerFactory.getLogger(SendToZenodoHDFS.class); + public static void main(final String[] args) throws Exception, MissingConceptDoiException { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils @@ -89,10 +94,12 @@ public class SendToZenodoHDFS implements Serializable { if (!pString.endsWith("_SUCCESS")) { String name = pString.substring(pString.lastIndexOf("/") + 1); + log.info("Upoloading: {}", name); FSDataInputStream inputStream = fileSystem.open(p); while (retry && numberOfRetries < NUMBER_OF_RETRIES) { int response_code = zenodoApiClient .uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); + log.info("response code: {}", response_code); if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) { retry = false; } else { @@ -105,7 +112,9 @@ public class SendToZenodoHDFS implements Serializable { } } + log.info(DateTime.now().toDateTimeISO().toString()); TimeUnit.SECONDS.sleep(DELAY); + log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString()); } if (!metadata.equals("")) { zenodoApiClient.sendMretadata(metadata); diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java index 721930d..d3565c6 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java @@ -15,21 +15,14 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -// import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.Gson; -import eu.dnetlib.dhp.oa.graph.dump.SparkCopyGraph; import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel; import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList; -import kotlin.Pair; import okhttp3.*; public class ZenodoAPIClient implements Serializable { @@ -76,34 +69,6 @@ public class ZenodoAPIClient implements Serializable { * @return response code * @throws IOException */ -// public int newDeposition() throws IOException { -// String json = "{}"; -// -// URL url = new URL(urlString); -// HttpURLConnection conn = (HttpURLConnection) url.openConnection(); -// conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()); -// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); -// conn.setRequestMethod("POST"); -// conn.setDoOutput(true); -// try (OutputStream os = conn.getOutputStream()) { -// byte[] input = json.getBytes("utf-8"); -// os.write(input, 0, input.length); -// } -// -// String body = getBody(conn); -// -// int responseCode = conn.getResponseCode(); -// conn.disconnect(); -// -// if (!checkOKStatus(responseCode)) -// throw new IOException("Unexpected code " + responseCode + body); -// -// ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class); -// this.bucket = newSubmission.getLinks().getBucket(); -// this.deposition_id = newSubmission.getId(); -// -// return responseCode; -// } public int newDeposition() throws IOException { String json = "{}"; @@ -137,50 +102,6 @@ public class ZenodoAPIClient implements Serializable { } -// public int uploadIS2(InputStream is, String fileName) throws IOException { -// -// final String crlf = "\r\n"; -// final String twoHyphens = "--"; -// final String boundary = "*****"; -// -// HttpPut put = new HttpPut(bucket + "/" + fileName); -// -// put.addHeader(HttpHeaders.CONTENT_TYPE, "application/zip"); -// put.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); -// -// put.setEntity(new InputStreamEntity(is)); -// -// int statusCode; -// try (CloseableHttpClient client = HttpClients.createDefault()) { -// CloseableHttpResponse response = client.execute(put); -// statusCode = response.getStatusLine().getStatusCode(); -// -// } -// -// if (!checkOKStatus(statusCode)) { -// throw new IOException("Unexpected code " + statusCode); -// } -// -// return statusCode; -// } - -// public int publish() throws IOException { -// String json = "{}"; -// HttpPost post = new HttpPost(urlString + "/" + deposition_id + "/actions/publish"); -// post.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); -// post.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); -// post.setEntity(new StringEntity(json)); -// int statusCode; -// try (CloseableHttpClient client = HttpClients.createDefault()) { -// CloseableHttpResponse response = client.execute(post); -// statusCode = response.getStatusLine().getStatusCode(); -// } -// if (!checkOKStatus(statusCode)) { -// throw new IOException("Unexpected code " + statusCode); -// } -// return statusCode; -// } - /** * Upload files in Zenodo. * @@ -302,39 +223,40 @@ public class ZenodoAPIClient implements Serializable { * @throws IOException * @throws MissingConceptDoiException */ -// public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException { -// setDepositionId(concept_rec_id, 1); -// String json = "{}"; -// -// URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion"); -// HttpURLConnection conn = (HttpURLConnection) url.openConnection(); -// -// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); -// conn.setDoOutput(true); -// conn.setRequestMethod("POST"); -// -// try (OutputStream os = conn.getOutputStream()) { -// byte[] input = json.getBytes("utf-8"); -// os.write(input, 0, input.length); -// -// } -// -// String body = getBody(conn); -// -// int responseCode = conn.getResponseCode(); -// -// conn.disconnect(); -// if (!checkOKStatus(responseCode)) -// throw new IOException("Unexpected code " + responseCode + body); -// -// ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); -// String latest_draft = zenodoModel.getLinks().getLatest_draft(); -// deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1); -// bucket = getBucket(latest_draft); -// -// return responseCode; -// -// } + public int newVersion2(String concept_rec_id) throws Exception, MissingConceptDoiException { + setDepositionId(concept_rec_id, 1); + String json = "{}"; + + URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + + conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + + try (OutputStream os = conn.getOutputStream()) { + byte[] input = json.getBytes("utf-8"); + os.write(input, 0, input.length); + + } + + String body = getBody(conn); + + int responseCode = conn.getResponseCode(); + + conn.disconnect(); + if (!checkOKStatus(responseCode)) + throw new IOException("Unexpected code " + responseCode + body); + + ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); + String latest_draft = zenodoModel.getLinks().getLatest_draft(); + deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1); + bucket = getBucket(latest_draft); + + return responseCode; + + } + public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException { setDepositionId(concept_rec_id, 1); String json = "{}"; @@ -352,7 +274,7 @@ public class ZenodoAPIClient implements Serializable { try (Response response = httpClient.newCall(request).execute()) { if (!response.isSuccessful()) { - //log.info("response headers " + response.headers().toString()); + // log.info("response headers " + response.headers().toString()); throw new IOException("Unexpected code " + response + response.body().string()); } @@ -424,22 +346,6 @@ public class ZenodoAPIClient implements Serializable { } -// private String getPrevDepositions(String page) throws Exception { -// -// HttpGet get = new HttpGet(urlString); -// URI uri = new URIBuilder(get.getURI()).addParameter("page", page).build(); -// -// get.setURI(uri); -// -// get.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()); -// get.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); -// try (CloseableHttpClient client = HttpClients.createDefault()) { -// CloseableHttpResponse response = client.execute(get); -// final String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); -// return body; -// } -// } - private String getPrevDepositions(String page) throws IOException { OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); @@ -447,13 +353,13 @@ public class ZenodoAPIClient implements Serializable { HttpUrl.Builder urlBuilder = HttpUrl .parse(urlString)// + "?access_token=" + access_token + "&page=" + page) .newBuilder(); - urlBuilder.addQueryParameter("page", page); + urlBuilder.addQueryParameter("page", page); String url = urlBuilder.build().toString(); Request request = new Request.Builder() .url(url) - .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) .get() .build(); @@ -463,7 +369,7 @@ public class ZenodoAPIClient implements Serializable { try (Response response = httpClient.newCall(request).execute()) { if (!response.isSuccessful()) { - log.info("response headers: " + response.headers().toString()); + log.info("response headers: " + response.headers()); throw new IOException("Unexpected code " + response + response.body().string()); }