D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

-

This commit is contained in:
Miriam Baglioni 2023-10-24 08:55:14 +02:00
parent 5529bbe3cc
commit f2b890f8a8
2 changed files with 48 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java
View File

@ -9,6 +9,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.http.HttpStatus;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
@ -24,6 +27,8 @@ public class SendToZenodoHDFS implements Serializable {
private static final Integer DELAY = 10;
private static final Integer MULTIPLIER = 5;
private static final Logger log = LoggerFactory.getLogger(SendToZenodoHDFS.class);
public static void main(final String[] args) throws Exception, MissingConceptDoiException {
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils
@ -89,10 +94,12 @@ public class SendToZenodoHDFS implements Serializable {
if (!pString.endsWith("_SUCCESS")) {
String name = pString.substring(pString.lastIndexOf("/") + 1);
log.info("Upoloading: {}", name);
FSDataInputStream inputStream = fileSystem.open(p);
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
int response_code = zenodoApiClient
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
log.info("response code: {}", response_code);
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
retry = false;
} else {
@ -105,7 +112,9 @@ public class SendToZenodoHDFS implements Serializable {
}
}
log.info(DateTime.now().toDateTimeISO().toString());
TimeUnit.SECONDS.sleep(DELAY);
log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString());
}
if (!metadata.equals("")) {
zenodoApiClient.sendMretadata(metadata);

172
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java
View File

@ -15,21 +15,14 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
// import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import eu.dnetlib.dhp.oa.graph.dump.SparkCopyGraph;
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
import kotlin.Pair;
import okhttp3.*;
public class ZenodoAPIClient implements Serializable {
@ -76,34 +69,6 @@ public class ZenodoAPIClient implements Serializable {
* @return response code
* @throws IOException
*/
// public int newDeposition() throws IOException {
// String json = "{}";
//
// URL url = new URL(urlString);
// HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
// conn.setRequestMethod("POST");
// conn.setDoOutput(true);
// try (OutputStream os = conn.getOutputStream()) {
// byte[] input = json.getBytes("utf-8");
// os.write(input, 0, input.length);
// }
//
// String body = getBody(conn);
//
// int responseCode = conn.getResponseCode();
// conn.disconnect();
//
// if (!checkOKStatus(responseCode))
// throw new IOException("Unexpected code " + responseCode + body);
//
// ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class);
// this.bucket = newSubmission.getLinks().getBucket();
// this.deposition_id = newSubmission.getId();
//
// return responseCode;
// }
public int newDeposition() throws IOException {
String json = "{}";
@ -137,50 +102,6 @@ public class ZenodoAPIClient implements Serializable {
}
// public int uploadIS2(InputStream is, String fileName) throws IOException {
//
// final String crlf = "\r\n";
// final String twoHyphens = "--";
// final String boundary = "*****";
//
// HttpPut put = new HttpPut(bucket + "/" + fileName);
//
// put.addHeader(HttpHeaders.CONTENT_TYPE, "application/zip");
// put.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
//
// put.setEntity(new InputStreamEntity(is));
//
// int statusCode;
// try (CloseableHttpClient client = HttpClients.createDefault()) {
// CloseableHttpResponse response = client.execute(put);
// statusCode = response.getStatusLine().getStatusCode();
//
// }
//
// if (!checkOKStatus(statusCode)) {
// throw new IOException("Unexpected code " + statusCode);
// }
//
// return statusCode;
// }
// public int publish() throws IOException {
// String json = "{}";
// HttpPost post = new HttpPost(urlString + "/" + deposition_id + "/actions/publish");
// post.addHeader(HttpHeaders.CONTENT_TYPE, "application/json");
// post.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
// post.setEntity(new StringEntity(json));
// int statusCode;
// try (CloseableHttpClient client = HttpClients.createDefault()) {
// CloseableHttpResponse response = client.execute(post);
// statusCode = response.getStatusLine().getStatusCode();
// }
// if (!checkOKStatus(statusCode)) {
// throw new IOException("Unexpected code " + statusCode);
// }
// return statusCode;
// }
/**
* Upload files in Zenodo.
*
@ -302,39 +223,40 @@ public class ZenodoAPIClient implements Serializable {
* @throws IOException
* @throws MissingConceptDoiException
*/
// public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException {
// setDepositionId(concept_rec_id, 1);
// String json = "{}";
//
// URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion");
// HttpURLConnection conn = (HttpURLConnection) url.openConnection();
//
// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
// conn.setDoOutput(true);
// conn.setRequestMethod("POST");
//
// try (OutputStream os = conn.getOutputStream()) {
// byte[] input = json.getBytes("utf-8");
// os.write(input, 0, input.length);
//
// }
//
// String body = getBody(conn);
//
// int responseCode = conn.getResponseCode();
//
// conn.disconnect();
// if (!checkOKStatus(responseCode))
// throw new IOException("Unexpected code " + responseCode + body);
//
// ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
// String latest_draft = zenodoModel.getLinks().getLatest_draft();
// deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
// bucket = getBucket(latest_draft);
//
// return responseCode;
//
// }
public int newVersion2(String concept_rec_id) throws Exception, MissingConceptDoiException {
setDepositionId(concept_rec_id, 1);
String json = "{}";
URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
conn.setDoOutput(true);
conn.setRequestMethod("POST");
try (OutputStream os = conn.getOutputStream()) {
byte[] input = json.getBytes("utf-8");
os.write(input, 0, input.length);
}
String body = getBody(conn);
int responseCode = conn.getResponseCode();
conn.disconnect();
if (!checkOKStatus(responseCode))
throw new IOException("Unexpected code " + responseCode + body);
ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
String latest_draft = zenodoModel.getLinks().getLatest_draft();
deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
bucket = getBucket(latest_draft);
return responseCode;
}
public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException {
setDepositionId(concept_rec_id, 1);
String json = "{}";
@ -352,7 +274,7 @@ public class ZenodoAPIClient implements Serializable {
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
//log.info("response headers " + response.headers().toString());
// log.info("response headers " + response.headers().toString());
throw new IOException("Unexpected code " + response + response.body().string());
}
@ -424,22 +346,6 @@ public class ZenodoAPIClient implements Serializable {
}
// private String getPrevDepositions(String page) throws Exception {
//
// HttpGet get = new HttpGet(urlString);
// URI uri = new URIBuilder(get.getURI()).addParameter("page", page).build();
//
// get.setURI(uri);
//
// get.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
// get.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
// try (CloseableHttpClient client = HttpClients.createDefault()) {
// CloseableHttpResponse response = client.execute(get);
// final String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
// return body;
// }
// }
private String getPrevDepositions(String page) throws IOException {
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
@ -447,13 +353,13 @@ public class ZenodoAPIClient implements Serializable {
HttpUrl.Builder urlBuilder = HttpUrl
.parse(urlString)// + "?access_token=" + access_token + "&page=" + page)
.newBuilder();
urlBuilder.addQueryParameter("page", page);
urlBuilder.addQueryParameter("page", page);
String url = urlBuilder.build().toString();
Request request = new Request.Builder()
.url(url)
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
.get()
.build();
@ -463,7 +369,7 @@ public class ZenodoAPIClient implements Serializable {
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.info("response headers: " + response.headers().toString());
log.info("response headers: " + response.headers());
throw new IOException("Unexpected code " + response + response.body().string());
}