This commit is contained in:
parent
5529bbe3cc
commit
f2b890f8a8
|
@ -9,6 +9,9 @@ import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.*;
|
import org.apache.hadoop.fs.*;
|
||||||
import org.apache.http.HttpStatus;
|
import org.apache.http.HttpStatus;
|
||||||
|
import org.joda.time.DateTime;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
|
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
|
||||||
|
@ -24,6 +27,8 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
private static final Integer DELAY = 10;
|
private static final Integer DELAY = 10;
|
||||||
private static final Integer MULTIPLIER = 5;
|
private static final Integer MULTIPLIER = 5;
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(SendToZenodoHDFS.class);
|
||||||
|
|
||||||
public static void main(final String[] args) throws Exception, MissingConceptDoiException {
|
public static void main(final String[] args) throws Exception, MissingConceptDoiException {
|
||||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||||
IOUtils
|
IOUtils
|
||||||
|
@ -89,10 +94,12 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
|
|
||||||
if (!pString.endsWith("_SUCCESS")) {
|
if (!pString.endsWith("_SUCCESS")) {
|
||||||
String name = pString.substring(pString.lastIndexOf("/") + 1);
|
String name = pString.substring(pString.lastIndexOf("/") + 1);
|
||||||
|
log.info("Upoloading: {}", name);
|
||||||
FSDataInputStream inputStream = fileSystem.open(p);
|
FSDataInputStream inputStream = fileSystem.open(p);
|
||||||
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
|
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
|
||||||
int response_code = zenodoApiClient
|
int response_code = zenodoApiClient
|
||||||
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
|
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
|
||||||
|
log.info("response code: {}", response_code);
|
||||||
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
|
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
|
||||||
retry = false;
|
retry = false;
|
||||||
} else {
|
} else {
|
||||||
|
@ -105,7 +112,9 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
log.info(DateTime.now().toDateTimeISO().toString());
|
||||||
TimeUnit.SECONDS.sleep(DELAY);
|
TimeUnit.SECONDS.sleep(DELAY);
|
||||||
|
log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString());
|
||||||
}
|
}
|
||||||
if (!metadata.equals("")) {
|
if (!metadata.equals("")) {
|
||||||
zenodoApiClient.sendMretadata(metadata);
|
zenodoApiClient.sendMretadata(metadata);
|
||||||
|
|
|
@ -15,21 +15,14 @@ import org.apache.http.client.methods.HttpPost;
|
||||||
import org.apache.http.client.methods.HttpPut;
|
import org.apache.http.client.methods.HttpPut;
|
||||||
import org.apache.http.client.utils.URIBuilder;
|
import org.apache.http.client.utils.URIBuilder;
|
||||||
import org.apache.http.entity.ContentType;
|
import org.apache.http.entity.ContentType;
|
||||||
import org.apache.http.entity.InputStreamEntity;
|
|
||||||
import org.apache.http.entity.StringEntity;
|
|
||||||
import org.apache.http.impl.client.CloseableHttpClient;
|
|
||||||
// import org.apache.http.impl.client.HttpClients;
|
|
||||||
import org.apache.http.util.EntityUtils;
|
|
||||||
import org.jetbrains.annotations.NotNull;
|
import org.jetbrains.annotations.NotNull;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.SparkCopyGraph;
|
|
||||||
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
|
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
|
||||||
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
|
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
|
||||||
import kotlin.Pair;
|
|
||||||
import okhttp3.*;
|
import okhttp3.*;
|
||||||
|
|
||||||
public class ZenodoAPIClient implements Serializable {
|
public class ZenodoAPIClient implements Serializable {
|
||||||
|
@ -76,34 +69,6 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
* @return response code
|
* @return response code
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
// public int newDeposition() throws IOException {
|
|
||||||
// String json = "{}";
|
|
||||||
//
|
|
||||||
// URL url = new URL(urlString);
|
|
||||||
// HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
|
||||||
// conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
|
|
||||||
// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
|
|
||||||
// conn.setRequestMethod("POST");
|
|
||||||
// conn.setDoOutput(true);
|
|
||||||
// try (OutputStream os = conn.getOutputStream()) {
|
|
||||||
// byte[] input = json.getBytes("utf-8");
|
|
||||||
// os.write(input, 0, input.length);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// String body = getBody(conn);
|
|
||||||
//
|
|
||||||
// int responseCode = conn.getResponseCode();
|
|
||||||
// conn.disconnect();
|
|
||||||
//
|
|
||||||
// if (!checkOKStatus(responseCode))
|
|
||||||
// throw new IOException("Unexpected code " + responseCode + body);
|
|
||||||
//
|
|
||||||
// ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class);
|
|
||||||
// this.bucket = newSubmission.getLinks().getBucket();
|
|
||||||
// this.deposition_id = newSubmission.getId();
|
|
||||||
//
|
|
||||||
// return responseCode;
|
|
||||||
// }
|
|
||||||
|
|
||||||
public int newDeposition() throws IOException {
|
public int newDeposition() throws IOException {
|
||||||
String json = "{}";
|
String json = "{}";
|
||||||
|
@ -137,50 +102,6 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// public int uploadIS2(InputStream is, String fileName) throws IOException {
|
|
||||||
//
|
|
||||||
// final String crlf = "\r\n";
|
|
||||||
// final String twoHyphens = "--";
|
|
||||||
// final String boundary = "*****";
|
|
||||||
//
|
|
||||||
// HttpPut put = new HttpPut(bucket + "/" + fileName);
|
|
||||||
//
|
|
||||||
// put.addHeader(HttpHeaders.CONTENT_TYPE, "application/zip");
|
|
||||||
// put.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
|
|
||||||
//
|
|
||||||
// put.setEntity(new InputStreamEntity(is));
|
|
||||||
//
|
|
||||||
// int statusCode;
|
|
||||||
// try (CloseableHttpClient client = HttpClients.createDefault()) {
|
|
||||||
// CloseableHttpResponse response = client.execute(put);
|
|
||||||
// statusCode = response.getStatusLine().getStatusCode();
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (!checkOKStatus(statusCode)) {
|
|
||||||
// throw new IOException("Unexpected code " + statusCode);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// return statusCode;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public int publish() throws IOException {
|
|
||||||
// String json = "{}";
|
|
||||||
// HttpPost post = new HttpPost(urlString + "/" + deposition_id + "/actions/publish");
|
|
||||||
// post.addHeader(HttpHeaders.CONTENT_TYPE, "application/json");
|
|
||||||
// post.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
|
|
||||||
// post.setEntity(new StringEntity(json));
|
|
||||||
// int statusCode;
|
|
||||||
// try (CloseableHttpClient client = HttpClients.createDefault()) {
|
|
||||||
// CloseableHttpResponse response = client.execute(post);
|
|
||||||
// statusCode = response.getStatusLine().getStatusCode();
|
|
||||||
// }
|
|
||||||
// if (!checkOKStatus(statusCode)) {
|
|
||||||
// throw new IOException("Unexpected code " + statusCode);
|
|
||||||
// }
|
|
||||||
// return statusCode;
|
|
||||||
// }
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Upload files in Zenodo.
|
* Upload files in Zenodo.
|
||||||
*
|
*
|
||||||
|
@ -302,39 +223,40 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @throws MissingConceptDoiException
|
* @throws MissingConceptDoiException
|
||||||
*/
|
*/
|
||||||
// public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException {
|
public int newVersion2(String concept_rec_id) throws Exception, MissingConceptDoiException {
|
||||||
// setDepositionId(concept_rec_id, 1);
|
setDepositionId(concept_rec_id, 1);
|
||||||
// String json = "{}";
|
String json = "{}";
|
||||||
//
|
|
||||||
// URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion");
|
URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion");
|
||||||
// HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||||
//
|
|
||||||
// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
|
conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
|
||||||
// conn.setDoOutput(true);
|
conn.setDoOutput(true);
|
||||||
// conn.setRequestMethod("POST");
|
conn.setRequestMethod("POST");
|
||||||
//
|
|
||||||
// try (OutputStream os = conn.getOutputStream()) {
|
try (OutputStream os = conn.getOutputStream()) {
|
||||||
// byte[] input = json.getBytes("utf-8");
|
byte[] input = json.getBytes("utf-8");
|
||||||
// os.write(input, 0, input.length);
|
os.write(input, 0, input.length);
|
||||||
//
|
|
||||||
// }
|
}
|
||||||
//
|
|
||||||
// String body = getBody(conn);
|
String body = getBody(conn);
|
||||||
//
|
|
||||||
// int responseCode = conn.getResponseCode();
|
int responseCode = conn.getResponseCode();
|
||||||
//
|
|
||||||
// conn.disconnect();
|
conn.disconnect();
|
||||||
// if (!checkOKStatus(responseCode))
|
if (!checkOKStatus(responseCode))
|
||||||
// throw new IOException("Unexpected code " + responseCode + body);
|
throw new IOException("Unexpected code " + responseCode + body);
|
||||||
//
|
|
||||||
// ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
|
ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
|
||||||
// String latest_draft = zenodoModel.getLinks().getLatest_draft();
|
String latest_draft = zenodoModel.getLinks().getLatest_draft();
|
||||||
// deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
|
deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
|
||||||
// bucket = getBucket(latest_draft);
|
bucket = getBucket(latest_draft);
|
||||||
//
|
|
||||||
// return responseCode;
|
return responseCode;
|
||||||
//
|
|
||||||
// }
|
}
|
||||||
|
|
||||||
public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException {
|
public int newVersion(String concept_rec_id) throws Exception, MissingConceptDoiException {
|
||||||
setDepositionId(concept_rec_id, 1);
|
setDepositionId(concept_rec_id, 1);
|
||||||
String json = "{}";
|
String json = "{}";
|
||||||
|
@ -352,7 +274,7 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
if (!response.isSuccessful()) {
|
if (!response.isSuccessful()) {
|
||||||
//log.info("response headers " + response.headers().toString());
|
// log.info("response headers " + response.headers().toString());
|
||||||
throw new IOException("Unexpected code " + response + response.body().string());
|
throw new IOException("Unexpected code " + response + response.body().string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -424,22 +346,6 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// private String getPrevDepositions(String page) throws Exception {
|
|
||||||
//
|
|
||||||
// HttpGet get = new HttpGet(urlString);
|
|
||||||
// URI uri = new URIBuilder(get.getURI()).addParameter("page", page).build();
|
|
||||||
//
|
|
||||||
// get.setURI(uri);
|
|
||||||
//
|
|
||||||
// get.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
|
|
||||||
// get.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
|
|
||||||
// try (CloseableHttpClient client = HttpClients.createDefault()) {
|
|
||||||
// CloseableHttpResponse response = client.execute(get);
|
|
||||||
// final String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
|
||||||
// return body;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
private String getPrevDepositions(String page) throws IOException {
|
private String getPrevDepositions(String page) throws IOException {
|
||||||
|
|
||||||
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
@ -447,13 +353,13 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
HttpUrl.Builder urlBuilder = HttpUrl
|
HttpUrl.Builder urlBuilder = HttpUrl
|
||||||
.parse(urlString)// + "?access_token=" + access_token + "&page=" + page)
|
.parse(urlString)// + "?access_token=" + access_token + "&page=" + page)
|
||||||
.newBuilder();
|
.newBuilder();
|
||||||
urlBuilder.addQueryParameter("page", page);
|
urlBuilder.addQueryParameter("page", page);
|
||||||
String url = urlBuilder.build().toString();
|
String url = urlBuilder.build().toString();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(url)
|
.url(url)
|
||||||
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.get()
|
.get()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -463,7 +369,7 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
if (!response.isSuccessful()) {
|
if (!response.isSuccessful()) {
|
||||||
log.info("response headers: " + response.headers().toString());
|
log.info("response headers: " + response.headers());
|
||||||
throw new IOException("Unexpected code " + response + response.body().string());
|
throw new IOException("Unexpected code " + response + response.body().string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue