This commit is contained in:
parent
176d6d7f2b
commit
8cf6a40bdf
|
@ -97,8 +97,15 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
log.info("Upoloading: {}", name);
|
log.info("Upoloading: {}", name);
|
||||||
FSDataInputStream inputStream = fileSystem.open(p);
|
FSDataInputStream inputStream = fileSystem.open(p);
|
||||||
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
|
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
|
||||||
int response_code = zenodoApiClient
|
int response_code = -1;
|
||||||
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
|
|
||||||
|
try {
|
||||||
|
response_code = zenodoApiClient
|
||||||
|
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
|
||||||
|
}catch (Exception e){
|
||||||
|
log.info(e.getMessage());
|
||||||
|
throw new RuntimeException("Error while uploading on Zenodo");
|
||||||
|
}
|
||||||
log.info("response code: {}", response_code);
|
log.info("response code: {}", response_code);
|
||||||
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
|
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
|
||||||
retry = false;
|
retry = false;
|
||||||
|
@ -112,9 +119,9 @@ public class SendToZenodoHDFS implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
log.info(DateTime.now().toDateTimeISO().toString());
|
// log.info(DateTime.now().toDateTimeISO().toString());
|
||||||
TimeUnit.SECONDS.sleep(DELAY);
|
TimeUnit.SECONDS.sleep(DELAY);
|
||||||
log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString());
|
// log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString());
|
||||||
}
|
}
|
||||||
if (!metadata.equals("")) {
|
if (!metadata.equals("")) {
|
||||||
zenodoApiClient.sendMretadata(metadata);
|
zenodoApiClient.sendMretadata(metadata);
|
||||||
|
|
|
@ -183,19 +183,29 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
public int uploadIS3(InputStream is, String file_name, long len) throws IOException {
|
public int uploadIS3(InputStream is, String file_name, long len) throws IOException {
|
||||||
|
// String urlString = "http://checkip.amazonaws.com/";
|
||||||
|
// URL url = new URL(urlString);
|
||||||
|
// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
|
||||||
|
// log.info(br.readLine());
|
||||||
|
// }
|
||||||
OkHttpClient httpClient = new OkHttpClient.Builder()
|
OkHttpClient httpClient = new OkHttpClient.Builder()
|
||||||
.writeTimeout(600, TimeUnit.SECONDS)
|
.writeTimeout(600, TimeUnit.SECONDS)
|
||||||
.readTimeout(600, TimeUnit.SECONDS)
|
.readTimeout(600, TimeUnit.SECONDS)
|
||||||
.connectTimeout(600, TimeUnit.SECONDS)
|
.connectTimeout(600, TimeUnit.SECONDS)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
System.out.println(bucket + "/" + file_name);
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(bucket + "/" + file_name)
|
.url(bucket + "/" + file_name)
|
||||||
// .addHeader(HttpHeaders.CONTENT_TYPE, "application/json") // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, "application/json") // add request headers
|
||||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.put(InputStreamRequestBody.create(MEDIA_TYPE_TAR, is, len))
|
.put(InputStreamRequestBody.create(MEDIA_TYPE_TAR, is, len))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
log.info("URL: " + request.url().toString());
|
||||||
|
log.info("Headers: " + request.headers().toString());
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
if (!response.isSuccessful())
|
if (!response.isSuccessful())
|
||||||
System.out.println("Unexpected code " + response + response.body().string());
|
System.out.println("Unexpected code " + response + response.body().string());
|
||||||
|
@ -329,7 +339,11 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
* @throws MissingConceptDoiException
|
* @throws MissingConceptDoiException
|
||||||
*/
|
*/
|
||||||
public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
|
public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
|
||||||
|
// String urlamazon = "http://checkip.amazonaws.com/";
|
||||||
|
// URL url = new URL(urlamazon);
|
||||||
|
// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
|
||||||
|
// log.info(br.readLine());
|
||||||
|
// }
|
||||||
this.deposition_id = deposition_id;
|
this.deposition_id = deposition_id;
|
||||||
|
|
||||||
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
@ -339,6 +353,9 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader("Authorization", "Bearer " + access_token)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
log.info("URL: " + request.url().toString());
|
||||||
|
log.info("Headers: " + request.headers().toString());
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
if (!response.isSuccessful())
|
if (!response.isSuccessful())
|
||||||
|
@ -354,12 +371,12 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException {
|
private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException {
|
||||||
String urlString = "http://checkip.amazonaws.com/";
|
// String urlString = "http://checkip.amazonaws.com/";
|
||||||
URL url = new URL(urlString);
|
// URL url = new URL(urlString);
|
||||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
|
// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
|
||||||
log.info(br.readLine());
|
// log.info(br.readLine());
|
||||||
}
|
// }
|
||||||
//log.info();
|
|
||||||
ZenodoModelList zenodoModelList = new Gson()
|
ZenodoModelList zenodoModelList = new Gson()
|
||||||
.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
|
.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
|
||||||
|
|
||||||
|
|
|
@ -95,46 +95,46 @@ public class ZenodoUploadTest {
|
||||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
ACCESS_TOKEN);
|
ACCESS_TOKEN);
|
||||||
|
|
||||||
client.newVersion("656628");
|
System.out.println(client.newVersion("3516917"));
|
||||||
|
|
||||||
CommunityMap communityMap = new CommunityMap();
|
// CommunityMap communityMap = new CommunityMap();
|
||||||
communityMap.put("ni", "Neuroinformatics");
|
// communityMap.put("ni", "Neuroinformatics");
|
||||||
communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage");
|
// communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage");
|
||||||
LocalFileSystem fs = FileSystem.getLocal(new Configuration());
|
// LocalFileSystem fs = FileSystem.getLocal(new Configuration());
|
||||||
|
//
|
||||||
|
// fs
|
||||||
|
// .copyFromLocalFile(
|
||||||
|
// false, new Path(getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
|
||||||
|
// .getPath()),
|
||||||
|
// new Path(workingDir + "/zenodo/ni/ni"));
|
||||||
|
// fs
|
||||||
|
// .copyFromLocalFile(
|
||||||
|
// false, new Path(getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
|
||||||
|
// .getPath()),
|
||||||
|
// new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
|
||||||
|
//
|
||||||
|
// RemoteIterator<LocatedFileStatus> fileStatusListIterator = fs
|
||||||
|
// .listFiles(
|
||||||
|
// new Path(workingDir + "/zenodo"), true);
|
||||||
|
// while (fileStatusListIterator.hasNext()) {
|
||||||
|
// LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
||||||
|
//
|
||||||
|
// String p_string = fileStatus.getPath().toString();
|
||||||
|
//
|
||||||
|
// int index = p_string.lastIndexOf("/");
|
||||||
|
// String community = p_string.substring(0, index);
|
||||||
|
// community = community.substring(community.lastIndexOf("/") + 1);
|
||||||
|
// String community_name = communityMap.get(community).replace(" ", "_");
|
||||||
|
// // fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name));
|
||||||
|
// System.out.println(community);
|
||||||
|
//
|
||||||
|
// // File f = new File("/tmp/" + community_name);
|
||||||
|
// FSDataInputStream inputStream = fs.open(fileStatus.getPath());
|
||||||
|
// System.out.println(client.uploadIS(inputStream, community_name));
|
||||||
|
|
||||||
fs
|
// }
|
||||||
.copyFromLocalFile(
|
|
||||||
false, new Path(getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
|
|
||||||
.getPath()),
|
|
||||||
new Path(workingDir + "/zenodo/ni/ni"));
|
|
||||||
fs
|
|
||||||
.copyFromLocalFile(
|
|
||||||
false, new Path(getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
|
|
||||||
.getPath()),
|
|
||||||
new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
|
|
||||||
|
|
||||||
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fs
|
|
||||||
.listFiles(
|
|
||||||
new Path(workingDir + "/zenodo"), true);
|
|
||||||
while (fileStatusListIterator.hasNext()) {
|
|
||||||
LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
|
||||||
|
|
||||||
String p_string = fileStatus.getPath().toString();
|
|
||||||
|
|
||||||
int index = p_string.lastIndexOf("/");
|
|
||||||
String community = p_string.substring(0, index);
|
|
||||||
community = community.substring(community.lastIndexOf("/") + 1);
|
|
||||||
String community_name = communityMap.get(community).replace(" ", "_");
|
|
||||||
// fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name));
|
|
||||||
System.out.println(community);
|
|
||||||
|
|
||||||
// File f = new File("/tmp/" + community_name);
|
|
||||||
FSDataInputStream inputStream = fs.open(fileStatus.getPath());
|
|
||||||
System.out.println(client.uploadIS(inputStream, community_name));
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// System.out.println(client.publish());
|
// System.out.println(client.publish());
|
||||||
|
|
||||||
|
@ -200,7 +200,7 @@ public class ZenodoUploadTest {
|
||||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
ACCESS_TOKEN);
|
ACCESS_TOKEN);
|
||||||
|
|
||||||
client.uploadOpenDeposition("8144316");
|
client.uploadOpenDeposition("10037121");
|
||||||
|
|
||||||
File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar");
|
File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar");
|
||||||
// File file = new File(getClass()
|
// File file = new File(getClass()
|
||||||
|
@ -209,7 +209,9 @@ public class ZenodoUploadTest {
|
||||||
|
|
||||||
InputStream is = new FileInputStream(file);
|
InputStream is = new FileInputStream(file);
|
||||||
|
|
||||||
Assertions.assertEquals(200, client.uploadIS3(is, "newVersion_deposition", file.length()));
|
Integer response_code = client.uploadIS3(is, "newVersion_deposition", file.length());
|
||||||
|
System.out.println(response_code);
|
||||||
|
Assertions.assertEquals(201, response_code);
|
||||||
|
|
||||||
// Assertions.assertEquals(202, client.publish());
|
// Assertions.assertEquals(202, client.publish());
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue