D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

-

This commit is contained in:
Miriam Baglioni 2023-10-24 16:02:21 +02:00
parent 176d6d7f2b
commit 8cf6a40bdf
3 changed files with 78 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java
View File

@ -97,8 +97,15 @@ public class SendToZenodoHDFS implements Serializable {
log.info("Upoloading: {}", name);
FSDataInputStream inputStream = fileSystem.open(p);
while (retry && numberOfRetries < NUMBER_OF_RETRIES) {
int response_code = zenodoApiClient
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
int response_code = -1;
try {
response_code = zenodoApiClient
.uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen());
}catch (Exception e){
log.info(e.getMessage());
throw new RuntimeException("Error while uploading on Zenodo");
}
log.info("response code: {}", response_code);
if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) {
retry = false;
@ -112,9 +119,9 @@ public class SendToZenodoHDFS implements Serializable {
}
}
log.info(DateTime.now().toDateTimeISO().toString());
// log.info(DateTime.now().toDateTimeISO().toString());
TimeUnit.SECONDS.sleep(DELAY);
log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString());
// log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString());
}
if (!metadata.equals("")) {
zenodoApiClient.sendMretadata(metadata);

33
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java
View File

@ -183,19 +183,29 @@ public class ZenodoAPIClient implements Serializable {
}
public int uploadIS3(InputStream is, String file_name, long len) throws IOException {
// String urlString = "http://checkip.amazonaws.com/";
// URL url = new URL(urlString);
// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
// log.info(br.readLine());
// }
OkHttpClient httpClient = new OkHttpClient.Builder()
.writeTimeout(600, TimeUnit.SECONDS)
.readTimeout(600, TimeUnit.SECONDS)
.connectTimeout(600, TimeUnit.SECONDS)
.build();
System.out.println(bucket + "/" + file_name);
Request request = new Request.Builder()
.url(bucket + "/" + file_name)
// .addHeader(HttpHeaders.CONTENT_TYPE, "application/json") // add request headers
.addHeader(HttpHeaders.CONTENT_TYPE, "application/json") // add request headers
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
.put(InputStreamRequestBody.create(MEDIA_TYPE_TAR, is, len))
.build();
log.info("URL: " + request.url().toString());
log.info("Headers: " + request.headers().toString());
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful())
System.out.println("Unexpected code " + response + response.body().string());
@ -329,7 +339,11 @@ public class ZenodoAPIClient implements Serializable {
* @throws MissingConceptDoiException
*/
public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
// String urlamazon = "http://checkip.amazonaws.com/";
// URL url = new URL(urlamazon);
// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
// log.info(br.readLine());
// }
this.deposition_id = deposition_id;
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
@ -339,6 +353,9 @@ public class ZenodoAPIClient implements Serializable {
.addHeader("Authorization", "Bearer " + access_token)
.build();
log.info("URL: " + request.url().toString());
log.info("Headers: " + request.headers().toString());
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful())
@ -354,12 +371,12 @@ public class ZenodoAPIClient implements Serializable {
}
private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException {
String urlString = "http://checkip.amazonaws.com/";
URL url = new URL(urlString);
try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
log.info(br.readLine());
}
//log.info();
// String urlString = "http://checkip.amazonaws.com/";
// URL url = new URL(urlString);
// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) {
// log.info(br.readLine());
// }
ZenodoModelList zenodoModelList = new Gson()
.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);

82
dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java
View File

@ -95,46 +95,46 @@ public class ZenodoUploadTest {
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
ACCESS_TOKEN);
client.newVersion("656628");
System.out.println(client.newVersion("3516917"));
CommunityMap communityMap = new CommunityMap();
communityMap.put("ni", "Neuroinformatics");
communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage");
LocalFileSystem fs = FileSystem.getLocal(new Configuration());
// CommunityMap communityMap = new CommunityMap();
// communityMap.put("ni", "Neuroinformatics");
// communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage");
// LocalFileSystem fs = FileSystem.getLocal(new Configuration());
//
// fs
// .copyFromLocalFile(
// false, new Path(getClass()
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
// .getPath()),
// new Path(workingDir + "/zenodo/ni/ni"));
// fs
// .copyFromLocalFile(
// false, new Path(getClass()
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
// .getPath()),
// new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
//
// RemoteIterator<LocatedFileStatus> fileStatusListIterator = fs
// .listFiles(
// new Path(workingDir + "/zenodo"), true);
// while (fileStatusListIterator.hasNext()) {
// LocatedFileStatus fileStatus = fileStatusListIterator.next();
//
// String p_string = fileStatus.getPath().toString();
//
// int index = p_string.lastIndexOf("/");
// String community = p_string.substring(0, index);
// community = community.substring(community.lastIndexOf("/") + 1);
// String community_name = communityMap.get(community).replace(" ", "_");
// // fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name));
// System.out.println(community);
//
// // File f = new File("/tmp/" + community_name);
// FSDataInputStream inputStream = fs.open(fileStatus.getPath());
// System.out.println(client.uploadIS(inputStream, community_name));
fs
.copyFromLocalFile(
false, new Path(getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
.getPath()),
new Path(workingDir + "/zenodo/ni/ni"));
fs
.copyFromLocalFile(
false, new Path(getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
.getPath()),
new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fs
.listFiles(
new Path(workingDir + "/zenodo"), true);
while (fileStatusListIterator.hasNext()) {
LocatedFileStatus fileStatus = fileStatusListIterator.next();
String p_string = fileStatus.getPath().toString();
int index = p_string.lastIndexOf("/");
String community = p_string.substring(0, index);
community = community.substring(community.lastIndexOf("/") + 1);
String community_name = communityMap.get(community).replace(" ", "_");
// fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name));
System.out.println(community);
// File f = new File("/tmp/" + community_name);
FSDataInputStream inputStream = fs.open(fileStatus.getPath());
System.out.println(client.uploadIS(inputStream, community_name));
}
// }
// System.out.println(client.publish());
@ -200,7 +200,7 @@ public class ZenodoUploadTest {
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
ACCESS_TOKEN);
client.uploadOpenDeposition("8144316");
client.uploadOpenDeposition("10037121");
File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar");
// File file = new File(getClass()
@ -209,7 +209,9 @@ public class ZenodoUploadTest {
InputStream is = new FileInputStream(file);
Assertions.assertEquals(200, client.uploadIS3(is, "newVersion_deposition", file.length()));
Integer response_code = client.uploadIS3(is, "newVersion_deposition", file.length());
System.out.println(response_code);
Assertions.assertEquals(201, response_code);
// Assertions.assertEquals(202, client.publish());