diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 72801ee..502bb5c 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -97,8 +97,15 @@ public class SendToZenodoHDFS implements Serializable { log.info("Upoloading: {}", name); FSDataInputStream inputStream = fileSystem.open(p); while (retry && numberOfRetries < NUMBER_OF_RETRIES) { - int response_code = zenodoApiClient - .uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); + int response_code = -1; + + try { + response_code = zenodoApiClient + .uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); + }catch (Exception e){ + log.info(e.getMessage()); + throw new RuntimeException("Error while uploading on Zenodo"); + } log.info("response code: {}", response_code); if (HttpStatus.SC_OK == response_code || HttpStatus.SC_CREATED == response_code) { retry = false; @@ -112,9 +119,9 @@ public class SendToZenodoHDFS implements Serializable { } } - log.info(DateTime.now().toDateTimeISO().toString()); + // log.info(DateTime.now().toDateTimeISO().toString()); TimeUnit.SECONDS.sleep(DELAY); - log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString()); + // log.info("Delayed: {}", DateTime.now().toDateTimeISO().toString()); } if (!metadata.equals("")) { zenodoApiClient.sendMretadata(metadata); diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java index edd3ee8..7e9231e 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java @@ -183,19 +183,29 @@ public class ZenodoAPIClient implements Serializable { } public int uploadIS3(InputStream is, String file_name, long len) throws IOException { +// String urlString = "http://checkip.amazonaws.com/"; +// URL url = new URL(urlString); +// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) { +// log.info(br.readLine()); +// } OkHttpClient httpClient = new OkHttpClient.Builder() .writeTimeout(600, TimeUnit.SECONDS) .readTimeout(600, TimeUnit.SECONDS) .connectTimeout(600, TimeUnit.SECONDS) .build(); + System.out.println(bucket + "/" + file_name); + Request request = new Request.Builder() .url(bucket + "/" + file_name) - // .addHeader(HttpHeaders.CONTENT_TYPE, "application/json") // add request headers + .addHeader(HttpHeaders.CONTENT_TYPE, "application/json") // add request headers .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) .put(InputStreamRequestBody.create(MEDIA_TYPE_TAR, is, len)) .build(); + log.info("URL: " + request.url().toString()); + log.info("Headers: " + request.headers().toString()); + try (Response response = httpClient.newCall(request).execute()) { if (!response.isSuccessful()) System.out.println("Unexpected code " + response + response.body().string()); @@ -329,7 +339,11 @@ public class ZenodoAPIClient implements Serializable { * @throws MissingConceptDoiException */ public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException { - +// String urlamazon = "http://checkip.amazonaws.com/"; +// URL url = new URL(urlamazon); +// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) { +// log.info(br.readLine()); +// } this.deposition_id = deposition_id; OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); @@ -339,6 +353,9 @@ public class ZenodoAPIClient implements Serializable { .addHeader("Authorization", "Bearer " + access_token) .build(); + log.info("URL: " + request.url().toString()); + log.info("Headers: " + request.headers().toString()); + try (Response response = httpClient.newCall(request).execute()) { if (!response.isSuccessful()) @@ -354,12 +371,12 @@ public class ZenodoAPIClient implements Serializable { } private void setDepositionId(String concept_rec_id, Integer page) throws Exception, MissingConceptDoiException { - String urlString = "http://checkip.amazonaws.com/"; - URL url = new URL(urlString); - try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) { - log.info(br.readLine()); - } -//log.info(); +// String urlString = "http://checkip.amazonaws.com/"; +// URL url = new URL(urlString); +// try (BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()))) { +// log.info(br.readLine()); +// } + ZenodoModelList zenodoModelList = new Gson() .fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class); diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java index 410a452..2ea16ea 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java @@ -95,46 +95,46 @@ public class ZenodoUploadTest { ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, ACCESS_TOKEN); - client.newVersion("656628"); + System.out.println(client.newVersion("3516917")); - CommunityMap communityMap = new CommunityMap(); - communityMap.put("ni", "Neuroinformatics"); - communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage"); - LocalFileSystem fs = FileSystem.getLocal(new Configuration()); +// CommunityMap communityMap = new CommunityMap(); +// communityMap.put("ni", "Neuroinformatics"); +// communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage"); +// LocalFileSystem fs = FileSystem.getLocal(new Configuration()); +// +// fs +// .copyFromLocalFile( +// false, new Path(getClass() +// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") +// .getPath()), +// new Path(workingDir + "/zenodo/ni/ni")); +// fs +// .copyFromLocalFile( +// false, new Path(getClass() +// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch") +// .getPath()), +// new Path(workingDir + "/zenodo/dh-ch/dh-ch")); +// +// RemoteIterator fileStatusListIterator = fs +// .listFiles( +// new Path(workingDir + "/zenodo"), true); +// while (fileStatusListIterator.hasNext()) { +// LocatedFileStatus fileStatus = fileStatusListIterator.next(); +// +// String p_string = fileStatus.getPath().toString(); +// +// int index = p_string.lastIndexOf("/"); +// String community = p_string.substring(0, index); +// community = community.substring(community.lastIndexOf("/") + 1); +// String community_name = communityMap.get(community).replace(" ", "_"); +// // fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name)); +// System.out.println(community); +// +// // File f = new File("/tmp/" + community_name); +// FSDataInputStream inputStream = fs.open(fileStatus.getPath()); +// System.out.println(client.uploadIS(inputStream, community_name)); - fs - .copyFromLocalFile( - false, new Path(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") - .getPath()), - new Path(workingDir + "/zenodo/ni/ni")); - fs - .copyFromLocalFile( - false, new Path(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch") - .getPath()), - new Path(workingDir + "/zenodo/dh-ch/dh-ch")); - - RemoteIterator fileStatusListIterator = fs - .listFiles( - new Path(workingDir + "/zenodo"), true); - while (fileStatusListIterator.hasNext()) { - LocatedFileStatus fileStatus = fileStatusListIterator.next(); - - String p_string = fileStatus.getPath().toString(); - - int index = p_string.lastIndexOf("/"); - String community = p_string.substring(0, index); - community = community.substring(community.lastIndexOf("/") + 1); - String community_name = communityMap.get(community).replace(" ", "_"); - // fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name)); - System.out.println(community); - - // File f = new File("/tmp/" + community_name); - FSDataInputStream inputStream = fs.open(fileStatus.getPath()); - System.out.println(client.uploadIS(inputStream, community_name)); - - } +// } // System.out.println(client.publish()); @@ -200,7 +200,7 @@ public class ZenodoUploadTest { ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, ACCESS_TOKEN); - client.uploadOpenDeposition("8144316"); + client.uploadOpenDeposition("10037121"); File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar"); // File file = new File(getClass() @@ -209,7 +209,9 @@ public class ZenodoUploadTest { InputStream is = new FileInputStream(file); - Assertions.assertEquals(200, client.uploadIS3(is, "newVersion_deposition", file.length())); + Integer response_code = client.uploadIS3(is, "newVersion_deposition", file.length()); + System.out.println(response_code); + Assertions.assertEquals(201, response_code); // Assertions.assertEquals(202, client.publish());