D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

[dumpSubset] wf for the creation of the dump per country start from the beginning

This commit is contained in:
Miriam Baglioni 2023-07-28 11:31:42 +02:00
parent 25be584028
commit a06640d344
3 changed files with 24 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java
View File

@ -18,7 +18,7 @@ import org.apache.http.entity.ContentType;
import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity; import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
//import org.apache.http.impl.client.HttpClients; // import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
@ -108,11 +108,11 @@ public class ZenodoAPIClient implements Serializable {
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
Request request = new Request.Builder() Request request = new Request.Builder()
.url(urlString) .url(urlString)
.addHeader("Content-Type", "application/json") // add request headers .addHeader("Content-Type", "application/json") // add request headers
.addHeader("Authorization", "Bearer " + access_token) .addHeader("Authorization", "Bearer " + access_token)
.post(body) .post(body)
.build(); .build();
try (Response response = httpClient.newCall(request).execute()) { try (Response response = httpClient.newCall(request).execute()) {
@ -122,7 +122,8 @@ public class ZenodoAPIClient implements Serializable {
// Get response body // Get response body
json = response.body().string(); json = response.body().string();
eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson().fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson()
.fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class);
this.bucket = newSubmission.getLinks().getBucket(); this.bucket = newSubmission.getLinks().getBucket();
this.deposition_id = newSubmission.getId(); this.deposition_id = newSubmission.getId();
@ -132,7 +133,6 @@ public class ZenodoAPIClient implements Serializable {
} }
// public int uploadIS2(InputStream is, String fileName) throws IOException { // public int uploadIS2(InputStream is, String fileName) throws IOException {
// //
// final String crlf = "\r\n"; // final String crlf = "\r\n";
@ -340,10 +340,10 @@ public class ZenodoAPIClient implements Serializable {
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
Request request = new Request.Builder() Request request = new Request.Builder()
.url(urlString + "/" + deposition_id + "/actions/newversion") .url(urlString + "/" + deposition_id + "/actions/newversion")
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
.post(body) .post(body)
.build(); .build();
try (Response response = httpClient.newCall(request).execute()) { try (Response response = httpClient.newCall(request).execute()) {
@ -445,11 +445,11 @@ public class ZenodoAPIClient implements Serializable {
String url = urlBuilder.build().toString(); String url = urlBuilder.build().toString();
Request request = new Request.Builder() Request request = new Request.Builder()
.url(url) .url(url)
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
.get() .get()
.build(); .build();
try (Response response = httpClient.newCall(request).execute()) { try (Response response = httpClient.newCall(request).execute()) {

2
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/countryresults/oozie_app/workflow.xml
View File

@ -81,7 +81,7 @@
</global> </global>
<!-- <start to="save_community_map" />--> <!-- <start to="save_community_map" />-->
<start to="make_archive" /> <start to="save_community_map" />
<action name="save_community_map"> <action name="save_community_map">
<java> <java>

13
dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java
View File

@ -7,8 +7,6 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.*;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
@ -19,7 +17,8 @@ import org.junit.jupiter.api.Test;
import com.google.gson.Gson; import com.google.gson.Gson;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException;
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
@Disabled @Disabled
public class ZenodoUploadTest { public class ZenodoUploadTest {
@ -86,8 +85,7 @@ public class ZenodoUploadTest {
System.out.println(client.sendMretadata(metadata)); System.out.println(client.sendMretadata(metadata));
// System.out.println(client.publish());
//System.out.println(client.publish());
} }
@ -138,7 +136,7 @@ public class ZenodoUploadTest {
} }
//System.out.println(client.publish()); // System.out.println(client.publish());
} }
@ -146,7 +144,7 @@ public class ZenodoUploadTest {
void testNewVersion2() throws Exception, MissingConceptDoiException { void testNewVersion2() throws Exception, MissingConceptDoiException {
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
ACCESS_TOKEN); ACCESS_TOKEN);
client.newVersion("1210237"); client.newVersion("1210237");
@ -162,6 +160,7 @@ public class ZenodoUploadTest {
// Assertions.assertEquals(202, client.publish()); // Assertions.assertEquals(202, client.publish());
} }
@Test @Test
void readCommunityMap() throws IOException { void readCommunityMap() throws IOException {
LocalFileSystem fs = FileSystem.getLocal(new Configuration()); LocalFileSystem fs = FileSystem.getLocal(new Configuration());