From a175ac2c7fecd96c79fccc715cdbd358aa36134f Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 19 Jul 2023 09:40:48 +0200 Subject: [PATCH] [dump] refactoring --- .../eu/dnetlib/dhp/oa/graph/dump/Utils.java | 2 +- .../dhp/oa/graph/dump/csv/Constants.java | 8 ++--- .../oa/graph/dump/csv/SparkDumpResults.java | 11 +++--- .../oa/graph/dump/csv/model/CSVAuthor.java | 4 +-- .../oa/graph/dump/csv/model/CSVCitation.java | 4 +-- .../dhp/oa/graph/dump/csv/model/CSVPid.java | 4 +-- .../dump/csv/model/CSVRELCommunityResult.java | 4 +-- .../oa/graph/dump/csv/model/CSVRelResAut.java | 4 +-- .../oa/graph/dump/csv/model/CSVResult.java | 9 ++--- .../dhp/oa/zenodoapi/ZenodoAPIClient.java | 34 +++++++++---------- .../dhp/oa/graph/dump/ZenodoUploadTest.java | 13 ++++--- .../dhp/oa/graph/dump/csv/DumpResultTest.java | 8 +++++ 12 files changed, 56 insertions(+), 49 deletions(-) diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java index 7328ce8..616f1b2 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java @@ -53,7 +53,7 @@ public class Utils { return String .format( - "%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, + "%s::%s", Constants.CONTEXT_NS_PREFIX, DHPUtils.md5(id)); } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/Constants.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/Constants.java index 24fff80..bca06fd 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/Constants.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/Constants.java @@ -1,10 +1,10 @@ package eu.dnetlib.dhp.oa.graph.dump.csv; -import org.apache.commons.lang.StringUtils; - import java.io.Serializable; +import org.apache.commons.lang.StringUtils; + /** * @author miriam.baglioni * @Date 10/05/23 @@ -13,8 +13,8 @@ public class Constants implements Serializable { public final static String SEP = "\t"; public static final String addQuotes(String id) { - if(StringUtils.isNotEmpty(id)) - return "\"" + id + "\""; +// if (StringUtils.isNotEmpty(id)) +// return "\"" + id + "\""; return id; } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/SparkDumpResults.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/SparkDumpResults.java index 70ed912..f74ad8d 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/SparkDumpResults.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/SparkDumpResults.java @@ -347,12 +347,15 @@ public class SparkDumpResults implements Serializable { } private static String removeBreaks(String input) { - return input.replace("\n", " ") + if (StringUtils.isNotEmpty(input)) + return input + .replace("\n", " ") .replace("\t", " ") .replace("\r", " ") - .replace("\\\"", " ") - .replace("\"", " ") - ; + //.replace("\\", " ") + .replace("\"", " "); + + return input; } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVAuthor.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVAuthor.java index c3057e9..c27190a 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVAuthor.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVAuthor.java @@ -1,10 +1,10 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model; -import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; - import java.io.Serializable; +import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; + /** * @author miriam.baglioni * @Date 11/05/23 diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVCitation.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVCitation.java index 9b0fe14..e027b6d 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVCitation.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVCitation.java @@ -1,10 +1,10 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model; -import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; - import java.io.Serializable; +import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; + /** * @author miriam.baglioni * @Date 11/05/23 diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVPid.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVPid.java index c61ae0c..a255fe1 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVPid.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVPid.java @@ -1,10 +1,10 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model; -import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; - import java.io.Serializable; +import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; + /** * @author miriam.baglioni * @Date 11/05/23 diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRELCommunityResult.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRELCommunityResult.java index bbc0a9a..c35a9ee 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRELCommunityResult.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRELCommunityResult.java @@ -1,10 +1,10 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model; -import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; - import java.io.Serializable; +import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; + /** * @author miriam.baglioni * @Date 11/05/23 diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRelResAut.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRelResAut.java index 1b334c8..47ec628 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRelResAut.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVRelResAut.java @@ -1,10 +1,10 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model; -import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; - import java.io.Serializable; +import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; + /** * @author miriam.baglioni * @Date 11/05/23 diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVResult.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVResult.java index da8a78c..27dd7dd 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVResult.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/csv/model/CSVResult.java @@ -3,13 +3,13 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model; import java.io.Serializable; -import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; import org.apache.commons.lang.StringUtils; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSetter; +import eu.dnetlib.dhp.oa.graph.dump.csv.Constants; import eu.dnetlib.dhp.schema.oaf.Country; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import sun.swing.StringUIClientPropertyKey; @@ -38,7 +38,6 @@ public class CSVResult implements Serializable { this.id = Constants.addQuotes(id); } - public String getType() { return type; } @@ -52,8 +51,7 @@ public class CSVResult implements Serializable { } public void setTitle(String title) { - this.title = - Constants.addQuotes(title); + this.title = Constants.addQuotes(title); } public String getDescription() { @@ -61,8 +59,7 @@ public class CSVResult implements Serializable { } public void setDescription(String description) { - this.description = - Constants.addQuotes(description); + this.description = Constants.addQuotes(description); } public String getAccessright() { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java index 1b3bb7a..e5d2353 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java @@ -18,7 +18,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -//import org.apache.http.impl.client.HttpClients; +// import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.NotNull; @@ -108,11 +108,11 @@ public class ZenodoAPIClient implements Serializable { RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); Request request = new Request.Builder() - .url(urlString) - .addHeader("Content-Type", "application/json") // add request headers - .addHeader("Authorization", "Bearer " + access_token) - .post(body) - .build(); + .url(urlString) + .addHeader("Content-Type", "application/json") // add request headers + .addHeader("Authorization", "Bearer " + access_token) + .post(body) + .build(); try (Response response = httpClient.newCall(request).execute()) { @@ -122,7 +122,8 @@ public class ZenodoAPIClient implements Serializable { // Get response body json = response.body().string(); - eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson().fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); + eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson() + .fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class); this.bucket = newSubmission.getLinks().getBucket(); this.deposition_id = newSubmission.getId(); @@ -132,7 +133,6 @@ public class ZenodoAPIClient implements Serializable { } - // public int uploadIS2(InputStream is, String fileName) throws IOException { // // final String crlf = "\r\n"; @@ -340,10 +340,10 @@ public class ZenodoAPIClient implements Serializable { RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); Request request = new Request.Builder() - .url(urlString + "/" + deposition_id + "/actions/newversion") - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) - .post(body) - .build(); + .url(urlString + "/" + deposition_id + "/actions/newversion") + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .post(body) + .build(); try (Response response = httpClient.newCall(request).execute()) { @@ -445,11 +445,11 @@ public class ZenodoAPIClient implements Serializable { String url = urlBuilder.build().toString(); Request request = new Request.Builder() - .url(url) - .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) - .get() - .build(); + .url(url) + .addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .get() + .build(); try (Response response = httpClient.newCall(request).execute()) { diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java index f119274..7117d00 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java @@ -7,8 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; -import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient; -import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.junit.jupiter.api.Assertions; @@ -19,7 +17,8 @@ import org.junit.jupiter.api.Test; import com.google.gson.Gson; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; - +import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException; +import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient; @Disabled public class ZenodoUploadTest { @@ -86,8 +85,7 @@ public class ZenodoUploadTest { System.out.println(client.sendMretadata(metadata)); - - //System.out.println(client.publish()); + // System.out.println(client.publish()); } @@ -138,7 +136,7 @@ public class ZenodoUploadTest { } - //System.out.println(client.publish()); + // System.out.println(client.publish()); } @@ -146,7 +144,7 @@ public class ZenodoUploadTest { void testNewVersion2() throws Exception, MissingConceptDoiException { ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); + ACCESS_TOKEN); client.newVersion("1210237"); @@ -162,6 +160,7 @@ public class ZenodoUploadTest { // Assertions.assertEquals(202, client.publish()); } + @Test void readCommunityMap() throws IOException { LocalFileSystem fs = FileSystem.getLocal(new Configuration()); diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/csv/DumpResultTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/csv/DumpResultTest.java index e285098..f233a5c 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/csv/DumpResultTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/csv/DumpResultTest.java @@ -347,4 +347,12 @@ public class DumpResultTest { builder.append(((Node) (root.selectNodes("//description").get(0))).getText()); System.out.println(builder.toString()); } + + @Test + public void provaReplacement(){ + String st = "\"Person in the sand " + "\""; + System.out.println(st); + + System.out.println(st.replace("\\\"", " ")); + } }