From be5ed8f5540c3601c123d89ff1bc25686692811d Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 5 Nov 2020 16:10:17 +0100 Subject: [PATCH 1/2] added check to avoid sending empty metadata. --- .../java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java | 13 +++++++------ .../dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java | 4 +++- .../java/eu/dnetlib/dhp/oa/graph/dump/Utils.java | 4 ---- .../dhp/oa/graph/dump/oozie_app/workflow.xml | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java index 08c7ee90e..00ddcb5a8 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java @@ -4,7 +4,6 @@ package eu.dnetlib.dhp.oa.graph.dump; import java.io.*; import java.util.Optional; -import eu.dnetlib.dhp.common.MakeTarArchive; import org.apache.commons.compress.archivers.ar.ArArchiveEntry; import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -16,6 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.MakeTarArchive; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; public class MakeTar implements Serializable { @@ -41,9 +41,10 @@ public class MakeTar implements Serializable { final String inputPath = parser.get("sourcePath"); log.info("input path : {}", inputPath); - final int gBperSplit = Optional.ofNullable(parser.get("splitSize")) - .map(Integer::valueOf) - .orElse(10); + final int gBperSplit = Optional + .ofNullable(parser.get("splitSize")) + .map(Integer::valueOf) + .orElse(10); Configuration conf = new Configuration(); conf.set("fs.defaultFS", hdfsNameNode); @@ -54,7 +55,8 @@ public class MakeTar implements Serializable { } - public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit) throws IOException { + public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit) + throws IOException { RemoteIterator dir_iterator = fileSystem.listLocatedStatus(new Path(inputPath)); @@ -70,5 +72,4 @@ public class MakeTar implements Serializable { } - } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 23cd27449..fd8262544 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -98,8 +98,10 @@ public class SendToZenodoHDFS implements Serializable { } } + if (!metadata.equals("")) { + zenodoApiClient.sendMretadata(metadata); + } - zenodoApiClient.sendMretadata(metadata); if (publish) zenodoApiClient.publish(); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java index bd2738eaa..f88980f15 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java @@ -5,7 +5,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.function.MapFunction; @@ -13,14 +12,12 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; - import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.graph.Constants; - import eu.dnetlib.dhp.utils.DHPUtils; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -74,5 +71,4 @@ public class Utils { return new Gson().fromJson(sb.toString(), CommunityMap.class); } - } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml index 7321fd076..98081c285 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml @@ -418,7 +418,7 @@ --metadata${metadata} --communityMapPath${workingDir}/communityMap --conceptRecordId${conceptRecordId} - --newDeposition${newDeposition} + --depositionType${depositionType} From 7ebdfacee99ceb65ad3e44800d15f30413af24b4 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 5 Nov 2020 16:30:36 +0100 Subject: [PATCH 2/2] removed commented code and added documentation to new method --- .../dhp/common/api/ZenodoAPIClient.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java index 634336c46..1f267733d 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java @@ -59,8 +59,6 @@ public class ZenodoAPIClient implements Serializable { String json = "{}"; OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); - // RequestBody body = RequestBody.create(MEDIA_TYPE_JSON, json); - RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); Request request = new Request.Builder() @@ -128,7 +126,6 @@ public class ZenodoAPIClient implements Serializable { OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); - // RequestBody body = RequestBody.create(MEDIA_TYPE_JSON, metadata); RequestBody body = RequestBody.create(metadata, MEDIA_TYPE_JSON); Request request = new Request.Builder() @@ -166,7 +163,6 @@ public class ZenodoAPIClient implements Serializable { Request request = new Request.Builder() .url(urlString + "/" + deposition_id + "/actions/publish") .addHeader("Authorization", "Bearer " + access_token) - // .post(RequestBody.create(MEDIA_TYPE_JSON, json)) .post(body) .build(); @@ -202,7 +198,6 @@ public class ZenodoAPIClient implements Serializable { Request request = new Request.Builder() .url(urlString + "/" + deposition_id + "/actions/newversion") .addHeader("Authorization", "Bearer " + access_token) - // .post(RequestBody.create(MEDIA_TYPE_JSON, json)) .post(body) .build(); @@ -220,18 +215,26 @@ public class ZenodoAPIClient implements Serializable { } } + /** + * To finish uploading a version or new deposition not published + * It sets the deposition_id and the bucket to be used + * + * + * @param deposition_id the deposition id of the not yet published upload + * concept_rec_id = 656930 + * @return response code + * @throws IOException + * @throws MissingConceptDoiException + */ public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException { this.deposition_id = deposition_id; - String json = "{}"; - OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); Request request = new Request.Builder() .url(urlString + "/" + deposition_id) .addHeader("Authorization", "Bearer " + access_token) - // .post(RequestBody.create(MEDIA_TYPE_JSON, json)) .build(); try (Response response = httpClient.newCall(request).execute()) {