From be5ed8f5540c3601c123d89ff1bc25686692811d Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 5 Nov 2020 16:10:17 +0100 Subject: [PATCH] added check to avoid sending empty metadata. --- .../java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java | 13 +++++++------ .../dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java | 4 +++- .../java/eu/dnetlib/dhp/oa/graph/dump/Utils.java | 4 ---- .../dhp/oa/graph/dump/oozie_app/workflow.xml | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java index 08c7ee90e..00ddcb5a8 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java @@ -4,7 +4,6 @@ package eu.dnetlib.dhp.oa.graph.dump; import java.io.*; import java.util.Optional; -import eu.dnetlib.dhp.common.MakeTarArchive; import org.apache.commons.compress.archivers.ar.ArArchiveEntry; import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -16,6 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.MakeTarArchive; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; public class MakeTar implements Serializable { @@ -41,9 +41,10 @@ public class MakeTar implements Serializable { final String inputPath = parser.get("sourcePath"); log.info("input path : {}", inputPath); - final int gBperSplit = Optional.ofNullable(parser.get("splitSize")) - .map(Integer::valueOf) - .orElse(10); + final int gBperSplit = Optional + .ofNullable(parser.get("splitSize")) + .map(Integer::valueOf) + .orElse(10); Configuration conf = new Configuration(); conf.set("fs.defaultFS", hdfsNameNode); @@ -54,7 +55,8 @@ public class MakeTar implements Serializable { } - public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit) throws IOException { + public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit) + throws IOException { RemoteIterator dir_iterator = fileSystem.listLocatedStatus(new Path(inputPath)); @@ -70,5 +72,4 @@ public class MakeTar implements Serializable { } - } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 23cd27449..fd8262544 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -98,8 +98,10 @@ public class SendToZenodoHDFS implements Serializable { } } + if (!metadata.equals("")) { + zenodoApiClient.sendMretadata(metadata); + } - zenodoApiClient.sendMretadata(metadata); if (publish) zenodoApiClient.publish(); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java index bd2738eaa..f88980f15 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java @@ -5,7 +5,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.function.MapFunction; @@ -13,14 +12,12 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; - import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.graph.Constants; - import eu.dnetlib.dhp.utils.DHPUtils; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -74,5 +71,4 @@ public class Utils { return new Gson().fromJson(sb.toString(), CommunityMap.class); } - } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml index 7321fd076..98081c285 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml @@ -418,7 +418,7 @@ --metadata${metadata} --communityMapPath${workingDir}/communityMap --conceptRecordId${conceptRecordId} - --newDeposition${newDeposition} + --depositionType${depositionType}