From 566a763175edfcbc1fe16310ccc774e16d2b6e63 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 1 Jul 2020 18:13:48 +0200 Subject: [PATCH] - --- .../oa/graph/dump/gcat/SendToCatalogue.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SendToCatalogue.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SendToCatalogue.java index 1750225935..e4dd188ae4 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SendToCatalogue.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SendToCatalogue.java @@ -3,7 +3,10 @@ package eu.dnetlib.dhp.oa.graph.dump.gcat; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import java.io.File; import java.io.Serializable; public class SendToCatalogue implements Serializable { @@ -22,6 +25,37 @@ public class SendToCatalogue implements Serializable { final String hdfsPath = parser.get("hdfsPath"); final String hdfsNameNode = parser.get("hdfsNameNode"); + fileStatusIterator = new FileStat + + while (fileStatusListIterator.hasNext()) { + LocatedFileStatus fileStatus = fileStatusListIterator.next(); + + Path p = fileStatus.getPath(); + String p_string = p.toString(); + String tmp = p_string.substring(0, p_string.lastIndexOf("/")); + String community = tmp.substring(tmp.lastIndexOf("/") + 1); + log.info("Sending information for community: " + community); + String community_name = communityMap.get(community).replace(" ", "_"); + log.info("Copying information for community: " + community); + fileSystem.copyToLocalFile(p, new Path("/tmp/" + community_name)); + File f = new File("/tmp/" + community_name); + try { + apiClient.upload(f, community_name); + apiClient.sendMretadata(metadata); + apiClient.publish(); + } catch (Exception e) { + if (f.exists()) { + log.info("Deleting information for community: " + community); + f.delete(); + } + } finally { + if (f.exists()) { + log.info("Deleting information for community: " + community); + f.delete(); + } + } + } + }