miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

-

This commit is contained in:
Miriam Baglioni 2020-07-01 18:13:48 +02:00
parent daa5d933e0
commit 566a763175
1 changed files with 34 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SendToCatalogue.java
View File

@ -3,7 +3,10 @@ package eu.dnetlib.dhp.oa.graph.dump.gcat;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.Serializable;
public class SendToCatalogue implements Serializable {
@ -22,6 +25,37 @@ public class SendToCatalogue implements Serializable {
final String hdfsPath = parser.get("hdfsPath");
final String hdfsNameNode = parser.get("hdfsNameNode");
fileStatusIterator = new FileStat
while (fileStatusListIterator.hasNext()) {
LocatedFileStatus fileStatus = fileStatusListIterator.next();
Path p = fileStatus.getPath();
String p_string = p.toString();
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
String community = tmp.substring(tmp.lastIndexOf("/") + 1);
log.info("Sending information for community: " + community);
String community_name = communityMap.get(community).replace(" ", "_");
log.info("Copying information for community: " + community);
fileSystem.copyToLocalFile(p, new Path("/tmp/" + community_name));
File f = new File("/tmp/" + community_name);
try {
apiClient.upload(f, community_name);
apiClient.sendMretadata(metadata);
apiClient.publish();
} catch (Exception e) {
if (f.exists()) {
log.info("Deleting information for community: " + community);
f.delete();
}
} finally {
if (f.exists()) {
log.info("Deleting information for community: " + community);
f.delete();
}
}
}
}