forked from D-Net/dnet-hadoop
This commit is contained in:
parent
daa5d933e0
commit
566a763175
|
@ -3,7 +3,10 @@ package eu.dnetlib.dhp.oa.graph.dump.gcat;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
||||||
public class SendToCatalogue implements Serializable {
|
public class SendToCatalogue implements Serializable {
|
||||||
|
@ -22,6 +25,37 @@ public class SendToCatalogue implements Serializable {
|
||||||
final String hdfsPath = parser.get("hdfsPath");
|
final String hdfsPath = parser.get("hdfsPath");
|
||||||
final String hdfsNameNode = parser.get("hdfsNameNode");
|
final String hdfsNameNode = parser.get("hdfsNameNode");
|
||||||
|
|
||||||
|
fileStatusIterator = new FileStat
|
||||||
|
|
||||||
|
while (fileStatusListIterator.hasNext()) {
|
||||||
|
LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
||||||
|
|
||||||
|
Path p = fileStatus.getPath();
|
||||||
|
String p_string = p.toString();
|
||||||
|
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
|
||||||
|
String community = tmp.substring(tmp.lastIndexOf("/") + 1);
|
||||||
|
log.info("Sending information for community: " + community);
|
||||||
|
String community_name = communityMap.get(community).replace(" ", "_");
|
||||||
|
log.info("Copying information for community: " + community);
|
||||||
|
fileSystem.copyToLocalFile(p, new Path("/tmp/" + community_name));
|
||||||
|
File f = new File("/tmp/" + community_name);
|
||||||
|
try {
|
||||||
|
apiClient.upload(f, community_name);
|
||||||
|
apiClient.sendMretadata(metadata);
|
||||||
|
apiClient.publish();
|
||||||
|
} catch (Exception e) {
|
||||||
|
if (f.exists()) {
|
||||||
|
log.info("Deleting information for community: " + community);
|
||||||
|
f.delete();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (f.exists()) {
|
||||||
|
log.info("Deleting information for community: " + community);
|
||||||
|
f.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue