miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

added check to verify the used record is not empty

This commit is contained in:
Miriam Baglioni 2020-07-10 17:00:30 +02:00
parent d77d213d7c
commit 4e494f5152
1 changed files with 31 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SendToCatalogue.java
View File

@ -1,10 +1,10 @@
package eu.dnetlib.dhp.oa.graph.dump.gcat;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.Serializable;
import java.io.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@ -55,23 +55,35 @@ public class SendToCatalogue implements Serializable {
Path p = fileStatus.getPath();
String p_string = p.toString();
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
String name = tmp.substring(tmp.lastIndexOf("/") + 1);
log.info("Sending information for : " + name);
// String community_name = communityMap.get(community).replace(" ", "_");
log.info("Copying information for : " + name);
fileSystem.copyToLocalFile(p, new Path("/tmp/" + name));
BufferedReader reader = new BufferedReader(new FileReader("/tmp/" + name));
String line;
while ((line = reader.readLine()) != null) {
if (HttpStatus.SC_CREATED != gCatAPIClient.publish(line)) {
log.error("entry not created for item " + line);
if (!p_string.endsWith("_SUCCESS")) {
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
String name = tmp.substring(tmp.lastIndexOf("/") + 1);
log.info("Copying information for : " + name);
fileSystem.copyToLocalFile(p, new Path("/tmp/" + name));
try {
InputStream in = new GZIPInputStream(new FileInputStream("/tmp/" + name));
BufferedReader reader = new BufferedReader(
new InputStreamReader(in));
String line;
while ((line = reader.readLine()) != null) {
if (HttpStatus.SC_CREATED != gCatAPIClient.publish(line)) {
log.error("entry not created for item " + line);
}
}
reader.close();
in.close();
} finally {
log.info("deleting information for: " + name);
File f = new File("/tmp/" + name);
if (f.exists()) {
f.delete();
}
}
}
reader.close();
log.info("deleting information for: " + name);
File f = new File("/tmp/" + name);
f.delete();
}