forked from D-Net/dnet-hadoop
added check to verify the used record is not empty
This commit is contained in:
parent
d77d213d7c
commit
4e494f5152
|
@ -1,10 +1,10 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.oa.graph.dump.gcat;
|
package eu.dnetlib.dhp.oa.graph.dump.gcat;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.*;
|
||||||
import java.io.File;
|
import java.util.zip.GZIPInputStream;
|
||||||
import java.io.FileReader;
|
import java.util.zip.ZipEntry;
|
||||||
import java.io.Serializable;
|
import java.util.zip.ZipInputStream;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
|
@ -55,23 +55,35 @@ public class SendToCatalogue implements Serializable {
|
||||||
|
|
||||||
Path p = fileStatus.getPath();
|
Path p = fileStatus.getPath();
|
||||||
String p_string = p.toString();
|
String p_string = p.toString();
|
||||||
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
|
if (!p_string.endsWith("_SUCCESS")) {
|
||||||
String name = tmp.substring(tmp.lastIndexOf("/") + 1);
|
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
|
||||||
log.info("Sending information for : " + name);
|
String name = tmp.substring(tmp.lastIndexOf("/") + 1);
|
||||||
// String community_name = communityMap.get(community).replace(" ", "_");
|
log.info("Copying information for : " + name);
|
||||||
log.info("Copying information for : " + name);
|
fileSystem.copyToLocalFile(p, new Path("/tmp/" + name));
|
||||||
fileSystem.copyToLocalFile(p, new Path("/tmp/" + name));
|
|
||||||
BufferedReader reader = new BufferedReader(new FileReader("/tmp/" + name));
|
try {
|
||||||
String line;
|
InputStream in = new GZIPInputStream(new FileInputStream("/tmp/" + name));
|
||||||
while ((line = reader.readLine()) != null) {
|
|
||||||
if (HttpStatus.SC_CREATED != gCatAPIClient.publish(line)) {
|
BufferedReader reader = new BufferedReader(
|
||||||
log.error("entry not created for item " + line);
|
new InputStreamReader(in));
|
||||||
|
String line;
|
||||||
|
while ((line = reader.readLine()) != null) {
|
||||||
|
if (HttpStatus.SC_CREATED != gCatAPIClient.publish(line)) {
|
||||||
|
log.error("entry not created for item " + line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reader.close();
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
log.info("deleting information for: " + name);
|
||||||
|
File f = new File("/tmp/" + name);
|
||||||
|
if (f.exists()) {
|
||||||
|
f.delete();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
reader.close();
|
|
||||||
log.info("deleting information for: " + name);
|
|
||||||
File f = new File("/tmp/" + name);
|
|
||||||
f.delete();
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue