D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

added command to close the writer

This commit is contained in:
Miriam Baglioni 2020-07-30 16:31:22 +02:00
parent f985bca37b
commit cf6d80b2ab
2 changed files with 21 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextEntities.java
View File

@ -21,6 +21,8 @@ import org.apache.hadoop.fs.Path;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.Utils;
import eu.dnetlib.dhp.schema.dump.oaf.graph.ResearchCommunity; import eu.dnetlib.dhp.schema.dump.oaf.graph.ResearchCommunity;
@ -49,7 +51,7 @@ public class CreateContextEntities implements Serializable {
final String hdfsPath = parser.get("hdfsPath"); final String hdfsPath = parser.get("hdfsPath");
log.info("hdfsPath: {}", hdfsPath); log.info("hdfsPath: {}", hdfsPath);
final String hdfsNameNode = parser.get("nameNode"); final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("nameNode: {}", hdfsNameNode); log.info("nameNode: {}", hdfsNameNode);
final String isLookUpUrl = parser.get("isLookUpUrl"); final String isLookUpUrl = parser.get("isLookUpUrl");
@ -60,6 +62,12 @@ public class CreateContextEntities implements Serializable {
log.info("Processing contexts..."); log.info("Processing contexts...");
cce.execute(Process::getEntity, isLookUpUrl); cce.execute(Process::getEntity, isLookUpUrl);
cce.close();
}
private void close() throws IOException {
writer.close();
} }
public CreateContextEntities(String hdfsPath, String hdfsNameNode) throws IOException { public CreateContextEntities(String hdfsPath, String hdfsNameNode) throws IOException {
@ -92,6 +100,7 @@ public class CreateContextEntities implements Serializable {
protected <R extends ResearchInitiative> void writeEntity(final R r) { protected <R extends ResearchInitiative> void writeEntity(final R r) {
try { try {
writer.write(Utils.OBJECT_MAPPER.writeValueAsString(r)); writer.write(Utils.OBJECT_MAPPER.writeValueAsString(r));
//log.info("writing context : {}", new Gson().toJson(r));
writer.newLine(); writer.newLine();
} catch (final Exception e) { } catch (final Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);

16
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextRelation.java
View File

@ -54,7 +54,7 @@ public class CreateContextRelation implements Serializable {
final String hdfsPath = parser.get("hdfsPath"); final String hdfsPath = parser.get("hdfsPath");
log.info("hdfsPath: {}", hdfsPath); log.info("hdfsPath: {}", hdfsPath);
final String hdfsNameNode = parser.get("nameNode"); final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("nameNode: {}", hdfsNameNode); log.info("nameNode: {}", hdfsNameNode);
final String isLookUpUrl = parser.get("isLookUpUrl"); final String isLookUpUrl = parser.get("isLookUpUrl");
@ -66,11 +66,17 @@ public class CreateContextRelation implements Serializable {
cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class)); cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class));
log.info("Creating relations for projects... "); log.info("Creating relations for projects... ");
cce // cce
.execute( // .execute(
Process::getRelation, CONTEX_RELATION_PROJECT, // Process::getRelation, CONTEX_RELATION_PROJECT,
ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class)); // ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class));
cce.close();
}
private void close() throws IOException {
writer.close();
} }
public CreateContextRelation(String hdfsPath, String hdfsNameNode, String isLookUpUrl) public CreateContextRelation(String hdfsPath, String hdfsNameNode, String isLookUpUrl)