From cf6d80b2ab92221e4942b32bf6bd5297e6090ce8 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 30 Jul 2020 16:31:22 +0200 Subject: [PATCH] added command to close the writer --- .../graph/dump/graph/CreateContextEntities.java | 11 ++++++++++- .../graph/dump/graph/CreateContextRelation.java | 16 +++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextEntities.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextEntities.java index 425202a62..be2b028bc 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextEntities.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextEntities.java @@ -21,6 +21,8 @@ import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gson.Gson; + import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.schema.dump.oaf.graph.ResearchCommunity; @@ -49,7 +51,7 @@ public class CreateContextEntities implements Serializable { final String hdfsPath = parser.get("hdfsPath"); log.info("hdfsPath: {}", hdfsPath); - final String hdfsNameNode = parser.get("nameNode"); + final String hdfsNameNode = parser.get("hdfsNameNode"); log.info("nameNode: {}", hdfsNameNode); final String isLookUpUrl = parser.get("isLookUpUrl"); @@ -60,6 +62,12 @@ public class CreateContextEntities implements Serializable { log.info("Processing contexts..."); cce.execute(Process::getEntity, isLookUpUrl); + cce.close(); + + } + + private void close() throws IOException { + writer.close(); } public CreateContextEntities(String hdfsPath, String hdfsNameNode) throws IOException { @@ -92,6 +100,7 @@ public class CreateContextEntities implements Serializable { protected void writeEntity(final R r) { try { writer.write(Utils.OBJECT_MAPPER.writeValueAsString(r)); + //log.info("writing context : {}", new Gson().toJson(r)); writer.newLine(); } catch (final Exception e) { throw new RuntimeException(e); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextRelation.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextRelation.java index 3c5ca2898..e5a2ee36b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextRelation.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/CreateContextRelation.java @@ -54,7 +54,7 @@ public class CreateContextRelation implements Serializable { final String hdfsPath = parser.get("hdfsPath"); log.info("hdfsPath: {}", hdfsPath); - final String hdfsNameNode = parser.get("nameNode"); + final String hdfsNameNode = parser.get("hdfsNameNode"); log.info("nameNode: {}", hdfsNameNode); final String isLookUpUrl = parser.get("isLookUpUrl"); @@ -66,11 +66,17 @@ public class CreateContextRelation implements Serializable { cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class)); log.info("Creating relations for projects... "); - cce - .execute( - Process::getRelation, CONTEX_RELATION_PROJECT, - ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class)); +// cce +// .execute( +// Process::getRelation, CONTEX_RELATION_PROJECT, +// ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class)); + cce.close(); + + } + + private void close() throws IOException { + writer.close(); } public CreateContextRelation(String hdfsPath, String hdfsNameNode, String isLookUpUrl)