D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

added check not to dump deletedByInference entities

This commit is contained in:
Miriam Baglioni 2021-07-13 16:11:46 +02:00
parent e9a17ec899
commit 5295d10691
3 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java
View File

@ -453,6 +453,7 @@ public class DumpGraphEntities implements Serializable {
.map( .map(
(MapFunction<E, Organization>) o -> mapOrganization((eu.dnetlib.dhp.schema.oaf.Organization) o), (MapFunction<E, Organization>) o -> mapOrganization((eu.dnetlib.dhp.schema.oaf.Organization) o),
Encoders.bean(Organization.class)) Encoders.bean(Organization.class))
.filter(Objects::nonNull)
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
@ -460,6 +461,8 @@ public class DumpGraphEntities implements Serializable {
} }
private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) { private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) {
if(org.getDataInfo().getDeletedbyinference())
return null;
Organization organization = new Organization(); Organization organization = new Organization();
Optional Optional

4
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java
View File

@ -110,9 +110,11 @@ public class SparkSelectValidRelationsJob implements Serializable {
"UNION ALL " + "UNION ALL " +
"SELECT id " + "SELECT id " +
"FROM project " + "FROM project " +
"WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " +
"UNION ALL " + "UNION ALL " +
"SELECT id " + "SELECT id " +
"FROM datasource ") "FROM datasource " +
"WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " )
.createOrReplaceTempView("identifiers"); .createOrReplaceTempView("identifiers");
spark spark

2
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java
View File

@ -88,7 +88,7 @@ public class DumpOrganizationProjectDatasourceTest {
org.apache.spark.sql.Dataset<eu.dnetlib.dhp.schema.dump.oaf.graph.Organization> verificationDataset = spark org.apache.spark.sql.Dataset<eu.dnetlib.dhp.schema.dump.oaf.graph.Organization> verificationDataset = spark
.createDataset(tmp.rdd(), Encoders.bean(eu.dnetlib.dhp.schema.dump.oaf.graph.Organization.class)); .createDataset(tmp.rdd(), Encoders.bean(eu.dnetlib.dhp.schema.dump.oaf.graph.Organization.class));
Assertions.assertEquals(34, verificationDataset.count()); Assertions.assertEquals(15, verificationDataset.count());
verificationDataset verificationDataset
.foreach( .foreach(