D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

WIP: dump of the OpenAIRE graph - Changes #103

Closed
miriam.baglioni wants to merge 77 commits from miriam.baglioni/dnet-hadoop:dump into master
pull from: miriam.baglioni/dnet-hadoop:dump
merge into: D-Net:master
Conversation 3 Commits 77 Files Changed 59 +4
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 5022f1b50d - Show all commits

4
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java
View File

@ -453,6 +453,7 @@ public class DumpGraphEntities implements Serializable {
.map( .map(
(MapFunction<E, Organization>) o -> mapOrganization((eu.dnetlib.dhp.schema.oaf.Organization) o), (MapFunction<E, Organization>) o -> mapOrganization((eu.dnetlib.dhp.schema.oaf.Organization) o),
Encoders.bean(Organization.class)) Encoders.bean(Organization.class))
.filter(Objects::nonNull)
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
@ -460,6 +461,9 @@ public class DumpGraphEntities implements Serializable {
} }
private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) { private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) {
if (org.getDataInfo().getDeletedbyinference())
return null;
Organization organization = new Organization(); Organization organization = new Organization();
Optional Optional