enrico.ottonello
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of https://code-repo.d3science.org/D-Net/dnet-hadoop

This commit is contained in:
Claudio Atzori 2019-10-25 10:55:43 +02:00
parent 4eaff36ea6 c8d6d6bbd1
commit 331b853ad6
2 changed files with 48 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java
View File

@ -1,9 +1,6 @@
package eu.dnetlib.dhp.graph; package eu.dnetlib.dhp.graph;
import eu.dnetlib.data.proto.DatasourceProtos; import eu.dnetlib.data.proto.*;
import eu.dnetlib.data.proto.KindProtos;
import eu.dnetlib.data.proto.OafProtos;
import eu.dnetlib.data.proto.ProjectProtos;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.*;
import java.io.Serializable; import java.io.Serializable;
@ -61,11 +58,28 @@ public class ProtoConverter implements Serializable {
} }
private static Organization convertOrganization(OafProtos.Oaf oaf) { private static Organization convertOrganization(OafProtos.Oaf oaf) {
final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata(); final OrganizationProtos.Organization.Metadata m = oaf.getEntity().getOrganization().getMetadata();
final Organization org = setOaf(new Organization(), oaf); final Organization org = setOaf(new Organization(), oaf);
return setEntity(org, oaf); return setEntity(org, oaf)
.setLegalshortname(mapStringField(m.getLegalshortname()))
//TODO set org fields .setLegalname(mapStringField(m.getLegalname()))
.setAlternativeNames(m.getAlternativeNamesList().
stream()
.map(ProtoUtils::mapStringField)
.collect(Collectors.toList()))
.setWebsiteurl(mapStringField(m.getWebsiteurl()))
.setLogourl(mapStringField(m.getLogourl()))
.setEclegalbody(mapStringField(m.getEclegalbody()))
.setEclegalperson(mapStringField(m.getEclegalperson()))
.setEcnonprofit(mapStringField(m.getEcnonprofit()))
.setEcresearchorganization(mapStringField(m.getEcresearchorganization()))
.setEchighereducation(mapStringField(m.getEchighereducation()))
.setEcinternationalorganizationeurinterests(mapStringField(m.getEcinternationalorganizationeurinterests()))
.setEcinternationalorganization(mapStringField(m.getEcinternationalorganization()))
.setEcenterprise(mapStringField(m.getEcenterprise()))
.setEcsmevalidated(mapStringField(m.getEcsmevalidated()))
.setEcnutscode(mapStringField(m.getEcnutscode()))
.setCountry(mapQualifier(m.getCountry()));
} }
private static Datasource convertDataSource(OafProtos.Oaf oaf) { private static Datasource convertDataSource(OafProtos.Oaf oaf) {

11
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.dhp.graph;
import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Datasource;
import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.Organization;
import eu.dnetlib.dhp.schema.oaf.Publication; import eu.dnetlib.dhp.schema.oaf.Publication;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@ -42,14 +43,14 @@ public class SparkGraphImporterJob {
final JavaRDD<Datasource> datasources = inputRDD final JavaRDD<Organization> datasources = inputRDD
.filter(s -> s._1().split("@")[2].equalsIgnoreCase("body")) .filter(s -> s._1().split("@")[2].equalsIgnoreCase("body"))
.map(Tuple2::_2) .map(Tuple2::_2)
.map(ProtoConverter::convert) .map(ProtoConverter::convert)
.filter(s-> s instanceof Datasource) .filter(s-> s instanceof Organization)
.map(s->(Datasource)s); .map(s->(Organization)s);
final Encoder<Datasource> encoder = Encoders.bean(Datasource.class); final Encoder<Organization> encoder = Encoders.bean(Organization.class);
final Dataset<Datasource> mdstore = spark.createDataset(datasources.rdd(), encoder); final Dataset<Organization> mdstore = spark.createDataset(datasources.rdd(), encoder);
System.out.println(mdstore.count()); System.out.println(mdstore.count());