antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Protobuf to internal graph model, early steps

This commit is contained in:
Claudio Atzori 2019-10-24 16:56:13 +02:00
parent d38aeb8c6e
commit 79c4f1bbd8
2 changed files with 76 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java
View File

@ -1,37 +1,45 @@
package eu.dnetlib.dhp.graph;
import com.googlecode.protobuf.format.JsonFormat;
import eu.dnetlib.data.proto.KindProtos;
import eu.dnetlib.data.proto.OafProtos;
import eu.dnetlib.dhp.schema.oaf.*;
import java.io.Serializable;
import java.util.stream.Collectors;
import static eu.dnetlib.dhp.graph.ProtoUtils.mapKV;
public class ProtoConverter implements Serializable {
public static Oaf convert(String s) {
try {
final OafProtos.Oaf.Builder builder = OafProtos.Oaf.newBuilder();
JsonFormat.merge(s, builder);
OafProtos.Oaf oaf = ProtoUtils.parse(s);
if (builder.getKind() == KindProtos.Kind.entity)
return convertEntity(builder);
if (oaf.getKind() == KindProtos.Kind.entity)
return convertEntity(oaf);
else {
return convertRelation(builder);
return convertRelation(oaf);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
private static Relation convertRelation(OafProtos.Oaf.Builder oaf) {
return new Relation();
private static Relation convertRelation(OafProtos.Oaf oaf) {
final Relation rel = new Relation();
final OafProtos.OafRel r = oaf.getRel();
return rel
.setSource(r.getSource())
.setTarget(r.getTarget())
.setRelType(r.getRelType().toString())
.setSubRelType(r.getSubRelType().toString())
.setRelClass(r.getRelClass())
.setCollectedFrom(r.getCollectedfromList().stream()
.map(kv -> mapKV(kv))
.collect(Collectors.toList()));
}
private static OafEntity convertEntity(OafProtos.Oaf.Builder oaf) {
private static OafEntity convertEntity(OafProtos.Oaf oaf) {
switch (oaf.getEntity().getType()) {
case result:
@ -47,19 +55,19 @@ public class ProtoConverter implements Serializable {
}
}
private static Organization convertOrganization(OafProtos.Oaf.Builder oaf) {
private static Organization convertOrganization(OafProtos.Oaf oaf) {
return new Organization();
}
private static Datasource convertDataSource(OafProtos.Oaf.Builder oaf) {
private static Datasource convertDataSource(OafProtos.Oaf oaf) {
return new Datasource();
}
private static Project convertProject(OafProtos.Oaf.Builder oaf) {
private static Project convertProject(OafProtos.Oaf oaf) {
return new Project();
}
private static Result convertResult(OafProtos.Oaf.Builder oaf) {
private static Result convertResult(OafProtos.Oaf oaf) {
switch (oaf.getEntity().getResult().getMetadata().getResulttype().getClassid()) {
case "dataset":
return createDataset(oaf);
@ -74,19 +82,19 @@ public class ProtoConverter implements Serializable {
}
}
private static Software createSoftware(OafProtos.Oaf.Builder oaf) {
private static Software createSoftware(OafProtos.Oaf oaf) {
return new Software();
}
private static OtherResearchProducts createORP(OafProtos.Oaf.Builder oaf) {
private static OtherResearchProducts createORP(OafProtos.Oaf oaf) {
return new OtherResearchProducts();
}
private static Publication createPublication(OafProtos.Oaf.Builder oaf) {
private static Publication createPublication(OafProtos.Oaf oaf) {
return new Publication();
}
private static Dataset createDataset(OafProtos.Oaf.Builder oaf) {
private static Dataset createDataset(OafProtos.Oaf oaf) {
return new Dataset();
}
}

49
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java
View File

@ -1,4 +1,53 @@
package eu.dnetlib.dhp.graph;
import com.googlecode.protobuf.format.JsonFormat;
import eu.dnetlib.data.proto.FieldTypeProtos;
import eu.dnetlib.data.proto.OafProtos;
import eu.dnetlib.dhp.schema.oaf.DataInfo;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import java.util.List;
public class ProtoUtils {
public static OafProtos.Oaf parse(String json) throws JsonFormat.ParseException {
final OafProtos.Oaf.Builder builder = OafProtos.Oaf.newBuilder();
JsonFormat.merge(json, builder);
return builder.build();
}
public static KeyValue mapKV(FieldTypeProtos.KeyValue kv) {
return new KeyValue()
.setKey(kv.getKey())
.setValue(kv.getValue())
.setDataInfo(mapDataInfo(kv.getDataInfo()));
}
public static DataInfo mapDataInfo(FieldTypeProtos.DataInfo d) {
return new DataInfo()
.setDeletedbyinference(d.getDeletedbyinference())
.setInferenceprovenance(d.getInferenceprovenance())
.setInferred(d.getInferred())
.setInvisible(d.getInvisible())
.setProvenanceaction(mapQualifier(d.getProvenanceaction()));
}
public static Qualifier mapQualifier(FieldTypeProtos.Qualifier q) {
return new Qualifier()
.setClassid(q.getClassid())
.setClassname(q.getClassname())
.setSchemeid(q.getSchemeid())
.setSchemename(q.getSchemename())
.setDataInfo(q.hasDataInfo() ? mapDataInfo(q.getDataInfo()) : null);
}
public static StructuredProperty mapStructuredProperty(FieldTypeProtos.StructuredProperty sp) {
return new StructuredProperty()
.setValue(sp.getValue())
.setQualifier(mapQualifier(sp.getQualifier()))
.setDataInfo(sp.hasDataInfo() ? mapDataInfo(sp.getDataInfo()) : null);
}
}