diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Author.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Author.java index 436bc1b6e..ebb6959e5 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Author.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Author.java @@ -1,19 +1,30 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; +import java.util.List; public class Author implements Serializable { + private String fullname; + private String name; - private String typology; + private String surname; - private String provenance; + private Integer rank; - private String trust; + private List pid; - // json containing a Citation or Statistics - private String value; + private List> affiliation; + + public String getFullname() { + return fullname; + } + + public Author setFullname(String fullname) { + this.fullname = fullname; + return this; + } public String getName() { return name; @@ -24,39 +35,39 @@ public class Author implements Serializable { return this; } - public String getTypology() { - return typology; + public String getSurname() { + return surname; } - public Author setTypology(String typology) { - this.typology = typology; + public Author setSurname(String surname) { + this.surname = surname; return this; } - public String getProvenance() { - return provenance; + public Integer getRank() { + return rank; } - public Author setProvenance(String provenance) { - this.provenance = provenance; + public Author setRank(Integer rank) { + this.rank = rank; return this; } - public String getTrust() { - return trust; + public List getPid() { + return pid; } - public Author setTrust(String trust) { - this.trust = trust; + public Author setPid(List pid) { + this.pid = pid; return this; } - public String getValue() { - return value; + public List> getAffiliation() { + return affiliation; } - public Author setValue(String value) { - this.value = value; + public Author setAffiliation(List> affiliation) { + this.affiliation = affiliation; return this; } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java index 25fd7e5d5..2b5562b4c 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public class Dataset extends Result implements Serializable { +public class Dataset extends Result implements Serializable { private Field storagedate; @@ -81,4 +81,9 @@ public class Dataset extends Result implements Serializable { this.geolocation = geolocation; return this; } + + @Override + protected Dataset self() { + return this; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java index 33e994d4a..ca261bec7 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public class Datasource extends OafEntity implements Serializable { +public class Datasource extends OafEntity implements Serializable { private Qualifier datasourcetype; @@ -396,4 +396,9 @@ public class Datasource extends OafEntity implements Serializable { this.journal = journal; return this; } + + @Override + protected Datasource self() { + return this; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java index 2924deaba..5d198c4e6 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java @@ -2,27 +2,29 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; -public abstract class Oaf implements Serializable { +public abstract class Oaf> implements Serializable { private DataInfo dataInfo; private Long lastupdatetimestamp; + protected abstract T self(); + public DataInfo getDataInfo() { return dataInfo; } - public Oaf setDataInfo(DataInfo dataInfo) { + public T setDataInfo(DataInfo dataInfo) { this.dataInfo = dataInfo; - return this; + return self(); } public Long getLastupdatetimestamp() { return lastupdatetimestamp; } - public Oaf setLastupdatetimestamp(Long lastupdatetimestamp) { + public T setLastupdatetimestamp(Long lastupdatetimestamp) { this.lastupdatetimestamp = lastupdatetimestamp; - return this; + return self(); } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java index f00d38583..5e5fdd70b 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public abstract class OafEntity extends Oaf implements Serializable { +public abstract class OafEntity> extends Oaf implements Serializable { private String id; @@ -21,76 +21,75 @@ public abstract class OafEntity extends Oaf implements Serializable { private OAIProvenance oaiprovenance; - public String getId() { return id; } - public OafEntity setId(String id) { + public T setId(String id) { this.id = id; - return this; + return self(); } public List getOriginalId() { return originalId; } - public OafEntity setOriginalId(List originalId) { + public T setOriginalId(List originalId) { this.originalId = originalId; - return this; + return self(); } public List getCollectedfrom() { return collectedfrom; } - public OafEntity setCollectedfrom(List collectedfrom) { + public T setCollectedfrom(List collectedfrom) { this.collectedfrom = collectedfrom; - return this; + return self(); } public List getPid() { return pid; } - public OafEntity setPid(List pid) { + public T setPid(List pid) { this.pid = pid; - return this; + return self(); } public String getDateofcollection() { return dateofcollection; } - public OafEntity setDateofcollection(String dateofcollection) { + public T setDateofcollection(String dateofcollection) { this.dateofcollection = dateofcollection; - return this; + return self(); } public String getDateoftransformation() { return dateoftransformation; } - public OafEntity setDateoftransformation(String dateoftransformation) { + public T setDateoftransformation(String dateoftransformation) { this.dateoftransformation = dateoftransformation; - return this; + return self(); } public List getExtraInfo() { return extraInfo; } - public OafEntity setExtraInfo(List extraInfo) { + public T setExtraInfo(List extraInfo) { this.extraInfo = extraInfo; - return this; + return self(); } public OAIProvenance getOaiprovenance() { return oaiprovenance; } - public OafEntity setOaiprovenance(OAIProvenance oaiprovenance) { + public T setOaiprovenance(OAIProvenance oaiprovenance) { this.oaiprovenance = oaiprovenance; - return this; + return self(); } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Organization.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Organization.java index 260fc22b6..c03a78419 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Organization.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Organization.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public class Organization extends OafEntity implements Serializable { +public class Organization extends OafEntity implements Serializable { private Field legalshortname; @@ -180,4 +180,9 @@ public class Organization extends OafEntity implements Serializable { this.country = country; return this; } + + @Override + protected Organization self() { + return this; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OtherResearchProducts.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OtherResearchProducts.java index 3ca231587..3fdf1ae32 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OtherResearchProducts.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OtherResearchProducts.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public class OtherResearchProducts extends Result implements Serializable { +public class OtherResearchProducts extends Result implements Serializable { private List> contactperson; @@ -37,4 +37,9 @@ public class OtherResearchProducts extends Result implements Serializable { this.tool = tool; return this; } + + @Override + protected OtherResearchProducts self() { + return this; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java index 5efe1526f..13e0b4431 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public class Project extends OafEntity implements Serializable { +public class Project extends OafEntity implements Serializable { private Field websiteurl; @@ -290,4 +290,9 @@ public class Project extends OafEntity implements Serializable { this.fundedamount = fundedamount; return this; } + + @Override + protected Project self() { + return this; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Publication.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Publication.java index bf4eab93c..433e217db 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Publication.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Publication.java @@ -2,7 +2,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; -public class Publication extends Result implements Serializable { +public class Publication extends Result implements Serializable { // publication specific private Journal journal; @@ -15,4 +15,9 @@ public class Publication extends Result implements Serializable { this.journal = journal; return this; } + + @Override + protected Publication self() { + return this; + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index db8807e21..da1b4b439 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -2,7 +2,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.util.List; -public class Relation extends Oaf { +public class Relation extends Oaf { private String relType; @@ -22,7 +22,7 @@ public class Relation extends Oaf { public Relation setRelType(String relType) { this.relType = relType; - return this; + return self(); } public String getSubRelType() { @@ -31,7 +31,7 @@ public class Relation extends Oaf { public Relation setSubRelType(String subRelType) { this.subRelType = subRelType; - return this; + return self(); } public String getRelClass() { @@ -40,7 +40,7 @@ public class Relation extends Oaf { public Relation setRelClass(String relClass) { this.relClass = relClass; - return this; + return self(); } public String getSource() { @@ -49,7 +49,7 @@ public class Relation extends Oaf { public Relation setSource(String source) { this.source = source; - return this; + return self(); } public String getTarget() { @@ -58,7 +58,7 @@ public class Relation extends Oaf { public Relation setTarget(String target) { this.target = target; - return this; + return self(); } public List getCollectedFrom() { @@ -67,6 +67,11 @@ public class Relation extends Oaf { public Relation setCollectedFrom(List collectedFrom) { this.collectedFrom = collectedFrom; + return self(); + } + + @Override + protected Relation self() { return this; } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index b0630825a..b496e9683 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public abstract class Result extends OafEntity implements Serializable { +public abstract class Result> extends OafEntity implements Serializable { private List author; @@ -53,188 +53,188 @@ public abstract class Result extends OafEntity implements Serializable { return author; } - public Result setAuthor(List author) { + public T setAuthor(List author) { this.author = author; - return this; + return self(); } public Qualifier getResulttype() { return resulttype; } - public Result setResulttype(Qualifier resulttype) { + public T setResulttype(Qualifier resulttype) { this.resulttype = resulttype; - return this; + return self(); } public Qualifier getLanguage() { return language; } - public Result setLanguage(Qualifier language) { + public T setLanguage(Qualifier language) { this.language = language; - return this; + return self(); } public List getCountry() { return country; } - public Result setCountry(List country) { + public T setCountry(List country) { this.country = country; - return this; + return self(); } public List getSubject() { return subject; } - public Result setSubject(List subject) { + public T setSubject(List subject) { this.subject = subject; - return this; + return self(); } public List getTitle() { return title; } - public Result setTitle(List title) { + public T setTitle(List title) { this.title = title; - return this; + return self(); } public List getRelevantdate() { return relevantdate; } - public Result setRelevantdate(List relevantdate) { + public T setRelevantdate(List relevantdate) { this.relevantdate = relevantdate; - return this; + return self(); } public List> getDescription() { return description; } - public Result setDescription(List> description) { + public T setDescription(List> description) { this.description = description; - return this; + return self(); } public Field getDateofacceptance() { return dateofacceptance; } - public Result setDateofacceptance(Field dateofacceptance) { + public T setDateofacceptance(Field dateofacceptance) { this.dateofacceptance = dateofacceptance; - return this; + return self(); } public Field getPublisher() { return publisher; } - public Result setPublisher(Field publisher) { + public T setPublisher(Field publisher) { this.publisher = publisher; - return this; + return self(); } public Field getEmbargoenddate() { return embargoenddate; } - public Result setEmbargoenddate(Field embargoenddate) { + public T setEmbargoenddate(Field embargoenddate) { this.embargoenddate = embargoenddate; - return this; + return self(); } public List> getSource() { return source; } - public Result setSource(List> source) { + public T setSource(List> source) { this.source = source; - return this; + return self(); } public List> getFulltext() { return fulltext; } - public Result setFulltext(List> fulltext) { + public T setFulltext(List> fulltext) { this.fulltext = fulltext; - return this; + return self(); } public List> getFormat() { return format; } - public Result setFormat(List> format) { + public T setFormat(List> format) { this.format = format; - return this; + return self(); } public List> getContributor() { return contributor; } - public Result setContributor(List> contributor) { + public T setContributor(List> contributor) { this.contributor = contributor; - return this; + return self(); } public Qualifier getResourcetype() { return resourcetype; } - public Result setResourcetype(Qualifier resourcetype) { + public T setResourcetype(Qualifier resourcetype) { this.resourcetype = resourcetype; - return this; + return self(); } public List> getCoverage() { return coverage; } - public Result setCoverage(List> coverage) { + public T setCoverage(List> coverage) { this.coverage = coverage; - return this; + return self(); } public Field getRefereed() { return refereed; } - public Result setRefereed(Field refereed) { + public T setRefereed(Field refereed) { this.refereed = refereed; - return this; + return self(); } public List getContext() { return context; } - public Result setContext(List context) { + public T setContext(List context) { this.context = context; - return this; + return self(); } public List getExternalReference() { return externalReference; } - public Result setExternalReference(List externalReference) { + public T setExternalReference(List externalReference) { this.externalReference = externalReference; - return this; + return self(); } public List getInstance() { return instance; } - public Result setInstance(List instance) { + public T setInstance(List instance) { this.instance = instance; - return this; + return self(); } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java index 451efb99a..26c673d14 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; import java.util.List; -public class Software extends Result implements Serializable { +public class Software extends Result implements Serializable { private List> documentationUrl; @@ -48,4 +48,9 @@ public class Software extends Result implements Serializable { this.programmingLanguage = programmingLanguage; return this; } + + @Override + protected Software self() { + return this; + } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java index 98dc375fa..fe8a87dac 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java @@ -58,21 +58,38 @@ public class ProtoConverter implements Serializable { } private static Organization convertOrganization(OafProtos.Oaf oaf) { - final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata(); + final OrganizationProtos.Organization.Metadata m = oaf.getEntity().getOrganization().getMetadata(); final Organization org = setOaf(new Organization(), oaf); - return setEntity(org, oaf); - - //TODO set org fields - } + return setEntity(org, oaf) + .setLegalshortname(mapStringField(m.getLegalshortname())) + .setLegalname(mapStringField(m.getLegalname())) + .setAlternativeNames(m.getAlternativeNamesList(). + stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setWebsiteurl(mapStringField(m.getWebsiteurl())) + .setLogourl(mapStringField(m.getLogourl())) + .setEclegalbody(mapStringField(m.getEclegalbody())) + .setEclegalperson(mapStringField(m.getEclegalperson())) + .setEcnonprofit(mapStringField(m.getEcnonprofit())) + .setEcresearchorganization(mapStringField(m.getEcresearchorganization())) + .setEchighereducation(mapStringField(m.getEchighereducation())) + .setEcinternationalorganizationeurinterests(mapStringField(m.getEcinternationalorganizationeurinterests())) + .setEcinternationalorganization(mapStringField(m.getEcinternationalorganization())) + .setEcenterprise(mapStringField(m.getEcenterprise())) + .setEcsmevalidated(mapStringField(m.getEcsmevalidated())) + .setEcnutscode(mapStringField(m.getEcnutscode())) + .setCountry(mapQualifier(m.getCountry())); + } private static Datasource convertDataSource(OafProtos.Oaf oaf) { final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata(); final Datasource datasource = setOaf(new Datasource(), oaf); return setEntity(datasource, oaf) .setAccessinfopackage(m.getAccessinfopackageList() - .stream() - .map(ProtoUtils::mapStringField) - .collect(Collectors.toList())) + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) .setCertificates(mapStringField(m.getCertificates())) .setCitationguidelineurl(mapStringField(m.getCitationguidelineurl())) .setContactemail(mapStringField(m.getContactemail())) @@ -92,13 +109,13 @@ public class ProtoConverter implements Serializable { .setMissionstatementurl(mapStringField(m.getMissionstatementurl())) .setNamespaceprefix(mapStringField(m.getNamespaceprefix())) .setOdcontenttypes(m.getOdcontenttypesList() - .stream() - .map(ProtoUtils::mapStringField) - .collect(Collectors.toList())) + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) .setOdlanguages(m.getOdlanguagesList() - .stream() - .map(ProtoUtils::mapStringField) - .collect(Collectors.toList())) + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) .setOdnumberofitems(mapStringField(m.getOdnumberofitems())) .setOdnumberofitemsdate(mapStringField(m.getOdnumberofitemsdate())) .setOdpolicies(mapStringField(m.getOdpolicies())) @@ -106,17 +123,17 @@ public class ProtoConverter implements Serializable { .setOpenairecompatibility(mapQualifier(m.getOpenairecompatibility())) .setPidsystems(mapStringField(m.getPidsystems())) .setPolicies(m.getPoliciesList() - .stream() - .map(ProtoUtils::mapKV) - .collect(Collectors.toList())) + .stream() + .map(ProtoUtils::mapKV) + .collect(Collectors.toList())) .setQualitymanagementkind(mapStringField(m.getQualitymanagementkind())) .setReleaseenddate(mapStringField(m.getReleaseenddate())) .setServiceprovider(mapBoolField(m.getServiceprovider())) .setReleasestartdate(mapStringField(m.getReleasestartdate())) .setSubjects(m.getSubjectsList() - .stream() - .map(ProtoUtils::mapStructuredProperty) - .collect(Collectors.toList())) + .stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList())) .setVersioning(mapBoolField(m.getVersioning())) .setWebsiteurl(mapStringField(m.getWebsiteurl())) .setJournal(mapJournal(m.getJournal())); @@ -145,13 +162,13 @@ public class ProtoConverter implements Serializable { .setTotalcost(m.getTotalcost()) .setKeywords(mapStringField(m.getKeywords())) .setSubjects(m.getSubjectsList().stream() - .map(sp -> mapStructuredProperty(sp)) - .collect(Collectors.toList())) + .map(sp -> mapStructuredProperty(sp)) + .collect(Collectors.toList())) .setTitle(mapStringField(m.getTitle())) .setWebsiteurl(mapStringField(m.getWebsiteurl())) .setFundingtree(m.getFundingtreeList().stream() - .map(f -> mapStringField(f)) - .collect(Collectors.toList())) + .map(f -> mapStringField(f)) + .collect(Collectors.toList())) .setJsonextrainfo(mapStringField(m.getJsonextrainfo())) .setSummary(mapStringField(m.getSummary())) .setOptional1(mapStringField(m.getOptional1())) @@ -186,69 +203,11 @@ public class ProtoConverter implements Serializable { ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata(); Publication publication = setOaf(new Publication(), oaf); - return setEntity(publication, oaf) - .setJournal(mapJournal(m.getJournal())) - .setRefereed(mapStringField(m.getRefereed())); - - //setting Entity fields - final OafProtos.OafEntity entity = oaf.getEntity(); - - result.setAuthor(null); - - result.setContext(null); - - result.setContributor(null); - - result.setCountry(null); - - result.setCoverage(null); - - result.setDateofacceptance(result.getDateofacceptance()); - - result.setDescription(entity.getResult().getMetadata().getDescriptionList() - .stream() - .map(ProtoUtils::mapStringField) - .collect(Collectors.toList())); - - result.setEmbargoenddate(null); - - result.setExternalReference(null); - - result.setFormat(entity.getResult().getMetadata().getFormatList() - .stream() - .map(ProtoUtils::mapStringField) - .collect(Collectors.toList())); - - result.setFulltext(null); + setEntity(publication, oaf); + return setResult(publication, oaf) + .setJournal(mapJournal(m.getJournal())); - result.setInstance(entity.getResult().getInstanceList()); - - result.setLanguage(ProtoUtils.mapQualifier(entity.getResult().getMetadata().getLanguage())); - - result.setOaiprovenance(null); - - result.setPublisher(ProtoUtils.mapStringField(entity.getResult().getMetadata().getPublisher())); - - result.setRelevantdate(null); - - result.setResourcetype(null); - - result.setResulttype(null); - - result.setSource(entity.getResult().getMetadata().getSourceList() - .stream() - .map(ProtoUtils::mapStringField) - .collect(Collectors.toList())); - - result.setSubject(null); - - result.setTitle(entity.getResult().getMetadata().getTitleList() - .stream() - .map(ProtoUtils::mapStructuredProperty) - .collect(Collectors.toList())); - - return result; } private static Dataset createDataset(OafProtos.Oaf oaf) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java index 7ea84316a..0c1c0d095 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.graph; import com.googlecode.protobuf.format.JsonFormat; import eu.dnetlib.data.proto.FieldTypeProtos; import eu.dnetlib.data.proto.OafProtos; +import eu.dnetlib.data.proto.ResultProtos; import eu.dnetlib.dhp.schema.oaf.*; import java.util.stream.Collectors; @@ -10,7 +11,7 @@ import java.util.stream.Collectors; public class ProtoUtils { public static OafProtos.Oaf parse(String json) throws JsonFormat.ParseException { - final OafProtos.Oaf.Builder builder = OafProtos.Oaf.newBuilder(); + final OafProtos.Oaf.Builder builder = OafProtos.Oaf.newBuilder(); JsonFormat.merge(json, builder); return builder.build(); } @@ -24,24 +25,98 @@ public class ProtoUtils { //setting Entity fields final OafProtos.OafEntity e = oaf.getEntity(); entity - .setId(e.getId()) - .setOriginalId(e.getOriginalIdList()) - .setCollectedfrom(e.getCollectedfromList() - .stream() - .map(ProtoUtils::mapKV) - .collect(Collectors.toList())) - .setPid(e.getPidList().stream() - .map(ProtoUtils::mapStructuredProperty) - .collect(Collectors.toList())) - .setDateofcollection(entity.getDateofcollection()) - .setDateoftransformation(entity.getDateoftransformation()) - .setExtraInfo(e.getExtraInfoList() - .stream() - .map(ProtoUtils::mapExtraInfo) - .collect(Collectors.toList())); + .setId(e.getId()) + .setOriginalId(e.getOriginalIdList()) + .setCollectedfrom(e.getCollectedfromList() + .stream() + .map(ProtoUtils::mapKV) + .collect(Collectors.toList())) + .setPid(e.getPidList().stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList())) + .setDateofcollection(entity.getDateofcollection()) + .setDateoftransformation(entity.getDateoftransformation()) + .setExtraInfo(e.getExtraInfoList() + .stream() + .map(ProtoUtils::mapExtraInfo) + .collect(Collectors.toList())); return entity; } + public static T setResult(T entity, OafProtos.Oaf oaf) { + //setting Entity fields + final ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata(); + entity + .setAuthor(m.getAuthorList() + .stream() + .map(ProtoUtils::mapAuthor) + .collect(Collectors.toList())) + .setResulttype(mapQualifier(m.getResulttype())) + .setLanguage(ProtoUtils.mapQualifier(m.getLanguage())) + .setCountry(m.getCountryList() + .stream() + .map(ProtoUtils::mapQualifier) + .collect(Collectors.toList())) + .setSubject(m.getSubjectList() + .stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList())) + .setTitle(m.getTitleList() + .stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList())) + .setRelevantdate(m.getRelevantdateList() + .stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList())) + .setDescription(m.getDescriptionList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setDateofacceptance(ProtoUtils.mapStringField(m.getDateofacceptance())) + .setPublisher(ProtoUtils.mapStringField(m.getPublisher())) + .setEmbargoenddate(ProtoUtils.mapStringField(m.getEmbargoenddate())) + .setSource(m.getSourceList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setFulltext(m.getFulltextList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setFormat(m.getFormatList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setContributor(m.getContributorList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setResourcetype(ProtoUtils.mapQualifier(m.getResourcetype())) + .setCoverage(m.getCoverageList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())) + .setRefereed(mapStringField(m.getRefereed())) + .setContext(m.getContextList() + .stream() + .map(ProtoUtils::mapContext) + .collect(Collectors.toList())); + + return entity; + } + + private static Context mapContext(ResultProtos.Result.Context context) { + + return new Context() + .setId(context.getId()) + .setDataInfo(context.getDataInfoList() + .stream() + .map(ProtoUtils::mapDataInfo) + .collect(Collectors.toList())); + } + + public static KeyValue mapKV(FieldTypeProtos.KeyValue kv) { return new KeyValue() .setKey(kv.getKey()) @@ -64,7 +139,7 @@ public class ProtoUtils { .setClassname(q.getClassname()) .setSchemeid(q.getSchemeid()) .setSchemename(q.getSchemename()); - //.setDataInfo(q.hasDataInfo() ? mapDataInfo(q.getDataInfo()) : null); + //.setDataInfo(q.hasDataInfo() ? mapDataInfo(q.getDataInfo()) : null); } public static StructuredProperty mapStructuredProperty(FieldTypeProtos.StructuredProperty sp) { @@ -134,4 +209,21 @@ public class ProtoUtils { .setDataInfo(mapDataInfo(j.getDataInfo())); } + public static Author mapAuthor(FieldTypeProtos.Author author) { + return new Author() + .setFullname(author.getFullname()) + .setName(author.getName()) + .setSurname(author.getSurname()) + .setRank(author.getRank()) + .setPid(author.getPidList() + .stream() + .map(ProtoUtils::mapKV) + .collect(Collectors.toList())) + .setAffiliation(author.getAffiliationList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList())); + + } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java index fbccccacc..cdce67628 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.graph; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.Organization; import eu.dnetlib.dhp.schema.oaf.Publication; import org.apache.hadoop.io.Text; import org.apache.spark.api.java.JavaRDD; @@ -39,14 +40,15 @@ public class SparkGraphImporterJob { final JavaRDD> inputRDD = sc.sequenceFile(path, Text.class, Text.class) .map(item -> new Tuple2<>(item._1.toString(), item._2.toString())); - final JavaRDD datasources = inputRDD + + final JavaRDD organization = inputRDD .filter(s -> s._1().split("@")[2].equalsIgnoreCase("body")) .map(Tuple2::_2) .map(ProtoConverter::convert) - .filter(s-> s instanceof Datasource) - .map(s->(Datasource)s); - final Encoder encoder = Encoders.bean(Datasource.class); - final Dataset mdstore = spark.createDataset(datasources.rdd(), encoder); + .filter(s-> s instanceof Organization) + .map(s->(Organization)s); + final Encoder encoder = Encoders.bean(Organization.class); + final Dataset mdstore = spark.createDataset(organization.rdd(), encoder); System.out.println(mdstore.count()); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java index 9e8a59ba0..47b6f373b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java @@ -43,7 +43,8 @@ public class ProtoConverterTest { assertTrue(result instanceof Publication); Publication p = (Publication) result; - System.out.println(p.getId()); + ObjectMapper mapper = new ObjectMapper(); + System.out.println(mapper.writeValueAsString(result)); }