From c929c1dface2381845f526574103b20588bdfd5c Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 25 Oct 2019 09:25:36 +0200 Subject: [PATCH] more proto 2 graph model mappings --- .../eu/dnetlib/dhp/schema/oaf/Datasource.java | 113 ++++++++++++------ .../eu/dnetlib/dhp/schema/oaf/Journal.java | 36 ++++-- .../java/eu/dnetlib/dhp/schema/oaf/Oaf.java | 6 +- .../eu/dnetlib/dhp/schema/oaf/OafEntity.java | 27 +++-- .../eu/dnetlib/dhp/graph/ProtoConverter.java | 107 ++++++++++------- .../java/eu/dnetlib/dhp/graph/ProtoUtils.java | 48 +++++++- 6 files changed, 231 insertions(+), 106 deletions(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java index 354045ba7..33e994d4a 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java @@ -65,7 +65,7 @@ public class Datasource extends OafEntity implements Serializable { private Field databaseaccessrestriction; // {feeRequired, registration, other} - private Field datauploadrestriction9; + private Field datauploadrestriction; private Field versioning; @@ -86,279 +86,314 @@ public class Datasource extends OafEntity implements Serializable { return datasourcetype; } - public void setDatasourcetype(Qualifier datasourcetype) { + public Datasource setDatasourcetype(Qualifier datasourcetype) { this.datasourcetype = datasourcetype; + return this; } public Qualifier getOpenairecompatibility() { return openairecompatibility; } - public void setOpenairecompatibility(Qualifier openairecompatibility) { + public Datasource setOpenairecompatibility(Qualifier openairecompatibility) { this.openairecompatibility = openairecompatibility; + return this; } public Field getOfficialname() { return officialname; } - public void setOfficialname(Field officialname) { + public Datasource setOfficialname(Field officialname) { this.officialname = officialname; + return this; } public Field getEnglishname() { return englishname; } - public void setEnglishname(Field englishname) { + public Datasource setEnglishname(Field englishname) { this.englishname = englishname; + return this; } public Field getWebsiteurl() { return websiteurl; } - public void setWebsiteurl(Field websiteurl) { + public Datasource setWebsiteurl(Field websiteurl) { this.websiteurl = websiteurl; + return this; } public Field getLogourl() { return logourl; } - public void setLogourl(Field logourl) { + public Datasource setLogourl(Field logourl) { this.logourl = logourl; + return this; } public Field getContactemail() { return contactemail; } - public void setContactemail(Field contactemail) { + public Datasource setContactemail(Field contactemail) { this.contactemail = contactemail; + return this; } public Field getNamespaceprefix() { return namespaceprefix; } - public void setNamespaceprefix(Field namespaceprefix) { + public Datasource setNamespaceprefix(Field namespaceprefix) { this.namespaceprefix = namespaceprefix; + return this; } public Field getLatitude() { return latitude; } - public void setLatitude(Field latitude) { + public Datasource setLatitude(Field latitude) { this.latitude = latitude; + return this; } public Field getLongitude() { return longitude; } - public void setLongitude(Field longitude) { + public Datasource setLongitude(Field longitude) { this.longitude = longitude; + return this; } public Field getDateofvalidation() { return dateofvalidation; } - public void setDateofvalidation(Field dateofvalidation) { + public Datasource setDateofvalidation(Field dateofvalidation) { this.dateofvalidation = dateofvalidation; + return this; } public Field getDescription() { return description; } - public void setDescription(Field description) { + public Datasource setDescription(Field description) { this.description = description; + return this; } public List getSubjects() { return subjects; } - public void setSubjects(List subjects) { + public Datasource setSubjects(List subjects) { this.subjects = subjects; + return this; } public Field getOdnumberofitems() { return odnumberofitems; } - public void setOdnumberofitems(Field odnumberofitems) { + public Datasource setOdnumberofitems(Field odnumberofitems) { this.odnumberofitems = odnumberofitems; + return this; } public Field getOdnumberofitemsdate() { return odnumberofitemsdate; } - public void setOdnumberofitemsdate(Field odnumberofitemsdate) { + public Datasource setOdnumberofitemsdate(Field odnumberofitemsdate) { this.odnumberofitemsdate = odnumberofitemsdate; + return this; } public Field getOdpolicies() { return odpolicies; } - public void setOdpolicies(Field odpolicies) { + public Datasource setOdpolicies(Field odpolicies) { this.odpolicies = odpolicies; + return this; } public List> getOdlanguages() { return odlanguages; } - public void setOdlanguages(List> odlanguages) { + public Datasource setOdlanguages(List> odlanguages) { this.odlanguages = odlanguages; + return this; } public List> getOdcontenttypes() { return odcontenttypes; } - public void setOdcontenttypes(List> odcontenttypes) { + public Datasource setOdcontenttypes(List> odcontenttypes) { this.odcontenttypes = odcontenttypes; + return this; } public List> getAccessinfopackage() { return accessinfopackage; } - public void setAccessinfopackage(List> accessinfopackage) { + public Datasource setAccessinfopackage(List> accessinfopackage) { this.accessinfopackage = accessinfopackage; + return this; } public Field getReleasestartdate() { return releasestartdate; } - public void setReleasestartdate(Field releasestartdate) { + public Datasource setReleasestartdate(Field releasestartdate) { this.releasestartdate = releasestartdate; + return this; } public Field getReleaseenddate() { return releaseenddate; } - public void setReleaseenddate(Field releaseenddate) { + public Datasource setReleaseenddate(Field releaseenddate) { this.releaseenddate = releaseenddate; + return this; } public Field getMissionstatementurl() { return missionstatementurl; } - public void setMissionstatementurl(Field missionstatementurl) { + public Datasource setMissionstatementurl(Field missionstatementurl) { this.missionstatementurl = missionstatementurl; + return this; } public Field getDataprovider() { return dataprovider; } - public void setDataprovider(Field dataprovider) { + public Datasource setDataprovider(Field dataprovider) { this.dataprovider = dataprovider; + return this; } public Field getServiceprovider() { return serviceprovider; } - public void setServiceprovider(Field serviceprovider) { + public Datasource setServiceprovider(Field serviceprovider) { this.serviceprovider = serviceprovider; + return this; } public Field getDatabaseaccesstype() { return databaseaccesstype; } - public void setDatabaseaccesstype(Field databaseaccesstype) { + public Datasource setDatabaseaccesstype(Field databaseaccesstype) { this.databaseaccesstype = databaseaccesstype; + return this; } public Field getDatauploadtype() { return datauploadtype; } - public void setDatauploadtype(Field datauploadtype) { + public Datasource setDatauploadtype(Field datauploadtype) { this.datauploadtype = datauploadtype; + return this; } public Field getDatabaseaccessrestriction() { return databaseaccessrestriction; } - public void setDatabaseaccessrestriction(Field databaseaccessrestriction) { + public Datasource setDatabaseaccessrestriction(Field databaseaccessrestriction) { this.databaseaccessrestriction = databaseaccessrestriction; + return this; } - public Field getDatauploadrestriction9() { - return datauploadrestriction9; + public Field getDatauploadrestriction() { + return datauploadrestriction; } - public void setDatauploadrestriction9(Field datauploadrestriction9) { - this.datauploadrestriction9 = datauploadrestriction9; + public Datasource setDatauploadrestriction(Field datauploadrestriction) { + this.datauploadrestriction = datauploadrestriction; + return this; } public Field getVersioning() { return versioning; } - public void setVersioning(Field versioning) { + public Datasource setVersioning(Field versioning) { this.versioning = versioning; + return this; } public Field getCitationguidelineurl() { return citationguidelineurl; } - public void setCitationguidelineurl(Field citationguidelineurl) { + public Datasource setCitationguidelineurl(Field citationguidelineurl) { this.citationguidelineurl = citationguidelineurl; + return this; } public Field getQualitymanagementkind() { return qualitymanagementkind; } - public void setQualitymanagementkind(Field qualitymanagementkind) { + public Datasource setQualitymanagementkind(Field qualitymanagementkind) { this.qualitymanagementkind = qualitymanagementkind; + return this; } public Field getPidsystems() { return pidsystems; } - public void setPidsystems(Field pidsystems) { + public Datasource setPidsystems(Field pidsystems) { this.pidsystems = pidsystems; + return this; } public Field getCertificates() { return certificates; } - public void setCertificates(Field certificates) { + public Datasource setCertificates(Field certificates) { this.certificates = certificates; + return this; } public List getPolicies() { return policies; } - public void setPolicies(List policies) { + public Datasource setPolicies(List policies) { this.policies = policies; + return this; } public Journal getJournal() { return journal; } - public void setJournal(Journal journal) { + public Datasource setJournal(Journal journal) { this.journal = journal; + return this; } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Journal.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Journal.java index 06e5ca7dc..b0a53e5c4 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Journal.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Journal.java @@ -32,95 +32,107 @@ public class Journal implements Serializable { return name; } - public void setName(String name) { + public Journal setName(String name) { this.name = name; + return this; } public String getIssnPrinted() { return issnPrinted; } - public void setIssnPrinted(String issnPrinted) { + public Journal setIssnPrinted(String issnPrinted) { this.issnPrinted = issnPrinted; + return this; } public String getIssnOnline() { return issnOnline; } - public void setIssnOnline(String issnOnline) { + public Journal setIssnOnline(String issnOnline) { this.issnOnline = issnOnline; + return this; } public String getIssnLinking() { return issnLinking; } - public void setIssnLinking(String issnLinking) { + public Journal setIssnLinking(String issnLinking) { this.issnLinking = issnLinking; + return this; } public String getEp() { return ep; } - public void setEp(String ep) { + public Journal setEp(String ep) { this.ep = ep; + return this; } public String getIss() { return iss; } - public void setIss(String iss) { + public Journal setIss(String iss) { this.iss = iss; + return this; } public String getSp() { return sp; } - public void setSp(String sp) { + public Journal setSp(String sp) { this.sp = sp; + return this; } public String getVol() { return vol; } - public void setVol(String vol) { + public Journal setVol(String vol) { this.vol = vol; + return this; } public String getEdition() { return edition; } - public void setEdition(String edition) { + public Journal setEdition(String edition) { this.edition = edition; + return this; } public String getConferenceplace() { return conferenceplace; } - public void setConferenceplace(String conferenceplace) { + public Journal setConferenceplace(String conferenceplace) { this.conferenceplace = conferenceplace; + return this; } public String getConferencedate() { return conferencedate; } - public void setConferencedate(String conferencedate) { + public Journal setConferencedate(String conferencedate) { this.conferencedate = conferencedate; + return this; } public DataInfo getDataInfo() { return dataInfo; } - public void setDataInfo(DataInfo dataInfo) { + public Journal setDataInfo(DataInfo dataInfo) { this.dataInfo = dataInfo; + return this; } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java index 352ebbc6e..2924deaba 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java @@ -12,15 +12,17 @@ public abstract class Oaf implements Serializable { return dataInfo; } - public void setDataInfo(DataInfo dataInfo) { + public Oaf setDataInfo(DataInfo dataInfo) { this.dataInfo = dataInfo; + return this; } public Long getLastupdatetimestamp() { return lastupdatetimestamp; } - public void setLastupdatetimestamp(Long lastupdatetimestamp) { + public Oaf setLastupdatetimestamp(Long lastupdatetimestamp) { this.lastupdatetimestamp = lastupdatetimestamp; + return this; } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java index 98f435fab..ef9d330cd 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java @@ -29,71 +29,80 @@ public abstract class OafEntity extends Oaf implements Serializable { return id; } - public void setId(String id) { + public OafEntity setId(String id) { this.id = id; + return this; } public List getOriginalId() { return originalId; } - public void setOriginalId(List originalId) { + public OafEntity setOriginalId(List originalId) { this.originalId = originalId; + return this; } public List getCollectedfrom() { return collectedfrom; } - public void setCollectedfrom(List collectedfrom) { + public OafEntity setCollectedfrom(List collectedfrom) { this.collectedfrom = collectedfrom; + return this; } public List getPid() { return pid; } - public void setPid(List pid) { + public OafEntity setPid(List pid) { this.pid = pid; + return this; } public String getDateofcollection() { return dateofcollection; } - public void setDateofcollection(String dateofcollection) { + public OafEntity setDateofcollection(String dateofcollection) { this.dateofcollection = dateofcollection; + return this; } public String getDateoftransformation() { return dateoftransformation; } - public void setDateoftransformation(String dateoftransformation) { + public OafEntity setDateoftransformation(String dateoftransformation) { this.dateoftransformation = dateoftransformation; + return this; } public List getChildren() { return children; } - public void setChildren(List children) { + public OafEntity setChildren(List children) { this.children = children; + return this; } public List getExtraInfo() { return extraInfo; } - public void setExtraInfo(List extraInfo) { + public OafEntity setExtraInfo(List extraInfo) { this.extraInfo = extraInfo; + return this; } public OAIProvenance getOaiprovenance() { return oaiprovenance; } - public void setOaiprovenance(OAIProvenance oaiprovenance) { + public OafEntity setOaiprovenance(OAIProvenance oaiprovenance) { this.oaiprovenance = oaiprovenance; + return this; } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java index b801aaae7..0e09ac172 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java @@ -1,5 +1,6 @@ package eu.dnetlib.dhp.graph; +import eu.dnetlib.data.proto.DatasourceProtos; import eu.dnetlib.data.proto.KindProtos; import eu.dnetlib.data.proto.OafProtos; import eu.dnetlib.data.proto.ProjectProtos; @@ -60,54 +61,78 @@ public class ProtoConverter implements Serializable { } private static Organization convertOrganization(OafProtos.Oaf oaf) { - return new Organization(); + final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata(); + final Organization org = setOaf(new Organization(), oaf); + return setEntity(org, oaf); + //TODO set org fields } - private static Datasource convertDataSource(OafProtos.Oaf oaf) { - final Datasource result = new Datasource(); - - - //Set Oaf Fields - result.setDataInfo(ProtoUtils.mapDataInfo(oaf.getDataInfo())); - - result.setLastupdatetimestamp(oaf.getLastupdatetimestamp()); - - //setting Entity fields - final OafProtos.OafEntity entity = oaf.getEntity(); - - result.setId(entity.getId()); - - result.setOriginalId(entity.getOriginalIdList()); - - result.setCollectedfrom(entity.getCollectedfromList() - .stream() - .map(ProtoUtils::mapKV) - .collect(Collectors.toList())); - - result.setPid(entity.getPidList() - .stream() - .map(ProtoUtils::mapStructuredProperty) - .collect(Collectors.toList())); - - result.setDateofcollection(entity.getDateofcollection()); - - result.setDateoftransformation(entity.getDateoftransformation()); - - result.setExtraInfo(entity.getExtraInfoList() - .stream() - .map(ProtoUtils::mapExtraInfo) - .collect(Collectors.toList())); - - return result; + final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata(); + final Datasource datasource = setOaf(new Datasource(), oaf); + return setEntity(datasource, oaf) + .setAccessinfopackage(m.getAccessinfopackageCount() > 0 ? + m.getAccessinfopackageList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList()) : null) + .setCertificates(mapStringField(m.getCertificates())) + .setCitationguidelineurl(mapStringField(m.getCitationguidelineurl())) + .setContactemail(mapStringField(m.getContactemail())) + .setDatabaseaccessrestriction(mapStringField(m.getDatabaseaccessrestriction())) + .setDatabaseaccesstype(mapStringField(m.getDatabaseaccesstype())) + .setDataprovider(mapBoolField(m.getDataprovider())) + .setDatasourcetype(mapQualifier(m.getDatasourcetype())) + .setDatauploadrestriction(mapStringField(m.getDatauploadrestriction())) + .setCitationguidelineurl(mapStringField(m.getCitationguidelineurl())) + .setDatauploadtype(mapStringField(m.getDatauploadtype())) + .setDateofvalidation(mapStringField(m.getDateofvalidation())) + .setDescription(mapStringField(m.getDescription())) + .setEnglishname(mapStringField(m.getEnglishname())) + .setLatitude(mapStringField(m.getLatitude())) + .setLongitude(mapStringField(m.getLongitude())) + .setLogourl(mapStringField(m.getLogourl())) + .setMissionstatementurl(mapStringField(m.getMissionstatementurl())) + .setNamespaceprefix(mapStringField(m.getNamespaceprefix())) + .setOdcontenttypes(m.getOdcontenttypesCount() > 0 ? + m.getOdcontenttypesList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList()) : null) + .setOdlanguages(m.getOdlanguagesCount() > 0 ? + m.getOdlanguagesList() + .stream() + .map(ProtoUtils::mapStringField) + .collect(Collectors.toList()) : null) + .setOdnumberofitems(mapStringField(m.getOdnumberofitems())) + .setOdnumberofitemsdate(mapStringField(m.getOdnumberofitemsdate())) + .setOdpolicies(mapStringField(m.getOdpolicies())) + .setOfficialname(mapStringField(m.getOfficialname())) + .setOpenairecompatibility(mapQualifier(m.getOpenairecompatibility())) + .setPidsystems(mapStringField(m.getPidsystems())) + .setPolicies(m.getPoliciesCount() > 0 ? + m.getPoliciesList() + .stream() + .map(ProtoUtils::mapKV) + .collect(Collectors.toList()) : null) + .setQualitymanagementkind(mapStringField(m.getQualitymanagementkind())) + .setReleaseenddate(mapStringField(m.getReleaseenddate())) + .setServiceprovider(mapBoolField(m.getServiceprovider())) + .setReleasestartdate(mapStringField(m.getReleasestartdate())) + .setSubjects(m.getSubjectsCount() > 0 ? + m.getSubjectsList() + .stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList()) : null) + .setVersioning(mapBoolField(m.getVersioning())) + .setWebsiteurl(mapStringField(m.getWebsiteurl())) + .setJournal(mapJournal(m.getJournal())); } private static Project convertProject(OafProtos.Oaf oaf) { final ProjectProtos.Project.Metadata m = oaf.getEntity().getProject().getMetadata(); - final Project project = new Project(); - project.setDataInfo(mapDataInfo(oaf.getDataInfo())); - project.setLastupdatetimestamp(oaf.getLastupdatetimestamp()); - return project + final Project project = setOaf(new Project(), oaf); + return setEntity(project, oaf) .setAcronym(mapStringField(m.getAcronym())) .setCallidentifier(mapStringField(m.getCallidentifier())) .setCode(mapStringField(m.getCode())) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java index 97b9c3297..c0a4ae74e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoUtils.java @@ -5,6 +5,8 @@ import eu.dnetlib.data.proto.FieldTypeProtos; import eu.dnetlib.data.proto.OafProtos; import eu.dnetlib.dhp.schema.oaf.*; +import java.util.stream.Collectors; + public class ProtoUtils { public static OafProtos.Oaf parse(String json) throws JsonFormat.ParseException { @@ -13,6 +15,33 @@ public class ProtoUtils { return builder.build(); } + public static T setOaf(T oaf, OafProtos.Oaf o) { + oaf.setDataInfo(mapDataInfo(o.getDataInfo())).setLastupdatetimestamp(o.getLastupdatetimestamp()); + return oaf; + } + + public static T setEntity(T entity, OafProtos.Oaf oaf) { + //setting Entity fields + final OafProtos.OafEntity e = oaf.getEntity(); + entity + .setId(e.getId()) + .setOriginalId(e.getOriginalIdList()) + .setCollectedfrom(e.getCollectedfromList() + .stream() + .map(ProtoUtils::mapKV) + .collect(Collectors.toList())) + .setPid(e.getPidList().stream() + .map(ProtoUtils::mapStructuredProperty) + .collect(Collectors.toList())) + .setDateofcollection(entity.getDateofcollection()) + .setDateoftransformation(entity.getDateoftransformation()) + .setExtraInfo(e.getExtraInfoList() + .stream() + .map(ProtoUtils::mapExtraInfo) + .collect(Collectors.toList())); + return entity; + } + public static KeyValue mapKV(FieldTypeProtos.KeyValue kv) { return new KeyValue() .setKey(kv.getKey()) @@ -54,12 +83,10 @@ public class ProtoUtils { .setValue(extraInfo.getValue()); } - public static OAIProvenance mapOAIProvenance(FieldTypeProtos.OAIProvenance oaiProvenance) { return new OAIProvenance().setOriginDescription(mapOriginalDescription(oaiProvenance.getOriginDescription())); } - public static OriginDescription mapOriginalDescription(FieldTypeProtos.OAIProvenance.OriginDescription originDescription) { final OriginDescription originDescriptionResult = new OriginDescription() .setHarvestDate(originDescription.getHarvestDate()) @@ -71,7 +98,6 @@ public class ProtoUtils { if (originDescription.hasOriginDescription()) originDescriptionResult.setOriginDescription(mapOriginalDescription(originDescription.getOriginDescription())); return originDescriptionResult; - } public static Field mapStringField(FieldTypeProtos.StringField s) { @@ -92,4 +118,20 @@ public class ProtoUtils { .setDataInfo(b.hasDataInfo() ? mapDataInfo(b.getDataInfo()) : null); } + public static Journal mapJournal(FieldTypeProtos.Journal j) { + return new Journal() + .setConferencedate(j.getConferencedate()) + .setConferenceplace(j.getConferenceplace()) + .setEdition(j.getEdition()) + .setEp(j.getEp()) + .setIss(j.getIss()) + .setIssnLinking(j.getIssnLinking()) + .setIssnOnline(j.getIssnOnline()) + .setIssnPrinted(j.getIssnPrinted()) + .setName(j.getName()) + .setSp(j.getSp()) + .setVol(j.getVol()) + .setDataInfo(mapDataInfo(j.getDataInfo())); + } + }