From d525785497cf93625769d44bc0865714c92d8db7 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Jan 2021 15:36:38 +0100 Subject: [PATCH] [#6282 open access status in the Graph] Result.Instance.accessRight defined with dedicated data type that includes the open access color. --- .../dhp/schema/oaf/CleaningFunctions.java | 40 +++++----------- .../dhp/schema/oaf/OafMapperUtils.java | 23 +++++++++ ...arator.java => AccessRightComparator.java} | 5 +- .../dhp/schema/common/ModelConstants.java | 3 +- .../dnetlib/dhp/schema/oaf/AccessRight.java | 48 +++++++++++++++++++ .../eu/dnetlib/dhp/schema/oaf/Instance.java | 6 +-- .../eu/dnetlib/dhp/schema/oaf/OAStatus.java | 13 +++++ .../eu/dnetlib/dhp/schema/oaf/Result.java | 5 +- .../dhp/schema/scholexplorer/OafUtils.scala | 11 ++++- .../migration/ProtoConverter.java | 13 ++++- .../doiboost/DoiBoostMappingUtil.scala | 36 +++++--------- .../doiboost/crossref/Crossref2Oaf.scala | 8 ++-- .../orcidnodoi/oaf/PublicationToOaf.java | 44 ++++++++++++----- .../dhp/oa/graph/clean/CleaningRuleMap.java | 2 + .../raw/AbstractMdRecordToOafMapper.java | 24 +++++++++- .../dhp/oa/graph/raw/OafToOafMapper.java | 2 +- .../dhp/oa/graph/raw/OdfToOafMapper.java | 27 ++++++----- .../java/eu/dnetlib/dhp/export/DLIToOAF.scala | 6 +-- 18 files changed, 217 insertions(+), 99 deletions(-) rename dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/{LicenseComparator.java => AccessRightComparator.java} (87%) create mode 100644 dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/AccessRight.java create mode 100644 dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAStatus.java diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/CleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/CleaningFunctions.java index da4ed63a9..d21a58395 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/CleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/CleaningFunctions.java @@ -84,7 +84,7 @@ public class CleaningFunctions { } else if (value instanceof Organization) { Organization o = (Organization) value; if (Objects.isNull(o.getCountry()) || StringUtils.isBlank(o.getCountry().getClassid())) { - o.setCountry(qualifier("UNKNOWN", "Unknown", ModelConstants.DNET_COUNTRY_TYPE)); + o.setCountry(ModelConstants.UNKNOWN_COUNTRY); } } else if (value instanceof Relation) { // nothing to clean here @@ -152,12 +152,14 @@ public class CleaningFunctions { if (Objects.isNull(r.getResourcetype()) || StringUtils.isBlank(r.getResourcetype().getClassid())) { r .setResourcetype( - qualifier("UNKNOWN", "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE)); + qualifier(ModelConstants.UNKNOWN, "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE)); } if (Objects.nonNull(r.getInstance())) { for (Instance i : r.getInstance()) { if (Objects.isNull(i.getAccessright()) || StringUtils.isBlank(i.getAccessright().getClassid())) { - i.setAccessright(qualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES)); + i + .setAccessright( + accessRight(ModelConstants.UNKNOWN, "not available", ModelConstants.DNET_ACCESS_MODES)); } if (Objects.isNull(i.getHostedby()) || StringUtils.isBlank(i.getHostedby().getKey())) { i.setHostedby(ModelConstants.UNKNOWN_REPOSITORY); @@ -203,6 +205,7 @@ public class CleaningFunctions { p.setValue(p.getValue().trim().replaceAll(ORCID_PREFIX_REGEX, "")); return p; }) + .filter(p -> StringUtils.isNotBlank(p.getValue())) .collect( Collectors .toMap( @@ -248,35 +251,18 @@ public class CleaningFunctions { } } + private static AccessRight accessRight(String classid, String classname, String scheme) { + return OafMapperUtils + .accessRight( + classid, classname, scheme, scheme); + } + private static Qualifier qualifier(String classid, String classname, String scheme) { return OafMapperUtils .qualifier( classid, classname, scheme, scheme); } - /** - * Utility method that filter PID values on a per-type basis. - * @param pid the PID whose value will be checked. - * @return true the PID containing the normalised value. - */ - private static boolean filterPid(StructuredProperty pid) { - String value = Optional - .ofNullable(pid.getValue()) - .map(s -> StringUtils.replaceAll(s, "\\s", "")) - .orElse(""); - if (StringUtils.isBlank(value)) { - return false; - } - switch (pid.getQualifier().getClassid()) { - - // TODO add cleaning for more PID types as needed - case "doi": - return value.startsWith("10."); - default: - return true; - } - } - /** * Utility method that normalises PID values on a per-type basis. * @param pid the PID whose value will be normalised. @@ -291,7 +277,7 @@ public class CleaningFunctions { // TODO add cleaning for more PID types as needed case "doi": - pid.setValue(value.toLowerCase().replaceAll(DOI_PREFIX_REGEX, "10.")); + pid.setValue(value.toLowerCase().replaceAll(DOI_URL_PREFIX_REGEX, "")); break; } return pid; diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java index 5d6e7565d..66f843056 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java @@ -105,6 +105,29 @@ public class OafMapperUtils { return qualifier("UNKNOWN", "Unknown", schemeid, schemename); } + public static AccessRight accessRight( + final String classid, + final String classname, + final String schemeid, + final String schemename) { + return accessRight(classid, classname, schemeid, schemename, null); + } + + public static AccessRight accessRight( + final String classid, + final String classname, + final String schemeid, + final String schemename, + final OAStatus oaStatus) { + final AccessRight accessRight = new AccessRight(); + accessRight.setClassid(classid); + accessRight.setClassname(classname); + accessRight.setSchemeid(schemeid); + accessRight.setSchemename(schemename); + accessRight.setOaStatus(oaStatus); + return accessRight; + } + public static Qualifier qualifier( final String classid, final String classname, diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/LicenseComparator.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/AccessRightComparator.java similarity index 87% rename from dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/LicenseComparator.java rename to dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/AccessRightComparator.java index db523ad1a..6116bc479 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/LicenseComparator.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/AccessRightComparator.java @@ -3,12 +3,13 @@ package eu.dnetlib.dhp.schema.common; import java.util.Comparator; +import eu.dnetlib.dhp.schema.oaf.AccessRight; import eu.dnetlib.dhp.schema.oaf.Qualifier; -public class LicenseComparator implements Comparator { +public class AccessRightComparator implements Comparator { @Override - public int compare(Qualifier left, Qualifier right) { + public int compare(T left, T right) { if (left == null && right == null) return 0; diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index 10b2c7418..143340b07 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -20,7 +20,8 @@ public class ModelConstants { public static final String DNET_ACCESS_MODES = "dnet:access_modes"; public static final String DNET_LANGUAGES = "dnet:languages"; public static final String DNET_PID_TYPES = "dnet:pid_types"; - public static final String DNET_DATA_CITE_DATE = "dnet:dataCite_date"; + public static final String DNET_DATACITE_DATE = "dnet:dataCite_date"; + public static final String DNET_DATACITE_TITLE = "dnet:dataCite_title"; public static final String DNET_DATA_CITE_RESOURCE = "dnet:dataCite_resource"; public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"; public static final String DNET_COUNTRY_TYPE = "dnet:countries"; diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/AccessRight.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/AccessRight.java new file mode 100644 index 000000000..ef67f7d4b --- /dev/null +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/AccessRight.java @@ -0,0 +1,48 @@ + +package eu.dnetlib.dhp.schema.oaf; + +import java.util.Optional; + +/** + * This class models the access rights of research products. + */ +public class AccessRight extends Qualifier { + + private OAStatus oaStatus; + + public OAStatus getOaStatus() { + return oaStatus; + } + + public void setOaStatus(OAStatus oaStatus) { + this.oaStatus = oaStatus; + } + + public String toComparableString() { + String s = super.toComparableString(); + return Optional + .ofNullable(getOaStatus()) + .map(x -> s + "::" + x.toString()) + .orElse(s); + } + + @Override + public int hashCode() { + return toComparableString().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + + Qualifier other = (Qualifier) obj; + + return toComparableString().equals(other.toComparableString()); + } + +} diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java index 29d495261..edf424aa8 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java @@ -8,7 +8,7 @@ public class Instance implements Serializable { private Field license; - private Qualifier accessright; + private AccessRight accessright; private Qualifier instancetype; @@ -41,11 +41,11 @@ public class Instance implements Serializable { this.license = license; } - public Qualifier getAccessright() { + public AccessRight getAccessright() { return accessright; } - public void setAccessright(Qualifier accessright) { + public void setAccessright(AccessRight accessright) { this.accessright = accessright; } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAStatus.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAStatus.java new file mode 100644 index 000000000..1d1cbf0aa --- /dev/null +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAStatus.java @@ -0,0 +1,13 @@ + +package eu.dnetlib.dhp.schema.oaf; + +/** + * This Enum models the OpenAccess status, currently including only the values from Unpaywall + * + * https://support.unpaywall.org/support/solutions/articles/44001777288-what-do-the-types-of-oa-status-green-gold-hybrid-and-bronze-mean- + */ +public enum OAStatus { + + gold, green, hybrid, bronze + +} diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index 845c4c982..945ebad50 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -2,12 +2,11 @@ package eu.dnetlib.dhp.schema.oaf; import java.io.Serializable; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; -import eu.dnetlib.dhp.schema.common.LicenseComparator; +import eu.dnetlib.dhp.schema.common.AccessRightComparator; public class Result extends OafEntity implements Serializable { @@ -248,7 +247,7 @@ public class Result extends OafEntity implements Serializable { instance = mergeLists(instance, r.getInstance()); if (r.getBestaccessright() != null - && new LicenseComparator().compare(r.getBestaccessright(), bestaccessright) < 0) + && new AccessRightComparator().compare(r.getBestaccessright(), bestaccessright) < 0) bestaccessright = r.getBestaccessright(); if (r.getResulttype() != null && compareTrust(this, r) < 0) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala index 27eec77fa..371efc3a7 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala @@ -1,6 +1,6 @@ package eu.dnetlib.dhp.schema.scholexplorer -import eu.dnetlib.dhp.schema.oaf.{DataInfo, Field, KeyValue, Qualifier, StructuredProperty} +import eu.dnetlib.dhp.schema.oaf.{AccessRight, DataInfo, Field, KeyValue, Qualifier, StructuredProperty} object OafUtils { @@ -39,6 +39,15 @@ object OafUtils { q } + def createAccessRight(classId: String, className: String, schemeId: String, schemeName: String): AccessRight = { + val accessRight: AccessRight = new AccessRight + accessRight.setClassid(classId) + accessRight.setClassname(className) + accessRight.setSchemeid(schemeId) + accessRight.setSchemename(schemeName) + accessRight + } + def asField[T](value: T): Field[T] = { val tmp = new Field[T] diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java index 8ea877aec..5aeb38bb5 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java @@ -82,7 +82,7 @@ public class ProtoConverter implements Serializable { private static Instance convertInstance(ResultProtos.Result.Instance ri) { final Instance i = new Instance(); - i.setAccessright(mapQualifier(ri.getAccessright())); + i.setAccessright(mapAccessRight(ri.getAccessright())); i.setCollectedfrom(mapKV(ri.getCollectedfrom())); i.setDateofacceptance(mapStringField(ri.getDateofacceptance())); i.setDistributionlocation(ri.getDistributionlocation()); @@ -510,7 +510,7 @@ public class ProtoConverter implements Serializable { .map(i -> i.getAccessright()) .min(new LicenseComparator()); - final Qualifier rights = min.isPresent() ? mapQualifier(min.get()) : new Qualifier(); + final Qualifier rights = min.isPresent() ? mapAccessRight(min.get()) : new Qualifier(); if (StringUtils.isBlank(rights.getClassid())) { rights.setClassid(UNKNOWN); @@ -579,6 +579,15 @@ public class ProtoConverter implements Serializable { return qualifier; } + public static AccessRight mapAccessRight(FieldTypeProtos.Qualifier q) { + final AccessRight accessRight = new AccessRight(); + accessRight.setClassid(q.getClassid()); + accessRight.setClassname(q.getClassname()); + accessRight.setSchemeid(q.getSchemeid()); + accessRight.setSchemename(q.getSchemename()); + return accessRight; + } + public static Country mapQualifierAsCountry(FieldTypeProtos.Qualifier q) { final Country c = new Country(); c.setClassid(q.getClassid()); diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala index 683986de2..efe8453a2 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala @@ -1,10 +1,11 @@ package eu.dnetlib.doiboost import eu.dnetlib.dhp.schema.action.AtomicAction -import eu.dnetlib.dhp.schema.oaf.{DataInfo, Dataset, Field, Instance, KeyValue, Oaf, Organization, Publication, Qualifier, Relation, Result, StructuredProperty} +import eu.dnetlib.dhp.schema.oaf.{AccessRight, DataInfo, Dataset, Field, Instance, KeyValue, Oaf, Organization, Publication, Qualifier, Relation, Result, StructuredProperty} import eu.dnetlib.dhp.utils.DHPUtils import org.apache.commons.lang3.StringUtils import com.fasterxml.jackson.databind.ObjectMapper +import eu.dnetlib.dhp.schema.scholexplorer.OafUtils import org.json4s import org.json4s.DefaultFormats import org.json4s.jackson.JsonMethods.parse @@ -125,13 +126,12 @@ object DoiBoostMappingUtil { } - def getOpenAccessQualifier():Qualifier = { - createQualifier("OPEN","Open Access","dnet:access_modes", "dnet:access_modes") - + def getOpenAccessQualifier():AccessRight = { + OafUtils.createAccessRight("OPEN","Open Access","dnet:access_modes", "dnet:access_modes") } - def getRestrictedQualifier():Qualifier = { - createQualifier("RESTRICTED","Restricted","dnet:access_modes", "dnet:access_modes") + def getRestrictedQualifier():AccessRight = { + OafUtils.createAccessRight("RESTRICTED","Restricted","dnet:access_modes", "dnet:access_modes") } @@ -260,7 +260,7 @@ object DoiBoostMappingUtil { di.setInferred(false) di.setInvisible(false) di.setTrust(trust) - di.setProvenanceaction(createQualifier("sysimport:actionset", "dnet:provenanceActions")) + di.setProvenanceaction(OafUtils.createQualifier("sysimport:actionset", "dnet:provenanceActions")) di } @@ -268,7 +268,7 @@ object DoiBoostMappingUtil { def createSP(value: String, classId: String,className:String, schemeId: String, schemeName:String): StructuredProperty = { val sp = new StructuredProperty - sp.setQualifier(createQualifier(classId,className, schemeId, schemeName)) + sp.setQualifier(OafUtils.createQualifier(classId,className, schemeId, schemeName)) sp.setValue(value) sp @@ -278,7 +278,7 @@ object DoiBoostMappingUtil { def createSP(value: String, classId: String,className:String, schemeId: String, schemeName:String, dataInfo: DataInfo): StructuredProperty = { val sp = new StructuredProperty - sp.setQualifier(createQualifier(classId,className, schemeId, schemeName)) + sp.setQualifier(OafUtils.createQualifier(classId,className, schemeId, schemeName)) sp.setValue(value) sp.setDataInfo(dataInfo) sp @@ -287,7 +287,7 @@ object DoiBoostMappingUtil { def createSP(value: String, classId: String, schemeId: String): StructuredProperty = { val sp = new StructuredProperty - sp.setQualifier(createQualifier(classId, schemeId)) + sp.setQualifier(OafUtils.createQualifier(classId, schemeId)) sp.setValue(value) sp @@ -297,7 +297,7 @@ object DoiBoostMappingUtil { def createSP(value: String, classId: String, schemeId: String, dataInfo: DataInfo): StructuredProperty = { val sp = new StructuredProperty - sp.setQualifier(createQualifier(classId, schemeId)) + sp.setQualifier(OafUtils.createQualifier(classId, schemeId)) sp.setValue(value) sp.setDataInfo(dataInfo) sp @@ -350,20 +350,6 @@ object DoiBoostMappingUtil { } - def createQualifier(clsName: String, clsValue: String, schName: String, schValue: String): Qualifier = { - val q = new Qualifier - q.setClassid(clsName) - q.setClassname(clsValue) - q.setSchemeid(schName) - q.setSchemename(schValue) - q - } - - def createQualifier(cls: String, sch: String): Qualifier = { - createQualifier(cls, cls, sch, sch) - } - - def asField[T](value: T): Field[T] = { val tmp = new Field[T] tmp.setValue(value) diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index 79aa7a586..9251bba0e 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -15,6 +15,8 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.matching.Regex +import eu.dnetlib.dhp.schema.scholexplorer.OafUtils; + case class CrossrefDT(doi: String, json:String, timestamp: Long) {} case class mappingAffiliation(name: String) {} @@ -179,14 +181,14 @@ case object Crossref2Oaf { if(has_review != JNothing) { instance.setRefereed( - createQualifier("0001", "peerReviewed", "dnet:review_levels", "dnet:review_levels")) + OafUtils.createQualifier("0001", "peerReviewed", "dnet:review_levels", "dnet:review_levels")) } instance.setAccessright(getRestrictedQualifier()) result.setInstance(List(instance).asJava) - instance.setInstancetype(createQualifier(cobjCategory.substring(0, 4), cobjCategory.substring(5), "dnet:publication_resource", "dnet:publication_resource")) - result.setResourcetype(createQualifier(cobjCategory.substring(0, 4),"dnet:dataCite_resource")) + instance.setInstancetype(OafUtils.createQualifier(cobjCategory.substring(0, 4), cobjCategory.substring(5), "dnet:publication_resource", "dnet:publication_resource")) + result.setResourcetype(OafUtils.createQualifier(cobjCategory.substring(0, 4),"dnet:dataCite_resource")) instance.setCollectedfrom(createCrossrefCollectedFrom()) if (StringUtils.isNotBlank(issuedDate)) { diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java index 1aed66dfd..8ba397048 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java @@ -18,6 +18,7 @@ import com.google.gson.*; import eu.dnetlib.dhp.common.PacePerson; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.scholexplorer.OafUtils; import eu.dnetlib.dhp.utils.DHPUtils; import eu.dnetlib.doiboost.orcidnodoi.util.DumpToActionsUtility; import eu.dnetlib.doiboost.orcidnodoi.util.Pair; @@ -102,8 +103,6 @@ public class PublicationToOaf implements Serializable { } } - public static final String PID_TYPES = "dnet:pid_types"; - public Oaf generatePublicationActionsFromJson(final String json) { try { if (parsedPublications != null) { @@ -138,8 +137,8 @@ public class PublicationToOaf implements Serializable { mapQualifier( "sysimport:actionset:orcidworks-no-doi", "sysimport:actionset:orcidworks-no-doi", - "dnet:provenanceActions", - "dnet:provenanceActions")); + ModelConstants.DNET_PROVENANCE_ACTIONS, + ModelConstants.DNET_PROVENANCE_ACTIONS)); publication.setDataInfo(dataInfo); publication.setLastupdatetimestamp(new Date().getTime()); @@ -159,7 +158,9 @@ public class PublicationToOaf implements Serializable { publication .getExternalReference() .add( - convertExtRef(extId, classid, classname, "dnet:pid_types", "dnet:pid_types")); + convertExtRef( + extId, classid, classname, ModelConstants.DNET_PID_TYPES, + ModelConstants.DNET_PID_TYPES)); } }); @@ -182,7 +183,8 @@ public class PublicationToOaf implements Serializable { } return null; } - Qualifier q = mapQualifier("main title", "main title", "dnet:dataCite_title", "dnet:dataCite_title"); + Qualifier q = mapQualifier( + "main title", "main title", ModelConstants.DNET_DATACITE_TITLE, ModelConstants.DNET_DATACITE_TITLE); publication .setTitle( titles @@ -214,7 +216,10 @@ public class PublicationToOaf implements Serializable { final String type = getStringValue(rootElement, "type"); String cobjValue = ""; if (StringUtils.isNotBlank(type)) { - publication.setResourcetype(mapQualifier(type, type, "dnet:dataCite_resource", "dnet:dataCite_resource")); + publication + .setResourcetype( + mapQualifier( + type, type, ModelConstants.DNET_DATA_CITE_RESOURCE, ModelConstants.DNET_DATA_CITE_RESOURCE)); final String typeValue = typologiesMapping.get(type).get("value"); cobjValue = typologiesMapping.get(type).get("cobj"); @@ -239,12 +244,21 @@ public class PublicationToOaf implements Serializable { instance.setCollectedfrom(createCollectedFrom()); // Adding accessright - instance.setAccessright(mapQualifier("UNKNOWN", "UNKNOWN", "dnet:access_modes", "dnet:access_modes")); + instance + .setAccessright( + OafUtils + .createAccessRight( + ModelConstants.UNKNOWN, + ModelConstants.UNKNOWN, + ModelConstants.DNET_ACCESS_MODES, + ModelConstants.DNET_ACCESS_MODES)); // Adding type instance .setInstancetype( - mapQualifier(cobjValue, typeValue, "dnet:publication_resource", "dnet:publication_resource")); + mapQualifier( + cobjValue, typeValue, ModelConstants.DNET_PUBLICATION_RESOURCE, + ModelConstants.DNET_PUBLICATION_RESOURCE)); publication.setInstance(Arrays.asList(instance)); } else { @@ -266,7 +280,10 @@ public class PublicationToOaf implements Serializable { } String classValue = getDefaultResulttype(cobjValue); publication - .setResulttype(mapQualifier(classValue, classValue, "dnet:result_typologies", "dnet:result_typologies")); + .setResulttype( + mapQualifier( + classValue, classValue, ModelConstants.DNET_RESULT_TYPOLOGIES, + ModelConstants.DNET_RESULT_TYPOLOGIES)); if (enrichedPublications != null) { enrichedPublications.add(1); } @@ -373,7 +390,8 @@ public class PublicationToOaf implements Serializable { if (addToDateOfAcceptance) { publication.setDateofacceptance(mapStringField(pubDate, null)); } - Qualifier q = mapQualifier(dictionaryKey, dictionaryKey, "dnet:dataCite_date", "dnet:dataCite_date"); + Qualifier q = mapQualifier( + dictionaryKey, dictionaryKey, ModelConstants.DNET_DATACITE_DATE, ModelConstants.DNET_DATACITE_DATE); publication .setRelevantdate( Arrays @@ -535,8 +553,8 @@ public class PublicationToOaf implements Serializable { mapQualifier( "sysimport:crosswalk:entityregistry", "Harvested", - "dnet:provenanceActions", - "dnet:provenanceActions")); + ModelConstants.DNET_PROVENANCE_ACTIONS, + ModelConstants.DNET_PROVENANCE_ACTIONS)); sp.setDataInfo(dataInfo); return sp; } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java index d2d4e118f..08695ec67 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java @@ -9,6 +9,7 @@ import org.apache.commons.lang3.StringUtils; import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableConsumer; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.AccessRight; import eu.dnetlib.dhp.schema.oaf.Country; import eu.dnetlib.dhp.schema.oaf.Qualifier; @@ -22,6 +23,7 @@ public class CleaningRuleMap extends HashMap public static CleaningRuleMap create(VocabularyGroup vocabularies) { CleaningRuleMap mapping = new CleaningRuleMap(); mapping.put(Qualifier.class, o -> cleanQualifier(vocabularies, (Qualifier) o)); + mapping.put(AccessRight.class, o -> cleanQualifier(vocabularies, (AccessRight) o)); mapping.put(Country.class, o -> { final Country c = (Country) o; if (StringUtils.isBlank(c.getSchemeid())) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index f03dcab7e..561c6a6b8 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -4,7 +4,13 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.*; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.dom4j.Document; @@ -15,7 +21,7 @@ import org.dom4j.Node; import com.google.common.collect.Lists; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.common.LicenseComparator; +import eu.dnetlib.dhp.schema.common.AccessRightComparator; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; @@ -418,6 +424,20 @@ public abstract class AbstractMdRecordToOafMapper { } + protected AccessRight prepareAccessRight(final Node node, final String xpath, final String schemeId) { + Qualifier qualifier = prepareQualifier(node.valueOf(xpath).trim(), schemeId); + AccessRight accessRight = new AccessRight(); + accessRight.setClassid(qualifier.getClassid()); + accessRight.setClassname(qualifier.getClassname()); + accessRight.setSchemeid(qualifier.getSchemeid()); + accessRight.setSchemename(qualifier.getSchemename()); + + // TODO set the OAStatus + // accessRight.setOaStatus(...); + + return accessRight; + } + protected Qualifier prepareQualifier(final Node node, final String xpath, final String schemeId) { return prepareQualifier(node.valueOf(xpath).trim(), schemeId); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 50208a079..48219be97 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -128,7 +128,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { instance.setDateofacceptance(field(doc.valueOf("//oaf:dateAccepted"), info)); instance.setDistributionlocation(doc.valueOf("//oaf:distributionlocation")); instance - .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); + .setAccessright(prepareAccessRight(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); instance.setLicense(field(doc.valueOf("//oaf:license"), info)); instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS)); instance diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index cddd00ad7..9d0d2368a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -102,10 +102,11 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { .setInstancetype(prepareQualifier(doc, "//dr:CobjCategory", DNET_PUBLICATION_RESOURCE)); instance.setCollectedfrom(collectedfrom); instance.setHostedby(hostedby); + instance.setPid(prepareResultPids(doc, info)); instance.setDateofacceptance(field(doc.valueOf("//oaf:dateAccepted"), info)); instance.setDistributionlocation(doc.valueOf("//oaf:distributionlocation")); instance - .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); + .setAccessright(prepareAccessRight(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); instance.setLicense(field(doc.valueOf("//oaf:license"), info)); instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS)); instance.setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info)); @@ -150,14 +151,20 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { for (final Object o : doc.selectNodes("//datacite:date")) { final String dateType = ((Node) o).valueOf("@dateType"); if (StringUtils.isBlank(dateType) - && !dateType.equalsIgnoreCase("Accepted") - && !dateType.equalsIgnoreCase("Issued") - && !dateType.equalsIgnoreCase("Updated") - && !dateType.equalsIgnoreCase("Available")) { + || (!dateType.equalsIgnoreCase("Accepted") + && !dateType.equalsIgnoreCase("Issued") + && !dateType.equalsIgnoreCase("Updated") + && !dateType.equalsIgnoreCase("Available"))) { res .add( structuredProperty( - ((Node) o).getText(), "UNKNOWN", "UNKNOWN", DNET_DATA_CITE_DATE, DNET_DATA_CITE_DATE, + ((Node) o).getText(), "UNKNOWN", "UNKNOWN", DNET_DATACITE_DATE, DNET_DATACITE_DATE, + info)); + } else { + res + .add( + structuredProperty( + ((Node) o).getText(), dateType, dateType, DNET_DATACITE_DATE, DNET_DATACITE_DATE, info)); } } @@ -186,13 +193,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { @Override protected List> prepareDescriptions(final Document doc, final DataInfo info) { - return prepareListFields(doc, "//datacite:description[@descriptionType='Abstract']", info) - .stream() - .map(d -> { - d.setValue(StringUtils.left(d.getValue(), ModelHardLimits.MAX_ABSTRACT_LENGTH)); - return d; - }) - .collect(Collectors.toList()); + return prepareListFields(doc, "//datacite:description[@descriptionType='Abstract']", info); } @Override diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala index 875c7cafd..607c4fa45 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala @@ -258,7 +258,7 @@ object DLIToOAF { result.setDateofacceptance(asField(inputPublication.getRelevantdate.get(0).getValue)) result.setPublisher(inputPublication.getPublisher) result.setSource(inputPublication.getSource) - result.setBestaccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + result.setBestaccessright(createAccessRight("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) val dois = result.getPid.asScala.filter(p => "doi".equalsIgnoreCase(p.getQualifier.getClassname)).map(p => p.getValue) if (dois.isEmpty) @@ -337,7 +337,7 @@ object DLIToOAF { result.setDateofacceptance(asField(d.getRelevantdate.get(0).getValue)) result.setPublisher(d.getPublisher) result.setSource(d.getSource) - result.setBestaccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + result.setBestaccessright(createAccessRight("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) val instance_urls = if (fpids.head.length < 5) s"https://www.rcsb.org/structure/${fpids.head}" else s"https://dx.doi.org/${fpids.head}" @@ -373,7 +373,7 @@ object DLIToOAF { if (originalInstance != null && originalInstance.getHostedby != null) i.setHostedby(originalInstance.getHostedby) - i.setAccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + i.setAccessright(createAccessRight("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) i.setDateofacceptance(doa) i