From 7941d7be294c02a513afc7b1690ac5b898676a78 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 31 Mar 2021 18:33:57 +0200 Subject: [PATCH] WIP: using common definitions from ModelConstants --- .../dhp/schema/common/ModelConstants.java | 20 ++++++++++------ .../dhp/schema/common/ModelSupport.java | 24 +++++++++---------- .../dhp/schema/scholexplorer/OafUtils.scala | 3 ++- .../dhp/schema/scholexplorer/DLItest.java | 7 +++--- .../migration/ProtoConverter.java | 5 ++-- .../DataciteToOAFTransformation.scala | 19 ++++++--------- .../dhp/broker/oa/util/BrokerConstants.java | 3 ++- .../oa/dedup/SparkCopyOpenorgsMergeRels.java | 16 ++++++------- .../dedup/SparkCreateConnectedComponent.java | 8 +++---- .../doiboost/DoiBoostMappingUtil.scala | 20 ++++------------ .../doiboost/crossref/Crossref2Oaf.scala | 10 ++++---- .../dnetlib/doiboost/mag/MagDataModel.scala | 17 ++++++------- .../dnetlib/doiboost/orcid/ORCIDToOAF.scala | 11 +++++---- .../orcidnodoi/oaf/PublicationToOaf.java | 15 ++++-------- .../dnetlib/doiboost/uw/UnpayWallToOAF.scala | 4 ++-- .../graph/merge/MergeGraphTableSparkJob.java | 3 ++- .../dhp/oa/graph/raw/OdfToOafMapper.java | 5 ++-- .../dhp/sx/ebi/SparkAddLinkUpdates.scala | 13 +++++----- .../parser/AbstractScholexplorerParser.java | 13 +++++----- .../parser/DatasetScholexplorerParser.java | 7 ++---- .../java/eu/dnetlib/dhp/export/DLIToOAF.scala | 8 +++---- .../dhp/oa/provision/RelationComparator.java | 2 +- 22 files changed, 112 insertions(+), 121 deletions(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index 364c36ee2..1f9a09c3a 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -35,10 +35,22 @@ public class ModelConstants { public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"; public static final String DNET_COUNTRY_TYPE = "dnet:countries"; public static final String DNET_REVIEW_LEVELS = "dnet:review_levels"; + public static final String DNET_PROGRAMMING_LANGUAGES = "dnet:programming_languages"; + public static final String DNET_PROVENANCEACTIONS = "dnet:provenanceActions"; + public static final String DNET_EXTERNAL_REF_TYPES = "dnet:externalReference_typologies"; public static final String SYSIMPORT_CROSSWALK_REPOSITORY = "sysimport:crosswalk:repository"; public static final String SYSIMPORT_CROSSWALK_ENTITYREGISTRY = "sysimport:crosswalk:entityregistry"; + public static final String SYSIMPORT_ACTIONSET = "sysimport:actionset"; + public static final String SYSIMPORT_ORCID_NO_DOI = "sysimport:actionset:orcidworks-no-doi"; + public static final String USER_CLAIM = "user:claim"; + public static final String HARVESTED = "Harvested"; + + public static final String PROVENANCE_DEDUP = "sysimport:dedup"; + + public static final Qualifier PROVENANCE_ACTION_SET_QUALIFIER = qualifier( + SYSIMPORT_ACTIONSET, SYSIMPORT_ACTIONSET, DNET_PROVENANCEACTIONS, DNET_PROVENANCEACTIONS); public static final String DATASET_RESULTTYPE_CLASSID = "dataset"; public static final String PUBLICATION_RESULTTYPE_CLASSID = "publication"; @@ -100,13 +112,6 @@ public class ModelConstants { public static final String UNKNOWN = "UNKNOWN"; public static final String NOT_AVAILABLE = "not available"; - public static final String ACTION_SET_SCHEME = "sysimport:actionset"; - - public static final String PROVENANCE_VOCABULARY = "dnet:provenanceActions"; - - public static final Qualifier ACTION_SET_PROVENANCE_QUALIFIER = qualifier( - ACTION_SET_SCHEME, ACTION_SET_SCHEME, PROVENANCE_VOCABULARY, PROVENANCE_VOCABULARY); - public static final Qualifier PUBLICATION_DEFAULT_RESULTTYPE = qualifier( PUBLICATION_RESULTTYPE_CLASSID, PUBLICATION_RESULTTYPE_CLASSID, DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES); @@ -131,6 +136,7 @@ public class ModelConstants { SYSIMPORT_CROSSWALK_ENTITYREGISTRY, SYSIMPORT_CROSSWALK_ENTITYREGISTRY, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS); + public static final String UNKNOWN_REPOSITORY_ORIGINALID = "openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18"; public static final KeyValue UNKNOWN_REPOSITORY = keyValue( "10|openaire____::55045bd2a65019fd8e6741a755395c8c", "Unknown Repository"); diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java index b6a2015ed..18ce2d967 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java @@ -158,10 +158,10 @@ public class ModelSupport { relationInverseMap .put( "organizationOrganization_dedupSimilarity_isSimilarTo", new RelationInverse() - .setInverse("isSimilarTo") - .setRelation("isSimilarTo") - .setRelType("organizationOrganization") - .setSubReltype("dedupSimilarity")); + .setInverse(ModelConstants.IS_SIMILAR_TO) + .setRelation(ModelConstants.IS_SIMILAR_TO) + .setRelType(ModelConstants.ORG_ORG_RELTYPE) + .setSubReltype(ModelConstants.DEDUP)); relationInverseMap .put( @@ -215,10 +215,10 @@ public class ModelSupport { relationInverseMap .put( "personPerson_dedupSimilarity_isSimilarTo", new RelationInverse() - .setInverse("isSimilarTo") - .setRelation("isSimilarTo") + .setInverse(ModelConstants.IS_SIMILAR_TO) + .setRelation(ModelConstants.IS_SIMILAR_TO) .setRelType("personPerson") - .setSubReltype("dedupSimilarity")); + .setSubReltype(ModelConstants.DEDUP)); relationInverseMap .put( "datasourceOrganization_provision_isProvidedBy", new RelationInverse() @@ -239,14 +239,14 @@ public class ModelSupport { .setInverse("isAmongTopNSimilarDocuments") .setRelation("hasAmongTopNSimilarDocuments") .setRelType(ModelConstants.RESULT_RESULT) - .setSubReltype("similarity")); + .setSubReltype(ModelConstants.SIMILARITY)); relationInverseMap .put( "resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse() .setInverse("hasAmongTopNSimilarDocuments") .setRelation("isAmongTopNSimilarDocuments") .setRelType(ModelConstants.RESULT_RESULT) - .setSubReltype("similarity")); + .setSubReltype(ModelConstants.SIMILARITY)); relationInverseMap .put( "resultResult_relationship_isRelatedTo", new RelationInverse() @@ -299,10 +299,10 @@ public class ModelSupport { relationInverseMap .put( "resultResult_dedupSimilarity_isSimilarTo", new RelationInverse() - .setInverse("isSimilarTo") - .setRelation("isSimilarTo") + .setInverse(ModelConstants.IS_SIMILAR_TO) + .setRelation(ModelConstants.IS_SIMILAR_TO) .setRelType(ModelConstants.RESULT_RESULT) - .setSubReltype("dedupSimilarity")); + .setSubReltype(ModelConstants.DEDUP)); } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala index defcb6a28..b0cd344be 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/OafUtils.scala @@ -1,5 +1,6 @@ package eu.dnetlib.dhp.schema.scholexplorer +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.{AccessRight, DataInfo, Field, KeyValue, Qualifier, StructuredProperty} object OafUtils { @@ -21,7 +22,7 @@ object OafUtils { di.setInferred(false) di.setInvisible(invisible) di.setTrust(trust) - di.setProvenanceaction(createQualifier("sysimport:actionset", "dnet:provenanceActions")) + di.setProvenanceaction(createQualifier(ModelConstants.SYSIMPORT_ACTIONSET, ModelConstants.DNET_PROVENANCE_ACTIONS)) di } diff --git a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/scholexplorer/DLItest.java b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/scholexplorer/DLItest.java index e4596fcdd..16ce1523d 100644 --- a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/scholexplorer/DLItest.java +++ b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/scholexplorer/DLItest.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; @@ -20,7 +21,7 @@ public class DLItest { @Test public void testMergePublication() throws JsonProcessingException { DLIPublication a1 = new DLIPublication(); - a1.setPid(Arrays.asList(createSP("123456", "pdb", "dnet:pid_types"))); + a1.setPid(Arrays.asList(createSP("123456", "pdb", ModelConstants.DNET_PID_TYPES))); a1.setTitle(Collections.singletonList(createSP("Un Titolo", "title", "dnetTitle"))); a1.setDlicollectedfrom(Arrays.asList(createCollectedFrom("znd", "Zenodo", "complete"))); a1.setCompletionStatus("complete"); @@ -30,8 +31,8 @@ public class DLItest { .setPid( Arrays .asList( - createSP("10.11", "doi", "dnet:pid_types"), - createSP("123456", "pdb", "dnet:pid_types"))); + createSP("10.11", "doi", ModelConstants.DNET_PID_TYPES), + createSP("123456", "pdb", ModelConstants.DNET_PID_TYPES))); a.setTitle(Collections.singletonList(createSP("A Title", "title", "dnetTitle"))); a .setDlicollectedfrom( diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java index 5aeb38bb5..2478da0e9 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java @@ -15,6 +15,7 @@ import com.google.common.collect.Lists; import com.googlecode.protobuf.format.JsonFormat; import eu.dnetlib.data.proto.*; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; public class ProtoConverter implements Serializable { @@ -106,8 +107,8 @@ public class ProtoConverter implements Serializable { Qualifier q = new Qualifier(); q.setClassid(refereed.getValue()); q.setSchemename(refereed.getValue()); - q.setSchemeid("dnet:review_levels"); - q.setSchemename("dnet:review_levels"); + q.setSchemeid(DNET_REVIEW_LEVELS); + q.setSchemename(DNET_REVIEW_LEVELS); return q; } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/DataciteToOAFTransformation.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/DataciteToOAFTransformation.scala index 871d5a5c3..6eaf2d377 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/DataciteToOAFTransformation.scala +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/DataciteToOAFTransformation.scala @@ -23,8 +23,6 @@ import java.util.regex.Pattern import scala.collection.JavaConverters._ import scala.io.{Codec, Source} - - case class DataciteType(doi:String,timestamp:Long,isActive:Boolean, json:String ){} case class NameIdentifiersType(nameIdentifierScheme: Option[String], schemeUri: Option[String], nameIdentifier: Option[String]) {} @@ -49,10 +47,7 @@ object DataciteToOAFTransformation { codec.onMalformedInput(CodingErrorAction.REPLACE) codec.onUnmappableCharacter(CodingErrorAction.REPLACE) - val ACCESS_MODE_VOCABULARY = "dnet:access_modes" val DOI_CLASS = "doi" - - val SUBJ_CLASS = "keywords" @@ -62,7 +57,7 @@ object DataciteToOAFTransformation { } val mapper = new ObjectMapper() - val unknown_repository: HostedByMapType = HostedByMapType("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "Unknown Repository", Some(1.0F)) + val unknown_repository: HostedByMapType = HostedByMapType(ModelConstants.UNKNOWN_REPOSITORY_ORIGINALID, ModelConstants.UNKNOWN_REPOSITORY.getValue, ModelConstants.UNKNOWN_REPOSITORY.getValue, Some(1.0F)) val dataInfo: DataInfo = generateDataInfo("0.9") val DATACITE_COLLECTED_FROM: KeyValue = OafMapperUtils.keyValue(ModelConstants.DATACITE_ID, "Datacite") @@ -244,9 +239,9 @@ object DataciteToOAFTransformation { val r = new Relation r.setSource(sourceId) r.setTarget(targetId) - r.setRelType("resultProject") + r.setRelType(ModelConstants.RESULT_PROJECT) r.setRelClass(relClass) - r.setSubRelType("outcome") + r.setSubRelType(ModelConstants.OUTCOME) r.setCollectedfrom(List(cf).asJava) r.setDataInfo(di) r @@ -381,7 +376,7 @@ object DataciteToOAFTransformation { result.setRelevantdate(dates.filter(d => d.date.isDefined && d.dateType.isDefined) .map(d => (extract_date(d.date.get), d.dateType.get)) .filter(d => d._1.isDefined) - .map(d => (d._1.get, vocabularies.getTermAsQualifier("dnet:dataCite_date", d._2.toLowerCase()))) + .map(d => (d._1.get, vocabularies.getTermAsQualifier(ModelConstants.DNET_DATACITE_DATE, d._2.toLowerCase()))) .filter(d => d._2 != null) .map(d => generateOAFDate(d._1, d._2)).asJava) @@ -413,7 +408,7 @@ object DataciteToOAFTransformation { val language: String = (json \\ "language").extractOrElse[String](null) if (language != null) - result.setLanguage(vocabularies.getSynonymAsQualifier("dnet:languages", language)) + result.setLanguage(vocabularies.getSynonymAsQualifier(ModelConstants.DNET_LANGUAGES, language)) val instance = result.getInstance().get(0) @@ -437,7 +432,7 @@ object DataciteToOAFTransformation { }) - val access_rights_qualifier = if (aRights.isDefined) aRights.get else OafMapperUtils.accessRight("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES) + val access_rights_qualifier = if (aRights.isDefined) aRights.get else OafMapperUtils.accessRight(ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE, ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES) if (client.isDefined) { val hb = hostedByMap.getOrElse(client.get.toUpperCase(), unknown_repository) @@ -475,7 +470,7 @@ object DataciteToOAFTransformation { di.setInferred(false) di.setInvisible(false) di.setTrust(trust) - di.setProvenanceaction(ModelConstants.ACTION_SET_PROVENANCE_QUALIFIER) + di.setProvenanceaction(ModelConstants.PROVENANCE_ACTION_SET_QUALIFIER) di } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java index 7a09862d8..2055a014e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java @@ -7,12 +7,13 @@ import java.util.Set; import eu.dnetlib.dhp.broker.model.Event; import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; public class BrokerConstants { public static final String OPEN_ACCESS = "OPEN"; - public static final String IS_MERGED_IN_CLASS = "isMergedIn"; + public static final String IS_MERGED_IN_CLASS = ModelConstants.IS_MERGED_IN; public static final String COLLECTED_FROM_REL = "collectedFrom"; diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java index aa05ab65c..00036ebb0 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java @@ -35,8 +35,6 @@ import net.sf.saxon.ma.trie.Tuple2; public class SparkCopyOpenorgsMergeRels extends AbstractSparkAction { private static final Logger log = LoggerFactory.getLogger(SparkCopyOpenorgsMergeRels.class); - public static final String PROVENANCE_ACTION_CLASS = "sysimport:dedup"; - public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"; public SparkCopyOpenorgsMergeRels(ArgumentApplicationParser parser, SparkSession spark) { super(parser, spark); @@ -93,7 +91,7 @@ public class SparkCopyOpenorgsMergeRels extends AbstractSparkAction { JavaRDD selfRawRels = rawRels .map(r -> r.getSource()) .distinct() - .map(s -> rel(s, s, "isSimilarTo", dedupConf)); + .map(s -> rel(s, s, ModelConstants.IS_SIMILAR_TO, dedupConf)); log.info("Number of raw Openorgs Relations collected: {}", rawRels.count()); @@ -109,8 +107,8 @@ public class SparkCopyOpenorgsMergeRels extends AbstractSparkAction { List mergerels = new ArrayList<>(); - mergerels.add(rel(rel.getSource(), rel.getTarget(), "merges", dedupConf)); - mergerels.add(rel(rel.getTarget(), rel.getSource(), "isMergedIn", dedupConf)); + mergerels.add(rel(rel.getSource(), rel.getTarget(), ModelConstants.MERGES, dedupConf)); + mergerels.add(rel(rel.getTarget(), rel.getSource(), ModelConstants.IS_MERGED_IN, dedupConf)); return mergerels.iterator(); }); @@ -174,10 +172,10 @@ public class SparkCopyOpenorgsMergeRels extends AbstractSparkAction { info.setInvisible(false); info.setInferenceprovenance(dedupConf.getWf().getConfigurationId()); Qualifier provenanceAction = new Qualifier(); - provenanceAction.setClassid(PROVENANCE_ACTION_CLASS); - provenanceAction.setClassname(PROVENANCE_ACTION_CLASS); - provenanceAction.setSchemeid(DNET_PROVENANCE_ACTIONS); - provenanceAction.setSchemename(DNET_PROVENANCE_ACTIONS); + provenanceAction.setClassid(ModelConstants.PROVENANCE_DEDUP); + provenanceAction.setClassname(ModelConstants.PROVENANCE_DEDUP); + provenanceAction.setSchemeid(ModelConstants.DNET_PROVENANCE_ACTIONS); + provenanceAction.setSchemename(ModelConstants.DNET_PROVENANCE_ACTIONS); info.setProvenanceaction(provenanceAction); // TODO calculate the trust value based on the similarity score of the elements in the CC diff --git a/dhp-workflows/dhp-dedup-scholexplorer/src/main/java/eu/dnetlib/dedup/SparkCreateConnectedComponent.java b/dhp-workflows/dhp-dedup-scholexplorer/src/main/java/eu/dnetlib/dedup/SparkCreateConnectedComponent.java index 9bc90d51d..2f0b1e574 100644 --- a/dhp-workflows/dhp-dedup-scholexplorer/src/main/java/eu/dnetlib/dedup/SparkCreateConnectedComponent.java +++ b/dhp-workflows/dhp-dedup-scholexplorer/src/main/java/eu/dnetlib/dedup/SparkCreateConnectedComponent.java @@ -22,6 +22,7 @@ import com.google.common.hash.Hashing; import eu.dnetlib.dedup.graph.ConnectedComponent; import eu.dnetlib.dedup.graph.GraphProcessor; import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.pace.config.DedupConfig; @@ -47,8 +48,7 @@ public class SparkCreateConnectedComponent { final String inputPath = parser.get("sourcePath"); final String entity = parser.get("entity"); final String targetPath = parser.get("targetPath"); - // final DedupConfig dedupConf = - // DedupConfig.load(IOUtils.toString(SparkCreateConnectedComponent.class.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/org.curr.conf2.json"))); + final DedupConfig dedupConf = DedupConfig.load(parser.get("dedupConf")); final JavaPairRDD vertexes = spark @@ -88,12 +88,12 @@ public class SparkCreateConnectedComponent { Relation r = new Relation(); r.setSource(c.getCcId()); r.setTarget(id); - r.setRelClass("merges"); + r.setRelClass(ModelConstants.MERGES); tmp.add(r); r = new Relation(); r.setTarget(c.getCcId()); r.setSource(id); - r.setRelClass("isMergedIn"); + r.setRelClass(ModelConstants.IS_MERGED_IN); tmp.add(r); return tmp.stream(); }) diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala index f2c63cee6..a6101c07e 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala @@ -30,8 +30,6 @@ object DoiBoostMappingUtil { //STATIC STRING val MAG = "microsoft" val MAG_NAME = "Microsoft Academic Graph" - val ORCID = "orcid" - val ORCID_PENDING = "orcid_pending" val CROSSREF = "Crossref" val UNPAYWALL = "UnpayWall" val GRID_AC = "grid.ac" @@ -39,8 +37,6 @@ object DoiBoostMappingUtil { val doiBoostNSPREFIX = "doiboost____" val OPENAIRE_PREFIX = "openaire____" val SEPARATOR = "::" - val DNET_LANGUAGES = "dnet:languages" - val PID_TYPES = "dnet:pid_types" val invalidName = List(",", "none none", "none, none", "none &na;", "(:null)", "test test test", "test test", "test", "&na; &na;") @@ -122,12 +118,11 @@ object DoiBoostMappingUtil { def getOpenAccessQualifier():AccessRight = { - OafUtils.createAccessRight("OPEN","Open Access","dnet:access_modes", "dnet:access_modes") + OafUtils.createAccessRight("OPEN","Open Access", ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES) } def getRestrictedQualifier():AccessRight = { - OafUtils.createAccessRight("RESTRICTED","Restricted","dnet:access_modes", "dnet:access_modes") - + OafUtils.createAccessRight("RESTRICTED","Restricted",ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES) } @@ -141,14 +136,12 @@ object DoiBoostMappingUtil { val item = if (input._2 != null) input._2._2 else null - val instanceType:Option[Instance] = extractInstance(publication) if (instanceType.isDefined) { publication.getInstance().asScala.foreach(i => i.setInstancetype(instanceType.get.getInstancetype)) } - publication.getInstance().asScala.foreach(i => { var hb = new KeyValue if (item != null) { @@ -179,7 +172,6 @@ object DoiBoostMappingUtil { publication } - def generateDSId(input: String): String = { val b = StringUtils.substringBefore(input, "::") @@ -187,12 +179,10 @@ object DoiBoostMappingUtil { s"10|${b}::${DHPUtils.md5(a)}" } - def generateDataInfo(): DataInfo = { generateDataInfo("0.9") } - def filterPublication(publication: Publication): Boolean = { //Case empty publication @@ -264,7 +254,7 @@ object DoiBoostMappingUtil { di.setInferred(false) di.setInvisible(false) di.setTrust(trust) - di.setProvenanceaction(OafUtils.createQualifier("sysimport:actionset", "dnet:provenanceActions")) + di.setProvenanceaction(OafUtils.createQualifier(ModelConstants.SYSIMPORT_ACTIONSET, ModelConstants.DNET_PROVENANCE_ACTIONS)) di } @@ -330,8 +320,8 @@ object DoiBoostMappingUtil { def createORIDCollectedFrom(): KeyValue = { val cf = new KeyValue - cf.setValue(ORCID) - cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ORCID.toLowerCase)) + cf.setValue(StringUtils.upperCase(ModelConstants.ORCID)) + cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ModelConstants.ORCID.toLowerCase)) cf } diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index b051177f5..5bbfa8ac0 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -1,5 +1,6 @@ package eu.dnetlib.doiboost.crossref +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf._ import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory import eu.dnetlib.dhp.utils.DHPUtils @@ -14,7 +15,6 @@ import org.slf4j.{Logger, LoggerFactory} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.matching.Regex - import eu.dnetlib.dhp.schema.scholexplorer.OafUtils case class CrossrefDT(doi: String, json:String, timestamp: Long) {} @@ -88,7 +88,7 @@ case object Crossref2Oaf { //MAPPING Crossref DOI into PID val doi: String = (json \ "DOI").extract[String] - result.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava) + result.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava) //MAPPING Crossref DOI into OriginalId //and Other Original Identifier of dataset like clinical-trial-number @@ -186,13 +186,13 @@ case object Crossref2Oaf { if(has_review != JNothing) { instance.setRefereed( - OafUtils.createQualifier("0001", "peerReviewed", "dnet:review_levels", "dnet:review_levels")) + OafUtils.createQualifier("0001", "peerReviewed", ModelConstants.DNET_REVIEW_LEVELS, ModelConstants.DNET_REVIEW_LEVELS)) } instance.setAccessright(getRestrictedQualifier()) result.setInstance(List(instance).asJava) - instance.setInstancetype(OafUtils.createQualifier(cobjCategory.substring(0, 4), cobjCategory.substring(5), "dnet:publication_resource", "dnet:publication_resource")) + instance.setInstancetype(OafUtils.createQualifier(cobjCategory.substring(0, 4), cobjCategory.substring(5), ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE)) result.setResourcetype(OafUtils.createQualifier(cobjCategory.substring(0, 4),"dnet:dataCite_resource")) instance.setCollectedfrom(createCrossrefCollectedFrom()) @@ -221,7 +221,7 @@ case object Crossref2Oaf { a.setFullname(s"$given $family") a.setRank(index+1) if (StringUtils.isNotBlank(orcid)) - a.setPid(List(createSP(orcid, ORCID_PENDING, PID_TYPES, generateDataInfo())).asJava) + a.setPid(List(createSP(orcid, ModelConstants.ORCID_PENDING, ModelConstants.DNET_PID_TYPES, generateDataInfo())).asJava) a } diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala index 6a8ae9928..11a335700 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala @@ -1,6 +1,7 @@ package eu.dnetlib.doiboost.mag +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty} import eu.dnetlib.doiboost.DoiBoostMappingUtil @@ -191,7 +192,7 @@ case object ConversionUtil { val authors = inputParams._2 val pub = new Publication - pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava) + pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava) pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava) //IMPORTANT @@ -200,8 +201,8 @@ case object ConversionUtil { pub.setId(IdentifierFactory.createDOIBoostIdentifier(pub)) - val mainTitles = createSP(paper.PaperTitle, "main title", "dnet:dataCite_title") - val originalTitles = createSP(paper.OriginalTitle, "alternative title", "dnet:dataCite_title") + val mainTitles = createSP(paper.PaperTitle, "main title", ModelConstants.DNET_DATACITE_TITLE) + val originalTitles = createSP(paper.OriginalTitle, "alternative title", ModelConstants.DNET_DATACITE_TITLE) pub.setTitle(List(mainTitles, originalTitles).asJava) pub.setSource(List(asField(paper.BookTitle)).asJava) @@ -214,7 +215,7 @@ case object ConversionUtil { a.setFullname(f.author.DisplayName.get) if(f.affiliation!= null) a.setAffiliation(List(asField(f.affiliation)).asJava) - a.setPid(List(createSP(s"https://academic.microsoft.com/#/detail/${f.author.AuthorId}", "URL", PID_TYPES)).asJava) + a.setPid(List(createSP(s"https://academic.microsoft.com/#/detail/${f.author.AuthorId}", "URL", ModelConstants.DNET_PID_TYPES)).asJava) a } pub.setAuthor(authorsOAF.asJava) @@ -253,14 +254,14 @@ case object ConversionUtil { val description = inputParams._2 val pub = new Publication - pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava) + pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava) pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava) //Set identifier as 50 | doiboost____::md5(DOI) pub.setId(generateIdentifier(pub, paper.Doi.toLowerCase)) - val mainTitles = createSP(paper.PaperTitle, "main title", "dnet:dataCite_title") - val originalTitles = createSP(paper.OriginalTitle, "alternative title", "dnet:dataCite_title") + val mainTitles = createSP(paper.PaperTitle, "main title", ModelConstants.DNET_DATACITE_TITLE) + val originalTitles = createSP(paper.OriginalTitle, "alternative title", ModelConstants.DNET_DATACITE_TITLE) pub.setTitle(List(mainTitles, originalTitles).asJava) pub.setSource(List(asField(paper.BookTitle)).asJava) @@ -281,7 +282,7 @@ case object ConversionUtil { a.setAffiliation(List(asField(f.affiliation)).asJava) - a.setPid(List(createSP(s"https://academic.microsoft.com/#/detail/${f.author.AuthorId}", "URL", PID_TYPES)).asJava) + a.setPid(List(createSP(s"https://academic.microsoft.com/#/detail/${f.author.AuthorId}", "URL", ModelConstants.DNET_PID_TYPES)).asJava) a diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala index 02016b47c..8cf6efb49 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala @@ -1,11 +1,12 @@ package eu.dnetlib.doiboost.orcid import com.fasterxml.jackson.databind.ObjectMapper +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Publication} import eu.dnetlib.dhp.schema.orcid.OrcidDOI import eu.dnetlib.doiboost.DoiBoostMappingUtil -import eu.dnetlib.doiboost.DoiBoostMappingUtil.{ORCID, PID_TYPES, createSP, generateDataInfo, generateIdentifier} +import eu.dnetlib.doiboost.DoiBoostMappingUtil.{createSP, generateDataInfo} import org.apache.commons.lang.StringUtils import org.slf4j.{Logger, LoggerFactory} @@ -48,7 +49,7 @@ object ORCIDToOAF { def convertTOOAF(input:OrcidDOI) :Publication = { val doi = input.getDoi val pub:Publication = new Publication - pub.setPid(List(createSP(doi.toLowerCase, "doi", PID_TYPES)).asJava) + pub.setPid(List(createSP(doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava) pub.setDataInfo(generateDataInfo()) pub.setId(IdentifierFactory.createDOIBoostIdentifier(pub)) @@ -74,8 +75,8 @@ object ORCIDToOAF { def generateOricPIDDatainfo():DataInfo = { val di =DoiBoostMappingUtil.generateDataInfo("0.91") - di.getProvenanceaction.setClassid("sysimport:crosswalk:entityregistry") - di.getProvenanceaction.setClassname("Harvested") + di.getProvenanceaction.setClassid(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY) + di.getProvenanceaction.setClassname(ModelConstants.HARVESTED) di } @@ -88,7 +89,7 @@ object ORCIDToOAF { else a.setFullname(s"$given $family") if (StringUtils.isNotBlank(orcid)) - a.setPid(List(createSP(orcid, ORCID, PID_TYPES, generateOricPIDDatainfo())).asJava) + a.setPid(List(createSP(orcid, ModelConstants.ORCID, ModelConstants.DNET_PID_TYPES, generateOricPIDDatainfo())).asJava) a } diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java index 8ba397048..3b8c74062 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java @@ -26,12 +26,11 @@ import eu.dnetlib.doiboost.orcidnodoi.util.Pair; /** * This class converts an orcid publication from json format to oaf */ - public class PublicationToOaf implements Serializable { static Logger logger = LoggerFactory.getLogger(PublicationToOaf.class); - public static final String ORCID = "ORCID"; + public static final String ORCID = StringUtils.upperCase(ModelConstants.ORCID); public final static String orcidPREFIX = "orcid_______"; public static final String OPENAIRE_PREFIX = "openaire____"; public static final String SEPARATOR = "::"; @@ -70,7 +69,7 @@ public class PublicationToOaf implements Serializable { private static Map> datasources = new HashMap>() { { - put(ORCID.toLowerCase(), new Pair<>(ORCID, OPENAIRE_PREFIX + SEPARATOR + "orcid")); + put(ORCID.toLowerCase(), new Pair<>(ORCID, OPENAIRE_PREFIX + SEPARATOR + ModelConstants.ORCID)); } }; @@ -135,8 +134,8 @@ public class PublicationToOaf implements Serializable { dataInfo .setProvenanceaction( mapQualifier( - "sysimport:actionset:orcidworks-no-doi", - "sysimport:actionset:orcidworks-no-doi", + ModelConstants.SYSIMPORT_ORCID_NO_DOI, + ModelConstants.SYSIMPORT_ORCID_NO_DOI, ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS)); publication.setDataInfo(dataInfo); @@ -183,15 +182,11 @@ public class PublicationToOaf implements Serializable { } return null; } - Qualifier q = mapQualifier( - "main title", "main title", ModelConstants.DNET_DATACITE_TITLE, ModelConstants.DNET_DATACITE_TITLE); publication .setTitle( titles .stream() - .map(t -> { - return mapStructuredProperty(t, q, null); - }) + .map(t -> mapStructuredProperty(t, ModelConstants.MAIN_TITLE_QUALIFIER, null)) .filter(s -> s != null) .collect(Collectors.toList())); // Adding identifier diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala index b9895dd09..a7f97aaf8 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala @@ -79,7 +79,7 @@ object UnpayWallToOAF { if (oaLocation.license.isDefined) i.setLicense(asField(oaLocation.license.get)) - pub.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava) + pub.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava) // Ticket #6282 Adding open Access Colour if (colour.isDefined) { @@ -90,7 +90,7 @@ object UnpayWallToOAF { a.setSchemename(ModelConstants.DNET_ACCESS_MODES) a.setOpenAccessRoute(colour.get) i.setAccessright(a) - i.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava) + i.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava) } pub.setInstance(List(i).asJava) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/merge/MergeGraphTableSparkJob.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/merge/MergeGraphTableSparkJob.java index 68623dd55..602213e58 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/merge/MergeGraphTableSparkJob.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/merge/MergeGraphTableSparkJob.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import scala.Tuple2; @@ -46,7 +47,7 @@ public class MergeGraphTableSparkJob { static { Qualifier compatibility = new Qualifier(); - compatibility.setClassid("UNKNOWN"); + compatibility.setClassid(ModelConstants.UNKNOWN); DATASOURCE.setOpenairecompatibility(compatibility); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index 7592ee525..bb6e68b7e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -13,6 +13,7 @@ import org.dom4j.Node; import eu.dnetlib.dhp.common.PacePerson; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; @@ -169,7 +170,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { res .add( structuredProperty( - ((Node) o).getText(), "UNKNOWN", "UNKNOWN", DNET_DATACITE_DATE, DNET_DATACITE_DATE, + ((Node) o).getText(), UNKNOWN, UNKNOWN, DNET_DATACITE_DATE, DNET_DATACITE_DATE, info)); } else { res @@ -242,7 +243,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { @Override protected Qualifier prepareSoftwareProgrammingLanguage(final Document doc, final DataInfo info) { - return prepareQualifier(doc, "//datacite:format", "dnet:programming_languages"); + return prepareQualifier(doc, "//datacite:format", DNET_PROGRAMMING_LANGUAGES); } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/ebi/SparkAddLinkUpdates.scala b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/ebi/SparkAddLinkUpdates.scala index d5cdb8a7c..bc04e22e7 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/ebi/SparkAddLinkUpdates.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/ebi/SparkAddLinkUpdates.scala @@ -1,5 +1,6 @@ package eu.dnetlib.dhp.sx.ebi import eu.dnetlib.dhp.application.ArgumentApplicationParser +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.{Author, Instance, Journal, KeyValue, Oaf, Publication, Relation, Dataset => OafDataset} import eu.dnetlib.dhp.schema.scholexplorer.OafUtils.createQualifier import eu.dnetlib.dhp.schema.scholexplorer.{DLIDataset, DLIPublication, OafUtils, ProvenaceInfo} @@ -50,7 +51,7 @@ case class EBILinks(relation:String, pubdate:String, tpid:String, tpidType:Strin val p = new DLIPublication p.setId(dnetPublicationId) p.setDataInfo(OafUtils.generateDataInfo()) - p.setPid(List(OafUtils.createSP(input.getPmid.toLowerCase.trim, "pmid", "dnet:pid_types")).asJava) + p.setPid(List(OafUtils.createSP(input.getPmid.toLowerCase.trim, "pmid", ModelConstants.DNET_PID_TYPES)).asJava) p.setCompletionStatus("complete") val pi = new ProvenaceInfo pi.setId("dli_________::europe_pmc__") @@ -76,13 +77,13 @@ case class EBILinks(relation:String, pubdate:String, tpid:String, tpidType:Strin if (input.getJournal != null) p.setJournal(journalToOAF(input.getJournal)) - p.setTitle(List(OafUtils.createSP(input.getTitle, "main title", "dnet:dataCite_title")).asJava) + p.setTitle(List(OafUtils.createSP(input.getTitle, "main title", ModelConstants.DNET_DATACITE_TITLE)).asJava) p.setDateofacceptance(OafUtils.asField(input.getDate)) val i = new Instance i.setCollectedfrom(generatePubmedDLICollectedFrom()) i.setDateofacceptance(p.getDateofacceptance) i.setUrl(List(s"https://pubmed.ncbi.nlm.nih.gov/${input.getPmid}").asJava) - i.setInstancetype(createQualifier("0001", "Article", "dnet:publication_resource", "dnet:publication_resource")) + i.setInstancetype(createQualifier("0001", "Article", ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE)) p.setInstance(List(i).asJava) p } @@ -139,7 +140,7 @@ case class EBILinks(relation:String, pubdate:String, tpid:String, tpidType:Strin val d = new DLIDataset d.setId(targetDnetId) d.setDataInfo(OafUtils.generateDataInfo()) - d.setPid(List(OafUtils.createSP(l.tpid.toLowerCase.trim, l.tpidType.toLowerCase.trim, "dnet:pid_types")).asJava) + d.setPid(List(OafUtils.createSP(l.tpid.toLowerCase.trim, l.tpidType.toLowerCase.trim, ModelConstants.DNET_PID_TYPES)).asJava) d.setCompletionStatus("complete") val pi = new ProvenaceInfo pi.setId("dli_________::europe_pmc__") @@ -149,13 +150,13 @@ case class EBILinks(relation:String, pubdate:String, tpid:String, tpidType:Strin d.setDlicollectedfrom(List(pi).asJava) d.setCollectedfrom(List(generatePubmedDLICollectedFrom()).asJava) d.setPublisher(OafUtils.asField(l.publisher)) - d.setTitle(List(OafUtils.createSP(l.title, "main title", "dnet:dataCite_title")).asJava) + d.setTitle(List(OafUtils.createSP(l.title, "main title", ModelConstants.DNET_DATACITE_TITLE)).asJava) d.setDateofacceptance(OafUtils.asField(l.pubdate)) val i = new Instance i.setCollectedfrom(generatePubmedDLICollectedFrom()) i.setDateofacceptance(d.getDateofacceptance) i.setUrl(List(l.turl).asJava) - i.setInstancetype(createQualifier("0021", "Dataset", "dnet:publication_resource", "dnet:publication_resource")) + i.setInstancetype(createQualifier("0021", "Dataset", ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE)) d.setInstance(List(i).asJava) List(relation, inverseRelation, d) }) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/AbstractScholexplorerParser.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/AbstractScholexplorerParser.java index f56760c82..6e3dad7fd 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/AbstractScholexplorerParser.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/AbstractScholexplorerParser.java @@ -13,6 +13,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import eu.dnetlib.dhp.parser.utility.VtdUtilityParser; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.scholexplorer.DLIUnknown; import eu.dnetlib.dhp.schema.scholexplorer.ProvenaceInfo; @@ -76,8 +77,8 @@ public abstract class AbstractScholexplorerParser { final Qualifier pidType = new Qualifier(); pidType.setClassname(result.getAttributes().get(fieldName)); pidType.setClassid(result.getAttributes().get(fieldName)); - pidType.setSchemename("dnet:pid_types"); - pidType.setSchemeid("dnet:pid_types"); + pidType.setSchemename(ModelConstants.DNET_PID_TYPES); + pidType.setSchemeid(ModelConstants.DNET_PID_TYPES); pid.setQualifier(pidType); return pid; } @@ -90,8 +91,8 @@ public abstract class AbstractScholexplorerParser { input.setValue(matcher.group()); if (input.getQualifier() == null) { input.setQualifier(new Qualifier()); - input.getQualifier().setSchemename("dnet:pid_types"); - input.getQualifier().setSchemeid("dnet:pid_types"); + input.getQualifier().setSchemename(ModelConstants.DNET_PID_TYPES); + input.getQualifier().setSchemeid(ModelConstants.DNET_PID_TYPES); } input.getQualifier().setClassid("doi"); input.getQualifier().setClassname("doi"); @@ -141,8 +142,8 @@ public abstract class AbstractScholexplorerParser { final Qualifier pt = new Qualifier(); pt.setClassname(pidType); pt.setClassid(pidType); - pt.setSchemename("dnet:pid_types"); - pt.setSchemeid("dnet:pid_types"); + pt.setSchemename(ModelConstants.DNET_PID_TYPES); + pt.setSchemeid(ModelConstants.DNET_PID_TYPES); sourcePid.setQualifier(pt); uk.setPid(Collections.singletonList(sourcePid)); uk.setDateofcollection(dateOfCollection); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java index 11d9905cc..4493010a0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/parser/DatasetScholexplorerParser.java @@ -15,6 +15,7 @@ import com.ximpleware.VTDNav; import eu.dnetlib.dhp.parser.utility.VtdUtilityParser; import eu.dnetlib.dhp.parser.utility.VtdUtilityParser.Node; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.scholexplorer.DLIDataset; import eu.dnetlib.dhp.schema.scholexplorer.ProvenaceInfo; @@ -281,11 +282,7 @@ public class DatasetScholexplorerParser extends AbstractScholexplorerParser { t -> { final StructuredProperty st = new StructuredProperty(); st.setValue(t); - st - .setQualifier( - generateQualifier( - "main title", "main title", "dnet:dataCite_title", - "dnet:dataCite_title")); + st.setQualifier(ModelConstants.MAIN_TITLE_QUALIFIER); return st; }) .collect(Collectors.toList())); diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala index 29d75cdbc..a00a7996f 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala @@ -159,7 +159,7 @@ object DLIToOAF { result.setUrl(e.url) result.setRefidentifier(e.pid) result.setDataInfo(generateDataInfo()) - result.setQualifier(createQualifier(e.classId, "dnet:externalReference_typologies")) + result.setQualifier(createQualifier(e.classId, ModelConstants.DNET_EXTERNAL_REF_TYPES)) result }) publication.setExternalReference(eRefs.asJava) @@ -238,7 +238,7 @@ object DLIToOAF { if (inputPublication.getAuthor == null || inputPublication.getAuthor.isEmpty) return null result.setAuthor(inputPublication.getAuthor.asScala.map(convertAuthor).asJava) - result.setResulttype(createQualifier(inputPublication.getResulttype.getClassid, inputPublication.getResulttype.getClassname, "dnet:result_typologies", "dnet:result_typologies")) + result.setResulttype(createQualifier(inputPublication.getResulttype.getClassid, inputPublication.getResulttype.getClassname, ModelConstants.DNET_RESULT_TYPOLOGIES, ModelConstants.DNET_RESULT_TYPOLOGIES)) if (inputPublication.getSubject != null) result.setSubject(inputPublication.getSubject.asScala.map(convertSubject).asJava) @@ -259,7 +259,7 @@ object DLIToOAF { result.setDateofacceptance(asField(inputPublication.getRelevantdate.get(0).getValue)) result.setPublisher(inputPublication.getPublisher) result.setSource(inputPublication.getSource) - result.setBestaccessright(createAccessRight("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + result.setBestaccessright(createAccessRight(ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE, ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES)) val dois = result.getPid.asScala.filter(p => "doi".equalsIgnoreCase(p.getQualifier.getClassname)).map(p => p.getValue) if (dois.isEmpty) @@ -390,7 +390,7 @@ object DLIToOAF { } def patchTitle(t: StructuredProperty): StructuredProperty = { - t.setQualifier(createQualifier("main title", ModelConstants.DNET_DATACITE_TITLE)) + t.setQualifier(ModelConstants.MAIN_TITLE_QUALIFIER) t } diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java index 84ee013aa..e13bc60eb 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java @@ -18,7 +18,7 @@ public class RelationComparator implements Comparator { static { weights.put(ModelConstants.OUTCOME, 0); weights.put(ModelConstants.SUPPLEMENT, 1); - weights.put(ModelConstants.REVIEWS, 2); + weights.put(ModelConstants.REVIEW, 2); weights.put(ModelConstants.CITATION, 3); weights.put(ModelConstants.AFFILIATION, 4); weights.put(ModelConstants.RELATIONSHIP, 5);