master #59

Closed
claudio.atzori wants to merge 3221 commits from master into stable_ids
6 changed files with 1578 additions and 1542 deletions
Showing only changes of commit 288ec0b7d6 - Show all commits

View File

@ -554,7 +554,7 @@ public class PublicationToOaf implements Serializable {
private KeyValue createCollectedFrom() { private KeyValue createCollectedFrom() {
KeyValue cf = new KeyValue(); KeyValue cf = new KeyValue();
cf.setValue(ModelConstants.ORCID.toUpperCase()); cf.setValue(ModelConstants.ORCID.toUpperCase());
cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "806360c771262b4d6770e7cdf04b5c5a"); cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "cd0f74b5955dc87fd0605745c4b49ee8");
return cf; return cf;
} }

View File

@ -391,6 +391,28 @@ object DoiBoostMappingUtil {
di di
} }
def createSubject(value: String, classId: String, schemeId: String): Subject = {
val s = new Subject
s.setQualifier(OafMapperUtils.qualifier(classId, classId, schemeId, schemeId))
s.setValue(value)
s
}
def createSubject(
value: String,
classId: String,
className: String,
schemeId: String,
schemeName: String
): Subject = {
val s = new Subject
s.setQualifier(OafMapperUtils.qualifier(classId, className, schemeId, schemeName))
s.setValue(value)
s
}
def createSP( def createSP(
value: String, value: String,
classId: String, classId: String,

View File

@ -201,7 +201,7 @@ case object Crossref2Oaf {
if (subjectList.nonEmpty) { if (subjectList.nonEmpty) {
result.setSubject( result.setSubject(
subjectList.map(s => createSP(s, "keywords", ModelConstants.DNET_SUBJECT_TYPOLOGIES)).asJava subjectList.map(s => createSubject(s, "keyword", ModelConstants.DNET_SUBJECT_TYPOLOGIES)).asJava
) )
} }
@ -370,10 +370,40 @@ case object Crossref2Oaf {
case dataset: Dataset => convertDataset(dataset) case dataset: Dataset => convertDataset(dataset)
} }
val doisReference: List[String] = for {
JObject(reference_json) <- json \ "reference"
JField("DOI", JString(doi_json)) <- reference_json
} yield doi_json
if (doisReference != null && doisReference.nonEmpty) {
val citation_relations: List[Relation] = generateCitationRelations(doisReference, result)
resultList = resultList ::: citation_relations
}
resultList = resultList ::: List(result) resultList = resultList ::: List(result)
resultList resultList
} }
private def createCiteRelation(source: Result, targetPid: String, targetPidType: String): List[Relation] = {
val targetId = IdentifierFactory.idFromPid("50", targetPidType, targetPid, true)
val from = new Relation
from.setSource(source.getId)
from.setTarget(targetId)
from.setRelType(ModelConstants.RESULT_RESULT)
from.setRelClass(ModelConstants.CITES)
from.setSubRelType(ModelConstants.CITATION)
from.setCollectedfrom(source.getCollectedfrom)
from.setDataInfo(source.getDataInfo)
from.setLastupdatetimestamp(source.getLastupdatetimestamp)
List(from)
}
def generateCitationRelations(dois: List[String], result: Result): List[Relation] = {
dois.flatMap(d => createCiteRelation(result, d, "doi"))
}
def mappingFunderToRelations( def mappingFunderToRelations(
funders: List[mappingFunder], funders: List[mappingFunder],
sourceId: String, sourceId: String,

View File

@ -2,7 +2,7 @@ package eu.dnetlib.doiboost.mag
import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.common.ModelConstants
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty} import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty, Subject}
import eu.dnetlib.doiboost.DoiBoostMappingUtil import eu.dnetlib.doiboost.DoiBoostMappingUtil
import eu.dnetlib.doiboost.DoiBoostMappingUtil._ import eu.dnetlib.doiboost.DoiBoostMappingUtil._
import org.json4s import org.json4s
@ -210,8 +210,8 @@ case object ConversionUtil {
val className = "Microsoft Academic Graph classification" val className = "Microsoft Academic Graph classification"
val classid = "MAG" val classid = "MAG"
val p: List[StructuredProperty] = fieldOfStudy.subjects.flatMap(s => { val p: List[Subject] = fieldOfStudy.subjects.flatMap(s => {
val s1 = createSP( val s1 = createSubject(
s.DisplayName, s.DisplayName,
classid, classid,
className, className,
@ -219,10 +219,10 @@ case object ConversionUtil {
ModelConstants.DNET_SUBJECT_TYPOLOGIES ModelConstants.DNET_SUBJECT_TYPOLOGIES
) )
val di = DoiBoostMappingUtil.generateDataInfo(s.Score.toString) val di = DoiBoostMappingUtil.generateDataInfo(s.Score.toString)
var resList: List[StructuredProperty] = List(s1) var resList: List[Subject] = List(s1)
if (s.MainType.isDefined) { if (s.MainType.isDefined) {
val maintp = s.MainType.get val maintp = s.MainType.get
val s2 = createSP( val s2 = createSubject(
s.MainType.get, s.MainType.get,
classid, classid,
className, className,
@ -232,7 +232,7 @@ case object ConversionUtil {
s2.setDataInfo(di) s2.setDataInfo(di)
resList = resList ::: List(s2) resList = resList ::: List(s2)
if (maintp.contains(".")) { if (maintp.contains(".")) {
val s3 = createSP( val s3 = createSubject(
maintp.split("\\.").head, maintp.split("\\.").head,
classid, classid,
className, className,

View File

@ -94,7 +94,6 @@
"family": "Stein", "family": "Stein",
"sequence": "first", "sequence": "first",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -104,7 +103,6 @@
"family": "Velzen", "family": "Velzen",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -114,7 +112,6 @@
"family": "Kowalski", "family": "Kowalski",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -122,7 +119,6 @@
"family": "Franckowiak", "family": "Franckowiak",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -132,7 +128,6 @@
"family": "Gezari", "family": "Gezari",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -142,7 +137,6 @@
"family": "Miller-Jones", "family": "Miller-Jones",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -150,7 +144,6 @@
"family": "Frederick", "family": "Frederick",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -160,7 +153,6 @@
"family": "Sfaradi", "family": "Sfaradi",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -168,7 +160,6 @@
"family": "Bietenholz", "family": "Bietenholz",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -178,7 +169,6 @@
"family": "Horesh", "family": "Horesh",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -186,7 +176,6 @@
"family": "Fender", "family": "Fender",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -196,7 +185,6 @@
"family": "Garrappa", "family": "Garrappa",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -206,7 +194,6 @@
"family": "Ahumada", "family": "Ahumada",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -214,7 +201,6 @@
"family": "Andreoni", "family": "Andreoni",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -222,7 +208,6 @@
"family": "Belicki", "family": "Belicki",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -232,7 +217,6 @@
"family": "Bellm", "family": "Bellm",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -240,7 +224,6 @@
"family": "Böttcher", "family": "Böttcher",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -248,7 +231,6 @@
"family": "Brinnel", "family": "Brinnel",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -256,7 +238,6 @@
"family": "Burruss", "family": "Burruss",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -266,7 +247,6 @@
"family": "Cenko", "family": "Cenko",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -276,7 +256,6 @@
"family": "Coughlin", "family": "Coughlin",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -286,7 +265,6 @@
"family": "Cunningham", "family": "Cunningham",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -294,7 +272,6 @@
"family": "Drake", "family": "Drake",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -302,7 +279,6 @@
"family": "Farrar", "family": "Farrar",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -310,7 +286,6 @@
"family": "Feeney", "family": "Feeney",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -318,7 +293,6 @@
"family": "Foley", "family": "Foley",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -328,7 +302,6 @@
"family": "Gal-Yam", "family": "Gal-Yam",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -336,7 +309,6 @@
"family": "Golkhou", "family": "Golkhou",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -346,7 +318,6 @@
"family": "Goobar", "family": "Goobar",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -356,7 +327,6 @@
"family": "Graham", "family": "Graham",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -364,7 +334,6 @@
"family": "Hammerstein", "family": "Hammerstein",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -374,7 +343,6 @@
"family": "Helou", "family": "Helou",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -384,7 +352,6 @@
"family": "Hung", "family": "Hung",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -392,7 +359,6 @@
"family": "Kasliwal", "family": "Kasliwal",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -402,7 +368,6 @@
"family": "Kilpatrick", "family": "Kilpatrick",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -412,7 +377,6 @@
"family": "Kong", "family": "Kong",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -422,7 +386,6 @@
"family": "Kupfer", "family": "Kupfer",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -432,7 +395,6 @@
"family": "Laher", "family": "Laher",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -442,7 +404,6 @@
"family": "Mahabal", "family": "Mahabal",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -452,7 +413,6 @@
"family": "Masci", "family": "Masci",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -462,7 +422,6 @@
"family": "Necker", "family": "Necker",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -472,7 +431,6 @@
"family": "Nordin", "family": "Nordin",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -480,7 +438,6 @@
"family": "Perley", "family": "Perley",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -490,7 +447,6 @@
"family": "Rigault", "family": "Rigault",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -500,7 +456,6 @@
"family": "Reusch", "family": "Reusch",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -508,7 +463,6 @@
"family": "Rodriguez", "family": "Rodriguez",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -518,7 +472,6 @@
"family": "Rojas-Bravo", "family": "Rojas-Bravo",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -528,7 +481,6 @@
"family": "Rusholme", "family": "Rusholme",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -538,7 +490,6 @@
"family": "Shupe", "family": "Shupe",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -548,7 +499,6 @@
"family": "Singer", "family": "Singer",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -558,7 +508,6 @@
"family": "Sollerman", "family": "Sollerman",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -566,7 +515,6 @@
"family": "Soumagnac", "family": "Soumagnac",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -574,7 +522,6 @@
"family": "Stern", "family": "Stern",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -582,7 +529,6 @@
"family": "Taggart", "family": "Taggart",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -590,7 +536,6 @@
"family": "van Santen", "family": "van Santen",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -598,7 +543,6 @@
"family": "Ward", "family": "Ward",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -606,7 +550,6 @@
"family": "Woudt", "family": "Woudt",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
}, },
{ {
@ -616,7 +559,6 @@
"family": "Yao", "family": "Yao",
"sequence": "additional", "sequence": "additional",
"affiliation": [ "affiliation": [
] ]
} }
], ],
@ -1412,7 +1354,6 @@
"Nature Astronomy" "Nature Astronomy"
], ],
"original-title": [ "original-title": [
], ],
"language": "en", "language": "en",
"link": [ "link": [
@ -1448,10 +1389,8 @@
}, },
"score": 1.0, "score": 1.0,
"subtitle": [ "subtitle": [
], ],
"short-title": [ "short-title": [
], ],
"issued": { "issued": {
"date-parts": [ "date-parts": [
@ -1480,7 +1419,6 @@
"URL": "http://dx.doi.org/10.1038/s41550-020-01295-8", "URL": "http://dx.doi.org/10.1038/s41550-020-01295-8",
"relation": { "relation": {
"cites": [ "cites": [
] ]
}, },
"ISSN": [ "ISSN": [

View File

@ -1,9 +1,14 @@
package eu.dnetlib.dhp.doiboost.crossref package eu.dnetlib.dhp.doiboost.crossref
import eu.dnetlib.dhp.schema.common.ModelConstants
import eu.dnetlib.dhp.schema.oaf._ import eu.dnetlib.dhp.schema.oaf._
import eu.dnetlib.dhp.utils.DHPUtils import eu.dnetlib.dhp.utils.DHPUtils
import eu.dnetlib.doiboost.crossref.Crossref2Oaf import eu.dnetlib.doiboost.crossref.Crossref2Oaf
import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig} import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig}
import org.json4s
import org.json4s.JsonAST.{JField, JObject, JString}
import org.json4s.{DefaultFormats, JValue}
import org.json4s.jackson.JsonMethods
import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.slf4j.{Logger, LoggerFactory} import org.slf4j.{Logger, LoggerFactory}
@ -109,6 +114,47 @@ class CrossrefMappingTest {
} }
private def parseJson(input: String): JValue = {
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
lazy val json: json4s.JValue = JsonMethods.parse(input)
json
}
@Test
def testCitationRelations(): Unit = {
val json = Source
.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_embargo.json"))
.mkString
assertNotNull(json)
assertFalse(json.isEmpty)
val result: List[Oaf] = Crossref2Oaf.convert(json)
assertTrue(result.nonEmpty)
val j = parseJson(json)
val doisReference: List[String] = for {
JObject(reference_json) <- j \ "reference"
JField("DOI", JString(doi_json)) <- reference_json
} yield doi_json
val relationList: List[Relation] = result
.filter(s => s.isInstanceOf[Relation])
.map(r => r.asInstanceOf[Relation])
.filter(r => r.getSubRelType.equalsIgnoreCase(ModelConstants.CITATION))
assertNotNull(relationList)
assertFalse(relationList.isEmpty)
assertEquals(doisReference.size, relationList.size)
mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT)
relationList.foreach(p => println(mapper.writeValueAsString(p)))
}
@Test @Test
def testEmptyTitle(): Unit = { def testEmptyTitle(): Unit = {
val json = Source val json = Source