master #59
|
|
@ -13,6 +13,8 @@ import java.util.stream.Collectors;
|
|||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.spark.api.java.function.MapFunction;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
|
||||
import com.github.sisyphsu.dateparser.DateParserUtils;
|
||||
import com.google.common.collect.Lists;
|
||||
|
|
@ -23,8 +25,6 @@ import eu.dnetlib.dhp.schema.common.ModelConstants;
|
|||
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||
import eu.dnetlib.dhp.schema.oaf.*;
|
||||
import me.xuender.unidecode.Unidecode;
|
||||
import org.apache.spark.api.java.function.MapFunction;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
|
||||
public class GraphCleaningFunctions extends CleaningFunctions {
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ object SparkCreateBaselineDataFrame {
|
|||
def requestBaseLineUpdatePage(maxFile: String): List[(String, String)] = {
|
||||
val data = requestPage("https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/")
|
||||
|
||||
val result = data.linesWithSeparators.map(l =>l.stripLineEnd)
|
||||
val result = data.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.filter(l => l.startsWith("<a href="))
|
||||
.map { l =>
|
||||
val end = l.lastIndexOf("\">")
|
||||
|
|
|
|||
|
|
@ -63,7 +63,9 @@ class BioScholixTest extends AbstractVocabularyTest {
|
|||
val records: String = Source
|
||||
.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed_dump"))
|
||||
.mkString
|
||||
val r: List[Oaf] = records.linesWithSeparators.map(l =>l.stripLineEnd).toList
|
||||
val r: List[Oaf] = records.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.toList
|
||||
.map(s => mapper.readValue(s, classOf[PMArticle]))
|
||||
.map(a => PubMedToOaf.convert(a, vocabularies))
|
||||
assertEquals(10, r.size)
|
||||
|
|
@ -175,7 +177,8 @@ class BioScholixTest extends AbstractVocabularyTest {
|
|||
.mkString
|
||||
records.linesWithSeparators.map(l => l.stripLineEnd).foreach(s => assertTrue(s.nonEmpty))
|
||||
|
||||
val result: List[Oaf] = records.linesWithSeparators.map(l =>l.stripLineEnd).toList.flatMap(o => BioDBToOAF.pdbTOOaf(o))
|
||||
val result: List[Oaf] =
|
||||
records.linesWithSeparators.map(l => l.stripLineEnd).toList.flatMap(o => BioDBToOAF.pdbTOOaf(o))
|
||||
|
||||
assertTrue(result.nonEmpty)
|
||||
result.foreach(r => assertNotNull(r))
|
||||
|
|
@ -196,7 +199,8 @@ class BioScholixTest extends AbstractVocabularyTest {
|
|||
.mkString
|
||||
records.linesWithSeparators.map(l => l.stripLineEnd).foreach(s => assertTrue(s.nonEmpty))
|
||||
|
||||
val result: List[Oaf] = records.linesWithSeparators.map(l =>l.stripLineEnd).toList.flatMap(o => BioDBToOAF.uniprotToOAF(o))
|
||||
val result: List[Oaf] =
|
||||
records.linesWithSeparators.map(l => l.stripLineEnd).toList.flatMap(o => BioDBToOAF.uniprotToOAF(o))
|
||||
|
||||
assertTrue(result.nonEmpty)
|
||||
result.foreach(r => assertNotNull(r))
|
||||
|
|
@ -241,7 +245,8 @@ class BioScholixTest extends AbstractVocabularyTest {
|
|||
.mkString
|
||||
records.linesWithSeparators.map(l => l.stripLineEnd).foreach(s => assertTrue(s.nonEmpty))
|
||||
|
||||
val result: List[Oaf] = records.linesWithSeparators.map(l =>l.stripLineEnd).map(s => BioDBToOAF.crossrefLinksToOaf(s)).toList
|
||||
val result: List[Oaf] =
|
||||
records.linesWithSeparators.map(l => l.stripLineEnd).map(s => BioDBToOAF.crossrefLinksToOaf(s)).toList
|
||||
|
||||
assertNotNull(result)
|
||||
assertTrue(result.nonEmpty)
|
||||
|
|
@ -280,10 +285,13 @@ class BioScholixTest extends AbstractVocabularyTest {
|
|||
|
||||
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||
|
||||
val l: List[ScholixResolved] = records.linesWithSeparators.map(l =>l.stripLineEnd).map { input =>
|
||||
val l: List[ScholixResolved] = records.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.map { input =>
|
||||
lazy val json = parse(input)
|
||||
json.extract[ScholixResolved]
|
||||
}.toList
|
||||
}
|
||||
.toList
|
||||
|
||||
val result: List[Oaf] = l.map(s => BioDBToOAF.scholixResolvedToOAF(s))
|
||||
|
||||
|
|
|
|||
|
|
@ -37,12 +37,24 @@ public class SubscriptionUtils {
|
|||
}
|
||||
|
||||
public static boolean verifyDateRange(final long date, final String min, final String max) {
|
||||
|
||||
long from = 0;
|
||||
long to = Long.MAX_VALUE;
|
||||
|
||||
try {
|
||||
return date >= DateUtils.parseDate(min, "yyyy-MM-dd").getTime()
|
||||
&& date < DateUtils.parseDate(max, "yyyy-MM-dd").getTime() + ONE_DAY;
|
||||
from = min != null ? DateUtils.parseDate(min, "yyyy-MM-dd").getTime() : 0;
|
||||
} catch (final ParseException e) {
|
||||
return false;
|
||||
from = 0;
|
||||
}
|
||||
|
||||
try {
|
||||
to = max != null ? DateUtils.parseDate(max, "yyyy-MM-dd").getTime() + ONE_DAY : Long.MAX_VALUE;
|
||||
} catch (final ParseException e) {
|
||||
to = Long.MAX_VALUE;
|
||||
}
|
||||
|
||||
return date >= from && date < to;
|
||||
|
||||
}
|
||||
|
||||
public static boolean verifyExact(final String s1, final String s2) {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,18 @@ public class SubscriptionUtilsTest {
|
|||
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, "2010-01-01", "2011-01-01"));
|
||||
assertFalse(SubscriptionUtils.verifyDateRange(date, "2020-01-01", "2021-01-01"));
|
||||
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, "2010-01-01", "NULL"));
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, "2010-01-01", null));
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, "NULL", "2011-01-01"));
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, null, "2011-01-01"));
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, "NULL", "NULL"));
|
||||
assertTrue(SubscriptionUtils.verifyDateRange(date, null, null));
|
||||
|
||||
assertFalse(SubscriptionUtils.verifyDateRange(date, "2020-01-01", null));
|
||||
assertFalse(SubscriptionUtils.verifyDateRange(date, "2020-01-01", "NULL"));
|
||||
assertFalse(SubscriptionUtils.verifyDateRange(date, null, "2005-01-01"));
|
||||
assertFalse(SubscriptionUtils.verifyDateRange(date, "NULL", "2005-01-01"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -309,6 +309,8 @@ case object Crossref2Oaf {
|
|||
result
|
||||
}
|
||||
|
||||
|
||||
|
||||
def generateAuhtor(given: String, family: String, orcid: String, index: Int): Author = {
|
||||
val a = new Author
|
||||
a.setName(given)
|
||||
|
|
@ -370,10 +372,57 @@ case object Crossref2Oaf {
|
|||
case dataset: Dataset => convertDataset(dataset)
|
||||
}
|
||||
|
||||
|
||||
val doisReference:List[String] = for {
|
||||
JObject(reference_json) <- json \ "reference"
|
||||
JField("DOI", JString(doi_json)) <- reference_json
|
||||
} yield doi_json
|
||||
|
||||
|
||||
|
||||
if (doisReference!= null && doisReference.nonEmpty) {
|
||||
val citation_relations:List[Relation] = generateCitationRelations(doisReference, result)
|
||||
resultList = resultList ::: citation_relations
|
||||
}
|
||||
resultList = resultList ::: List(result)
|
||||
resultList
|
||||
}
|
||||
|
||||
|
||||
|
||||
private def createCiteRelation(source:Result, targetPid:String, targetPidType:String) :List[Relation] = {
|
||||
|
||||
|
||||
val targetId = IdentifierFactory.idFromPid("50",targetPidType, targetPid, true)
|
||||
|
||||
val from = new Relation
|
||||
from.setSource(source.getId)
|
||||
from.setTarget(targetId)
|
||||
from.setRelType(ModelConstants.RESULT_RESULT)
|
||||
from.setRelClass(ModelConstants.CITES)
|
||||
from.setSubRelType(ModelConstants.CITATION)
|
||||
from.setCollectedfrom(source.getCollectedfrom)
|
||||
from.setDataInfo(source.getDataInfo)
|
||||
from.setLastupdatetimestamp(source.getLastupdatetimestamp)
|
||||
|
||||
|
||||
val to = new Relation
|
||||
to.setTarget(source.getId)
|
||||
to.setSource(targetId)
|
||||
to.setRelType(ModelConstants.RESULT_RESULT)
|
||||
to.setRelClass(ModelConstants.IS_CITED_BY)
|
||||
to.setSubRelType(ModelConstants.CITATION)
|
||||
to.setCollectedfrom(source.getCollectedfrom)
|
||||
to.setDataInfo(source.getDataInfo)
|
||||
to.setLastupdatetimestamp(source.getLastupdatetimestamp)
|
||||
|
||||
List(from,to)
|
||||
}
|
||||
|
||||
def generateCitationRelations(dois:List[String], result:Result):List[Relation] = {
|
||||
dois.flatMap(d => createCiteRelation(result, d, "doi"))
|
||||
}
|
||||
|
||||
def mappingFunderToRelations(
|
||||
funders: List[mappingFunder],
|
||||
sourceId: String,
|
||||
|
|
|
|||
|
|
@ -94,7 +94,6 @@
|
|||
"family": "Stein",
|
||||
"sequence": "first",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -104,7 +103,6 @@
|
|||
"family": "Velzen",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -114,7 +112,6 @@
|
|||
"family": "Kowalski",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -122,7 +119,6 @@
|
|||
"family": "Franckowiak",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -132,7 +128,6 @@
|
|||
"family": "Gezari",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -142,7 +137,6 @@
|
|||
"family": "Miller-Jones",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -150,7 +144,6 @@
|
|||
"family": "Frederick",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -160,7 +153,6 @@
|
|||
"family": "Sfaradi",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -168,7 +160,6 @@
|
|||
"family": "Bietenholz",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -178,7 +169,6 @@
|
|||
"family": "Horesh",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -186,7 +176,6 @@
|
|||
"family": "Fender",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -196,7 +185,6 @@
|
|||
"family": "Garrappa",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -206,7 +194,6 @@
|
|||
"family": "Ahumada",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -214,7 +201,6 @@
|
|||
"family": "Andreoni",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -222,7 +208,6 @@
|
|||
"family": "Belicki",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -232,7 +217,6 @@
|
|||
"family": "Bellm",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -240,7 +224,6 @@
|
|||
"family": "Böttcher",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -248,7 +231,6 @@
|
|||
"family": "Brinnel",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -256,7 +238,6 @@
|
|||
"family": "Burruss",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -266,7 +247,6 @@
|
|||
"family": "Cenko",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -276,7 +256,6 @@
|
|||
"family": "Coughlin",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -286,7 +265,6 @@
|
|||
"family": "Cunningham",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -294,7 +272,6 @@
|
|||
"family": "Drake",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -302,7 +279,6 @@
|
|||
"family": "Farrar",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -310,7 +286,6 @@
|
|||
"family": "Feeney",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -318,7 +293,6 @@
|
|||
"family": "Foley",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -328,7 +302,6 @@
|
|||
"family": "Gal-Yam",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -336,7 +309,6 @@
|
|||
"family": "Golkhou",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -346,7 +318,6 @@
|
|||
"family": "Goobar",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -356,7 +327,6 @@
|
|||
"family": "Graham",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -364,7 +334,6 @@
|
|||
"family": "Hammerstein",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -374,7 +343,6 @@
|
|||
"family": "Helou",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -384,7 +352,6 @@
|
|||
"family": "Hung",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -392,7 +359,6 @@
|
|||
"family": "Kasliwal",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -402,7 +368,6 @@
|
|||
"family": "Kilpatrick",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -412,7 +377,6 @@
|
|||
"family": "Kong",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -422,7 +386,6 @@
|
|||
"family": "Kupfer",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -432,7 +395,6 @@
|
|||
"family": "Laher",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -442,7 +404,6 @@
|
|||
"family": "Mahabal",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -452,7 +413,6 @@
|
|||
"family": "Masci",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -462,7 +422,6 @@
|
|||
"family": "Necker",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -472,7 +431,6 @@
|
|||
"family": "Nordin",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -480,7 +438,6 @@
|
|||
"family": "Perley",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -490,7 +447,6 @@
|
|||
"family": "Rigault",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -500,7 +456,6 @@
|
|||
"family": "Reusch",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -508,7 +463,6 @@
|
|||
"family": "Rodriguez",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -518,7 +472,6 @@
|
|||
"family": "Rojas-Bravo",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -528,7 +481,6 @@
|
|||
"family": "Rusholme",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -538,7 +490,6 @@
|
|||
"family": "Shupe",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -548,7 +499,6 @@
|
|||
"family": "Singer",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -558,7 +508,6 @@
|
|||
"family": "Sollerman",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -566,7 +515,6 @@
|
|||
"family": "Soumagnac",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -574,7 +522,6 @@
|
|||
"family": "Stern",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -582,7 +529,6 @@
|
|||
"family": "Taggart",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -590,7 +536,6 @@
|
|||
"family": "van Santen",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -598,7 +543,6 @@
|
|||
"family": "Ward",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -606,7 +550,6 @@
|
|||
"family": "Woudt",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -616,7 +559,6 @@
|
|||
"family": "Yao",
|
||||
"sequence": "additional",
|
||||
"affiliation": [
|
||||
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -1412,7 +1354,6 @@
|
|||
"Nature Astronomy"
|
||||
],
|
||||
"original-title": [
|
||||
|
||||
],
|
||||
"language": "en",
|
||||
"link": [
|
||||
|
|
@ -1448,10 +1389,8 @@
|
|||
},
|
||||
"score": 1.0,
|
||||
"subtitle": [
|
||||
|
||||
],
|
||||
"short-title": [
|
||||
|
||||
],
|
||||
"issued": {
|
||||
"date-parts": [
|
||||
|
|
@ -1480,7 +1419,6 @@
|
|||
"URL": "http://dx.doi.org/10.1038/s41550-020-01295-8",
|
||||
"relation": {
|
||||
"cites": [
|
||||
|
||||
]
|
||||
},
|
||||
"ISSN": [
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
package eu.dnetlib.dhp.doiboost.crossref
|
||||
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants
|
||||
import eu.dnetlib.dhp.schema.oaf._
|
||||
import eu.dnetlib.dhp.utils.DHPUtils
|
||||
import eu.dnetlib.doiboost.crossref.Crossref2Oaf
|
||||
import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig}
|
||||
import org.json4s
|
||||
import org.json4s.JsonAST.{JField, JObject, JString}
|
||||
import org.json4s.{DefaultFormats, JValue}
|
||||
import org.json4s.jackson.JsonMethods
|
||||
import org.junit.jupiter.api.Assertions._
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.slf4j.{Logger, LoggerFactory}
|
||||
|
|
@ -109,6 +114,44 @@ class CrossrefMappingTest {
|
|||
|
||||
}
|
||||
|
||||
|
||||
private def parseJson(input:String):JValue = {
|
||||
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||
lazy val json: json4s.JValue = JsonMethods.parse(input)
|
||||
|
||||
json
|
||||
}
|
||||
|
||||
@Test
|
||||
def testCitationRelations():Unit = {
|
||||
val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_embargo.json")).mkString
|
||||
|
||||
|
||||
assertNotNull(json)
|
||||
assertFalse(json.isEmpty)
|
||||
|
||||
val result:List[Oaf] = Crossref2Oaf.convert(json)
|
||||
|
||||
assertTrue(result.nonEmpty)
|
||||
|
||||
|
||||
val j = parseJson(json)
|
||||
|
||||
val doisReference: List[String] = for {
|
||||
JObject(reference_json) <- j \ "reference"
|
||||
JField("DOI", JString(doi_json)) <- reference_json
|
||||
} yield doi_json
|
||||
|
||||
|
||||
|
||||
val relationList:List[Relation] = result.filter(s => s.isInstanceOf[Relation]).map(r=> r.asInstanceOf[Relation]).filter(r => r.getSubRelType.equalsIgnoreCase(ModelConstants.CITATION))
|
||||
|
||||
assertNotNull(relationList)
|
||||
assertFalse(relationList.isEmpty)
|
||||
|
||||
assertEquals(doisReference.size*2, relationList.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
def testEmptyTitle(): Unit = {
|
||||
val json = Source
|
||||
|
|
|
|||
|
|
@ -25,7 +25,9 @@ class MappingORCIDToOAFTest {
|
|||
.mkString
|
||||
assertNotNull(json)
|
||||
assertFalse(json.isEmpty)
|
||||
json.linesWithSeparators.map(l =>l.stripLineEnd).foreach(s => {
|
||||
json.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.foreach(s => {
|
||||
assertNotNull(ORCIDToOAF.extractValueFromInputString(s))
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import java.io.IOException;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
|
|
@ -27,6 +26,7 @@ import org.slf4j.LoggerFactory;
|
|||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.clean.country.CleanCountrySparkJob;
|
||||
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||
import eu.dnetlib.dhp.schema.oaf.Publication;
|
||||
|
||||
public class CleanCountryTest {
|
||||
|
|
@ -185,7 +185,6 @@ public class CleanCountryTest {
|
|||
|
||||
Assertions.assertEquals(0, tmp.first().getCountry().size());
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@ class ResolveEntitiesTest extends Serializable {
|
|||
def generateUpdates(spark: SparkSession): Unit = {
|
||||
val template = Source.fromInputStream(this.getClass.getResourceAsStream("updates")).mkString
|
||||
|
||||
val pids: List[String] = template.linesWithSeparators.map(l =>l.stripLineEnd)
|
||||
val pids: List[String] = template.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.map { id =>
|
||||
val r = new Result
|
||||
r.setId(id.toLowerCase.trim)
|
||||
|
|
@ -264,7 +265,8 @@ class ResolveEntitiesTest extends Serializable {
|
|||
Source
|
||||
.fromInputStream(this.getClass.getResourceAsStream(s"publication"))
|
||||
.mkString
|
||||
.linesWithSeparators.map(l =>l.stripLineEnd)
|
||||
.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.next(),
|
||||
classOf[Publication]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -69,7 +69,8 @@ class ScholixGraphTest extends AbstractVocabularyTest {
|
|||
getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/merge_result_scholix")
|
||||
)
|
||||
.mkString
|
||||
val result: List[(Relation, ScholixSummary)] = inputRelations.linesWithSeparators.map(l =>l.stripLineEnd)
|
||||
val result: List[(Relation, ScholixSummary)] = inputRelations.linesWithSeparators
|
||||
.map(l => l.stripLineEnd)
|
||||
.sliding(2)
|
||||
.map(s => (s.head, s(1)))
|
||||
.map(p => (mapper.readValue(p._1, classOf[Relation]), mapper.readValue(p._2, classOf[ScholixSummary])))
|
||||
|
|
|
|||
Loading…
Reference in New Issue