From 8c22e5c30a45272196da8a15dc41e408bfd551e7 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Tue, 26 Apr 2022 09:56:27 +0200 Subject: [PATCH] added fix to include date array with only year or year and month --- .../doiboost/crossref/Crossref2Oaf.scala | 13 +- .../dnetlib/doiboost/crossref/issue_date.json | 330 ++++++++++++++++++ .../crossref/CrossrefMappingTest.scala | 16 +- 3 files changed, 355 insertions(+), 4 deletions(-) create mode 100644 dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/issue_date.json diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index b4d6d67e2..0ba77ca84 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -2,7 +2,7 @@ package eu.dnetlib.doiboost.crossref import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf._ -import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils} +import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, IdentifierFactory, OafMapperUtils} import eu.dnetlib.dhp.utils.DHPUtils import eu.dnetlib.doiboost.DoiBoostMappingUtil import eu.dnetlib.doiboost.DoiBoostMappingUtil._ @@ -576,15 +576,22 @@ case object Crossref2Oaf { def extractDate(dt: String, datePart: List[List[Int]]): String = { if (StringUtils.isNotBlank(dt)) - return dt + return GraphCleaningFunctions.cleanDate(dt) if (datePart != null && datePart.size == 1) { val res = datePart.head if (res.size == 3) { val dp = f"${res.head}-${res(1)}%02d-${res(2)}%02d" if (dp.length == 10) { - return dp + return GraphCleaningFunctions.cleanDate(dp) } } + else if (res.size ==2) { + val dp = f"${res.head}-${res(1)}%02d-01" + return GraphCleaningFunctions.cleanDate(dp) + } + else if (res.size ==1) { + return GraphCleaningFunctions.cleanDate(s"${res.head}-01-01") + } } null diff --git a/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/issue_date.json b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/issue_date.json new file mode 100644 index 000000000..d74b2c167 --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/issue_date.json @@ -0,0 +1,330 @@ +{ + "indexed":{ + "date-parts":[ + [ + 2022, + 4, + 14 + ] + ], + "date-time":"2022-04-14T11:27:30Z", + "timestamp":1649935650109 + }, + "reference-count":22, + "publisher":"SAGE Publications", + "issue":"2", + "license":[ + { + "start":{ + "date-parts":[ + [ + 1980, + 4, + 1 + ] + ], + "date-time":"1980-04-01T00:00:00Z", + "timestamp":323395200000 + }, + "content-version":"tdm", + "delay-in-days":0, + "URL":"http:\/\/journals.sagepub.com\/page\/policies\/text-and-data-mining-license" + } + ], + "content-domain":{ + "domain":[ + + ], + "crossmark-restriction":false + }, + "short-container-title":[ + "Perception" + ], + "published-print":{ + "date-parts":[ + [ + 1980, + 4 + ] + ] + }, + "abstract":" To answer the question \u2018What is suppressed during binocular rivalry?\u2019 a series of three experiments was performed. In the first experiment observers viewed binocular rivalry between orthogonally oriented patterns. When the dominant and suppressed patterns were interchanged between the eyes observers continued seeing with the dominant eye, indicating that an eye, not a pattern, is suppressed during rivalry. In a second experiment it was found that a suppressed eye was able to contribute to stereopsis. A third experiment demonstrated that the predominance of an eye could be influenced by prior adaptation of the other eye, indicating that binocular mechanisms participate in the rivalry process. <\/jats:p>", + "DOI":"10.1068\/p090223", + "type":"journal-article", + "created":{ + "date-parts":[ + [ + 2007, + 1, + 23 + ] + ], + "date-time":"2007-01-23T15:21:36Z", + "timestamp":1169565696000 + }, + "page":"223-231", + "source":"Crossref", + "is-referenced-by-count":123, + "title":[ + "What is Suppressed during Binocular Rivalry?" + ], + "prefix":"10.1177", + "volume":"9", + "author":[ + { + "given":"Randolph", + "family":"Blake", + "sequence":"first", + "affiliation":[ + { + "name":"Cresap Neuroscience Laboratory, Northwestern University, Evanston, Illinois 60201, USA" + } + ] + }, + { + "given":"David H", + "family":"Westendorf", + "sequence":"additional", + "affiliation":[ + { + "name":"Department of Psychology, University of Arkansas, Fayetteville, Arkansas 72701, USA" + } + ] + }, + { + "given":"Randall", + "family":"Overton", + "sequence":"additional", + "affiliation":[ + { + "name":"Department of Psychology, Illinois State University, Normal, Illinois 61761, USA" + } + ] + } + ], + "member":"179", + "published-online":{ + "date-parts":[ + [ + 2016, + 6, + 25 + ] + ] + }, + "reference":[ + { + "key":"bibr1-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1136\/bjo.37.1.37" + }, + { + "key":"bibr2-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1037\/0096-1523.5.2.315" + }, + { + "key":"bibr3-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1016\/0042-6989(74)90065-0" + }, + { + "key":"bibr4-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1068\/p080143" + }, + { + "key":"bibr5-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1016\/0042-6989(70)90036-2" + }, + { + "key":"bibr6-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1113\/jphysiol.1969.sp008862" + }, + { + "key":"bibr7-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1113\/jphysiol.1972.sp010006" + }, + { + "key":"bibr8-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1146\/annurev.ps.23.020172.002213" + }, + { + "key":"bibr9-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1126\/science.166.3902.245" + }, + { + "key":"bibr10-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1037\/h0075805" + }, + { + "key":"bibr11-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1113\/jphysiol.1968.sp008552" + }, + { + "key":"bibr12-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1113\/jphysiol.1965.sp007784" + }, + { + "key":"bibr13-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1037\/h0032455" + }, + { + "key":"bibr14-p090223", + "volume-title":"Treatise on Physiological Optics", + "volume":"3", + "author":"von Helmholtz H", + "year":"1866", + "edition":"3" + }, + { + "key":"bibr15-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1068\/p040125" + }, + { + "key":"bibr16-p090223", + "volume-title":"On Binocular Rivalry", + "author":"Levelt W J M", + "year":"1965" + }, + { + "key":"bibr17-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1001\/archopht.1935.00840020011001" + }, + { + "key":"bibr18-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.3758\/BF03205796" + }, + { + "key":"bibr19-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.3758\/BF03210180" + }, + { + "key":"bibr20-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1037\/0033-2909.85.2.376" + }, + { + "key":"bibr21-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.1016\/0042-6989(79)90169-X" + }, + { + "key":"bibr22-p090223", + "doi-asserted-by":"publisher", + "DOI":"10.3758\/BF03210465" + } + ], + "container-title":[ + "Perception" + ], + "original-title":[ + + ], + "language":"en", + "link":[ + { + "URL":"http:\/\/journals.sagepub.com\/doi\/pdf\/10.1068\/p090223", + "content-type":"application\/pdf", + "content-version":"vor", + "intended-application":"text-mining" + }, + { + "URL":"http:\/\/journals.sagepub.com\/doi\/pdf\/10.1068\/p090223", + "content-type":"unspecified", + "content-version":"vor", + "intended-application":"similarity-checking" + } + ], + "deposited":{ + "date-parts":[ + [ + 2021, + 12, + 3 + ] + ], + "date-time":"2021-12-03T11:49:48Z", + "timestamp":1638532188000 + }, + "score":1, + "resource":{ + "primary":{ + "URL":"http:\/\/journals.sagepub.com\/doi\/10.1068\/p090223" + } + }, + "subtitle":[ + + ], + "short-title":[ + + ], + "issued":{ + "date-parts":[ + [ + 1980, + 4 + ] + ] + }, + "references-count":22, + "journal-issue":{ + "issue":"2", + "published-print":{ + "date-parts":[ + [ + 1980, + 4 + ] + ] + } + }, + "alternative-id":[ + "10.1068\/p090223" + ], + "URL":"http:\/\/dx.doi.org\/10.1068\/p090223", + "relation":{ + + }, + "ISSN":[ + "0301-0066", + "1468-4233" + ], + "issn-type":[ + { + "value":"0301-0066", + "type":"print" + }, + { + "value":"1468-4233", + "type":"electronic" + } + ], + "subject":[ + "Artificial Intelligence", + "Sensory Systems", + "Experimental and Cognitive Psychology", + "Ophthalmology" + ], + "published":{ + "date-parts":[ + [ + 1980, + 4 + ] + ] + } +} \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index 8124a5aae..9025cd6d2 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -73,6 +73,20 @@ class CrossrefMappingTest { } + + @Test + def crossrefIssueDateTest(): Unit = { + val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/issue_date.json")).mkString + assertNotNull(json) + assertFalse(json.isEmpty) + val resultList: List[Oaf] = Crossref2Oaf.convert(json) + assertTrue(resultList.nonEmpty) + + val items = resultList.filter(p => p.isInstanceOf[Result]) + + println(mapper.writeValueAsString(items.head)) + } + @Test def testOrcidID(): Unit = { val json = Source @@ -82,7 +96,7 @@ class CrossrefMappingTest { .mkString assertNotNull(json) - assertFalse(json.isEmpty); + assertFalse(json.isEmpty) val resultList: List[Oaf] = Crossref2Oaf.convert(json)