improved crossref mapping

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala

@@ -2,8 +2,10 @@ package eu.dnetlib.doiboost.crossref
 
 import eu.dnetlib.dhp.schema.oaf._
 import eu.dnetlib.dhp.utils.DHPUtils
+import org.apache.commons.lang.StringUtils
 import org.json4s
 import org.json4s.DefaultFormats
+import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods._
 import org.slf4j.Logger
 
@@ -108,12 +110,58 @@ class Crossref2Oaf {
     result.setLastupdatetimestamp((json \ "indexed" \ "timestamp").extract[Long])
     result.setDateofcollection((json \ "indexed" \ "date-time").extract[String])
 
-    //result.setCollectedfrom()
+    result.setCollectedfrom(List(createCollectedFrom()).asJava)
 
+    // Publisher ( Name of work's publisher mapped into  Result/Publisher)
+    val publisher = (json \ "publisher").extract[String]
+    result.setPublisher(asField(publisher))
+
+    // TITLE
+    val mainTitles = for {JString(title) <- json \ "title"} yield createSP(title, "main title", "dnet:dataCite_title")
+    val originalTitles = for {JString(title) <- json \ "original-title"} yield createSP(title, "alternative title", "dnet:dataCite_title")
+    val shortTitles = for {JString(title) <- json \ "short-title"} yield createSP(title, "alternative title", "dnet:dataCite_title")
+    result.setTitle((mainTitles ::: originalTitles ::: shortTitles).asJava)
+
+    // DESCRIPTION
+    val descriptionList = for {JString(description) <- json \ "abstract"} yield asField(description)
+    result.setDescription(descriptionList.asJava)
+    // Source
+    val sourceList = for {JString(source) <- json \ "source"} yield asField(source)
+
+    result.setSource(sourceList.asJava)
+
+
+    //RELEVANT DATE Mapping
+    val createdDate =generateDate((json \ "created" \"date-time").extract[String],(json \ "created"\"date-parts").extract[List[List[Int]]],"created", "dnet:dataCite_date" )
+    val postedDate =generateDate((json \ "posted" \"date-time").extractOrElse[String](null),(json \ "posted"\"date-parts").extract[List[List[Int]]],"available", "dnet:dataCite_date" )
+    val acceptedDate =generateDate((json \ "accepted" \"date-time").extractOrElse[String](null),(json \ "accepted"\"date-parts").extract[List[List[Int]]],"accepted", "dnet:dataCite_date" )
+    val publishedPrintDate =generateDate((json \ "published-print" \"date-time").extractOrElse[String](null),(json \ "published-print"\"date-parts").extract[List[List[Int]]],"published-print", "dnet:dataCite_date" )
+    val publishedOnlineDate =generateDate((json \ "published-online" \"date-time").extractOrElse[String](null),(json \ "published-online"\"date-parts").extract[List[List[Int]]],"published-online", "dnet:dataCite_date" )
+
+    result.setRelevantdate(List(createdDate ,postedDate, acceptedDate,publishedOnlineDate, publishedPrintDate).asJava)
     result
   }
 
 
+  def generateDate(dt: String, datePart: List[List[Int]], classId: String, schemeId: String): StructuredProperty = {
+    if (StringUtils.isNotBlank(dt))
+      return createSP(dt, classId, schemeId)
+
+
+    if (datePart != null && datePart.size == 1) {
+      val res = datePart.head
+      if (res.size == 3) {
+        val dp = f"${res.head}-${res(1)}%02d-${res(2)}%02d"
+        println(dp)
+        if (dp.length == 10) {
+          return createSP(dp, classId, schemeId)
+        }
+      }
+    }
+    null
+  }
+
+
     def generateIdentifier(oaf: Result, doi: String): String = {
       val id = DHPUtils.md5(doi.toLowerCase)
       if (oaf.isInstanceOf[Dataset])
@@ -121,6 +169,15 @@ class Crossref2Oaf {
       s"50|${doiBoostNSPREFIX}${SEPARATOR}${id}"
     }
 
+    def asField[T](value: T): Field[T] = {
+      val tmp = new Field[T]
+      tmp.setValue(value)
+      tmp
+
+
+    }
+
+
     def generateDataInfo(): DataInfo = {
       val di = new DataInfo
       di.setDeletedbyinference(false)
@@ -140,6 +197,15 @@ class Crossref2Oaf {
 
     }
 
+    def createCollectedFrom(): KeyValue = {
+
+      val cf = new KeyValue
+      cf.setValue(CROSSREF)
+      cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5("crossref"))
+      cf
+
+    }
+
     def createQualifier(cls: String, sch: String): Qualifier = {
       val q = new Qualifier
       q.setClassid(cls)

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid/oozie_app/workflow.xml

@@ -1,4 +1,4 @@
-<workflow-app name="import Crossref from index into HDFS" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="import ORCID from index into HDFS" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>workingPath</name>

dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/DoiBoostTest.java

@@ -1,68 +1,107 @@
 package eu.dnetlib.doiboost;
 
+import static org.junit.jupiter.api.Assertions.*;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.jayway.jsonpath.JsonPath;
 import eu.dnetlib.dhp.schema.oaf.Result;
 import eu.dnetlib.doiboost.crossref.Crossref2Oaf;
 import eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF;
+import java.io.IOException;
+import java.util.List;
 import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.platform.commons.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
-import java.io.IOException;
-import java.util.List;
-
 public class DoiBoostTest {
 
     Logger logger = LoggerFactory.getLogger(DoiBoostTest.class);
 
-
     public void test() throws Exception {
 
         // SparkDownloadContentFromCrossref.main(null);
-        //CrossrefImporter.main("-n file:///tmp -t file:///tmp/p.seq -ts 1586110000749".split(" "));
+        // CrossrefImporter.main("-n file:///tmp -t file:///tmp/p.seq -ts 1586110000749".split("
-        SparkMapDumpIntoOAF.main("-m local[*] -s file:///data/doiboost/crossref_dump.seq".split(" "));
+        // "));
+        SparkMapDumpIntoOAF.main(
+                "-m local[*] -s file:///data/doiboost/crossref_dump.seq".split(" "));
     }
 
-
-
     @Test
     public void testConvertCrossRef2Oaf() throws IOException {
 
         final String json = IOUtils.toString(getClass().getResourceAsStream("pc.json"));
         ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
-        Assertions.assertNotNull(json);
+        assertNotNull(json);
-        Assertions.assertFalse(StringUtils.isBlank(json));
+        assertFalse(StringUtils.isBlank(json));
-
-
-
         Crossref2Oaf cf = new Crossref2Oaf();
         final Result result = cf.convert(json, logger);
-        Assertions.assertNotNull(result);
+        assertNotNull(result);
+
+        assertNotNull(result.getDataInfo(), "Datainfo test not null Failed");
+        assertNotNull(
+                result.getDataInfo().getProvenanceaction(),
+                "DataInfo/Provenance test not null Failed");
+        assertFalse(
+                StringUtils.isBlank(result.getDataInfo().getProvenanceaction().getClassid()),
+                "DataInfo/Provenance/classId test not null Failed");
+        assertFalse(
+                StringUtils.isBlank(result.getDataInfo().getProvenanceaction().getClassname()),
+                "DataInfo/Provenance/className test not null Failed");
+        assertFalse(
+                StringUtils.isBlank(result.getDataInfo().getProvenanceaction().getSchemeid()),
+                "DataInfo/Provenance/SchemeId test not null Failed");
+        assertFalse(
+                StringUtils.isBlank(result.getDataInfo().getProvenanceaction().getSchemename()),
+                "DataInfo/Provenance/SchemeName test not null Failed");
+
+        assertNotNull(result.getCollectedfrom(), "CollectedFrom test not null Failed");
+        assertTrue(result.getCollectedfrom().size() > 0);
+        assertTrue(
+                result.getCollectedfrom().stream()
+                        .anyMatch(
+                                c ->
+                                        c.getKey()
+                                                .equalsIgnoreCase(
+                                                        "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")));
+        assertTrue(
+                result.getCollectedfrom().stream()
+                        .anyMatch(c -> c.getValue().equalsIgnoreCase("crossref")));
+
+        assertTrue(
+                result.getRelevantdate().stream()
+                        .anyMatch(d -> d.getQualifier().getClassid().equalsIgnoreCase("created")));
+        assertTrue(
+                result.getRelevantdate().stream()
+                        .anyMatch(
+                                d -> d.getQualifier().getClassid().equalsIgnoreCase("available")));
+        assertTrue(
+                result.getRelevantdate().stream()
+                        .anyMatch(d -> d.getQualifier().getClassid().equalsIgnoreCase("accepted")));
+        assertTrue(
+                result.getRelevantdate().stream()
+                        .anyMatch(
+                                d ->
+                                        d.getQualifier()
+                                                .getClassid()
+                                                .equalsIgnoreCase("published-online")));
+        assertTrue(
+                result.getRelevantdate().stream()
+                        .anyMatch(
+                                d ->
+                                        d.getQualifier()
+                                                .getClassid()
+                                                .equalsIgnoreCase("published-print")));
 
         logger.info(mapper.writeValueAsString(result));
-
     }
 
-
-
-
-
     @Test
     public void testPath() throws Exception {
         final String json = IOUtils.toString(getClass().getResourceAsStream("response.json"));
         final List<String> res = JsonPath.read(json, "$.hits.hits[*]._source.blob");
         System.out.println(res.size());
-
     }
-
-
-
-
-
 }

dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/pc.json

@@ -112,9 +112,35 @@
       ]
     ]
   },
+  "published-print": {
+    "timestamp": 1446095513000,
+    "date-time": "2015-10-29T05:11:53Z",
+    "date-parts": [
+      [
+        2015,
+        2,
+        29
+      ]
+    ]
+  },
+  "published-online": {
+        "date-parts": [
+      [
+        2015,
+        2,
+        2
+      ]
+    ]
+  },
   "title": [
     "Genetic transformation of micropropagated shoots ofPinus radiataD.Don"
   ],
+  "original-title": [
+    "OR TITLE"
+  ],
+  "short-title": [
+    "SHORT TITLE"
+  ],
   "group-title": "Plant Biology",
   "subtype": "preprint"
 }