implemented scholix Flat mapping

dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixFlat.java

@@ -0,0 +1,158 @@
+package eu.dnetlib.dhp.sx.graph.scholix;
+
+import java.util.List;
+
+public class ScholixFlat {
+    private String identifier;
+    private String relationType;
+    private String sourceId;
+    private String sourceType;
+    private String sourceSubType;
+    private List<String> sourcePid;
+    private List<String> sourcePidType;
+    private List<String> sourcePublisher;
+    private String targetId;
+    private String targetType;
+    private String targetSubType;
+    private List<String> targetPid;
+    private List<String> targetPidType;
+    private List<String> targetPublisher;
+    private List<String> linkProviders;
+    private String publicationDate;
+    private String blob;
+
+    public String getIdentifier() {
+        return identifier;
+    }
+
+    public void setIdentifier(String identifier) {
+        this.identifier = identifier;
+    }
+
+    public String getRelationType() {
+        return relationType;
+    }
+
+    public void setRelationType(String relationType) {
+        this.relationType = relationType;
+    }
+    public String getSourceId() {
+        return sourceId;
+    }
+
+    public void setSourceId(String sourceId) {
+        this.sourceId = sourceId;
+    }
+
+    public String getSourceType() {
+        return sourceType;
+    }
+
+    public void setSourceType(String sourceType) {
+        this.sourceType = sourceType;
+    }
+
+    public String getSourceSubType() {
+        return sourceSubType;
+    }
+
+    public void setSourceSubType(String sourceSubType) {
+        this.sourceSubType = sourceSubType;
+    }
+
+    public List<String> getSourcePid() {
+        return sourcePid;
+    }
+
+    public void setSourcePid(List<String> sourcePid) {
+        this.sourcePid = sourcePid;
+    }
+
+    public List<String> getSourcePidType() {
+        return sourcePidType;
+    }
+
+    public void setSourcePidType(List<String> sourcePidType) {
+        this.sourcePidType = sourcePidType;
+    }
+
+    public List<String> getSourcePublisher() {
+        return sourcePublisher;
+    }
+
+    public void setSourcePublisher(List<String> sourcePublisher) {
+        this.sourcePublisher = sourcePublisher;
+    }
+
+    public String getTargetId() {
+        return targetId;
+    }
+
+    public void setTargetId(String targetId) {
+        this.targetId = targetId;
+    }
+
+    public String getTargetType() {
+        return targetType;
+    }
+
+    public void setTargetType(String targetType) {
+        this.targetType = targetType;
+    }
+
+    public String getTargetSubType() {
+        return targetSubType;
+    }
+
+    public void setTargetSubType(String targetSubType) {
+        this.targetSubType = targetSubType;
+    }
+
+    public List<String> getTargetPid() {
+        return targetPid;
+    }
+
+    public void setTargetPid(List<String> targetPid) {
+        this.targetPid = targetPid;
+    }
+
+    public List<String> getTargetPidType() {
+        return targetPidType;
+    }
+
+    public void setTargetPidType(List<String> targetPidType) {
+        this.targetPidType = targetPidType;
+    }
+
+    public List<String> getTargetPublisher() {
+        return targetPublisher;
+    }
+
+    public void setTargetPublisher(List<String> targetPublisher) {
+        this.targetPublisher = targetPublisher;
+    }
+
+    public List<String> getLinkProviders() {
+        return linkProviders;
+    }
+
+    public void setLinkProviders(List<String> linkProviders) {
+        this.linkProviders = linkProviders;
+    }
+
+    public String getPublicationDate() {
+        return publicationDate;
+    }
+
+    public void setPublicationDate(String publicationDate) {
+        this.publicationDate = publicationDate;
+    }
+
+    public String getBlob() {
+        return blob;
+    }
+
+    public void setBlob(String blob) {
+        this.blob = blob;
+    }
+}

dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala

@@ -1,14 +1,15 @@
 package eu.dnetlib.dhp.sx.graph.scholix
 
-import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Result, StructuredProperty}
+import eu.dnetlib.dhp.schema.oaf.{Dataset, OtherResearchProduct, Publication, Relation, Result, Software, StructuredProperty}
 import eu.dnetlib.dhp.schema.sx.scholix._
-import eu.dnetlib.dhp.schema.sx.summary.{CollectedFromType, SchemeValue, ScholixSummary, Typology}
+import eu.dnetlib.dhp.schema.sx.summary.{AuthorPid, CollectedFromType, SchemeValue, ScholixSummary, Typology}
 import eu.dnetlib.dhp.utils.DHPUtils
 import org.apache.spark.sql.expressions.Aggregator
 import org.apache.spark.sql.{Encoder, Encoders}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
+
 import scala.collection.JavaConverters._
 import scala.io.Source
 
@@ -59,6 +60,36 @@ object ScholixUtils extends Serializable {
     }
   }
 
+  def flattenizeScholix(input: Scholix, json: String): ScholixFlat = {
+
+    val flat: ScholixFlat = new ScholixFlat
+    flat.setIdentifier(input.getIdentifier)
+    flat.setRelationType(input.getRelationship.getName)
+    if (input.getSource != null && input.getSource.getIdentifier != null) {
+      flat.setSourceId(input.getSource.getDnetIdentifier)
+      flat.setSourcePid(input.getSource.getIdentifier.asScala.map(p => p.getIdentifier).distinct.toList.asJava)
+      flat.setSourcePidType(input.getSource.getIdentifier.asScala.map(p => p.getSchema).distinct.toList.asJava)
+      flat.setSourceType(input.getSource.getObjectType)
+      flat.setSourceSubType(input.getSource.getObjectSubType)
+    } else return null
+    if (input.getSource.getPublisher != null)
+      flat.setSourcePublisher(input.getSource.getPublisher.asScala.map(p => p.getName).toList.asJava)
+    if (input.getTarget != null && input.getTarget.getIdentifier != null) {
+      flat.setTargetId(input.getTarget.getDnetIdentifier)
+      flat.setTargetPid(input.getTarget.getIdentifier.asScala.map(p => p.getIdentifier).distinct.toList.asJava)
+      flat.setTargetPidType(input.getTarget.getIdentifier.asScala.map(p => p.getSchema).distinct.toList.asJava)
+      flat.setTargetType(input.getTarget.getObjectType)
+      flat.setTargetSubType(input.getTarget.getObjectSubType)
+    } else return null
+    if (input.getTarget.getPublisher != null)
+      flat.setTargetPublisher(input.getTarget.getPublisher.asScala.map(p => p.getName).distinct.toList.asJava)
+    flat.setPublicationDate(input.getPublicationDate)
+    if (input.getLinkprovider != null)
+      flat.setLinkProviders(input.getLinkprovider.asScala.map(l => l.getName).distinct.toList.asJava)
+    flat.setBlob(json);
+    flat
+  }
+
   def inverseRelationShip(rel: ScholixRelationship): ScholixRelationship = {
     new ScholixRelationship(rel.getInverse, rel.getSchema, rel.getName)
 
@@ -232,7 +263,14 @@ object ScholixUtils extends Serializable {
 
     if (summaryObject.getAuthor != null && !summaryObject.getAuthor.isEmpty) {
       val l: List[ScholixEntityId] =
-        summaryObject.getAuthor.asScala.map(a => new ScholixEntityId(a, null)).toList
+        summaryObject.getAuthor.asScala.map(a => {
+          if (a.getORCID != null)
+            new ScholixEntityId(
+              a.getFullname,
+              List(new ScholixIdentifier(a.getORCID, "ORCID", s"https://orcid.org/${a.getORCID}")).asJava
+            )
+          else new ScholixEntityId(a.getFullname, null)
+        }).toList
       if (l.nonEmpty)
         r.setCreator(l.asJava)
     }
@@ -377,10 +415,13 @@ object ScholixUtils extends Serializable {
     if (persistentIdentifiers.isEmpty)
       return null
     s.setLocalIdentifier(persistentIdentifiers.asJava)
-    if (r.isInstanceOf[Publication])
+    r match {
-      s.setTypology(Typology.publication)
+      case _: Publication => s.setTypology(Typology.publication)
-    else
+      case _: Dataset => s.setTypology(Typology.dataset)
-      s.setTypology(Typology.dataset)
+      case _: Software => s.setTypology(Typology.software)
+      case _: OtherResearchProduct => s.setTypology(Typology.otherresearchproduct)
+      case _ =>
+    }
 
     s.setSubType(r.getInstance().get(0).getInstancetype.getClassname)
 
@@ -393,7 +434,20 @@ object ScholixUtils extends Serializable {
     }
 
     if (r.getAuthor != null && !r.getAuthor.isEmpty) {
-      val authors: List[String] = r.getAuthor.asScala.map(a => a.getFullname).toList
+      val authors: List[AuthorPid] = r.getAuthor.asScala
+        .map(a => {
+          var ORCID: String = null
+          if (a.getPid != null) {
+            val result = a.getPid.asScala.find(p =>
+              p.getQualifier != null && p.getQualifier.getClassid != null && p.getQualifier.getClassid.toLowerCase
+                .contains("orcid")
+            )
+            if (result.isDefined)
+              ORCID = result.get.getValue
+          }
+          new AuthorPid(a.getFullname, ORCID)
+        })
+        .toList
       if (authors.nonEmpty)
         s.setAuthor(authors.asJava)
     }

dhp-common/src/test/java/eu/dnetlib/dhp/sx/graph/scholix/ScholixFlatTest.java

@@ -0,0 +1,42 @@
+
+package eu.dnetlib.dhp.sx.graph.scholix;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.*;
+import java.util.zip.GZIPInputStream;
+
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
+
+public class ScholixFlatTest {
+
+	@Test
+	public void flattenScholixTest() throws IOException {
+		final ObjectMapper mapper = new ObjectMapper();
+		InputStream gzipStream = new GZIPInputStream(getClass().getResourceAsStream("scholix_records.gz"));
+		Reader decoder = new InputStreamReader(gzipStream, "UTF-8");
+		BufferedReader buffered = new BufferedReader(decoder);
+		String line;
+		FileWriter myWriter = new FileWriter("/Users/sandro/Downloads/records");
+		while ((line = buffered.readLine()) != null) {
+			final Scholix s = mapper.readValue(line, Scholix.class);
+			final ScholixFlat flat = ScholixUtils.flattenizeScholix(s, line);
+			assertNotNull(s);
+			assertNotNull(flat);
+			assertEquals(s.getIdentifier(), flat.getIdentifier());
+			assertEquals(s.getRelationship().getName(), flat.getRelationType());
+			assertEquals(s.getSource().getObjectType(), flat.getSourceType());
+			assertEquals(s.getSource().getObjectSubType(), flat.getSourceSubType());
+			myWriter.write(mapper.writeValueAsString(flat));
+			myWriter.write("\n");
+
+		}
+
+		myWriter.close();
+
+	}
+}

dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/sx/provision/ScholixFlat.java

@@ -1,14 +1,15 @@
+
 package eu.dnetlib.dhp.sx.provision;
 
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
-import eu.dnetlib.dhp.schema.sx.scholix.ScholixResource;
-
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
+import eu.dnetlib.dhp.schema.sx.scholix.ScholixResource;
+
 public class ScholixFlat {
 	private static ObjectMapper MAPPER = new ObjectMapper();
 	private List<String> linkProvider = new ArrayList<>();
@@ -27,8 +28,6 @@ public class ScholixFlat {
 
 	private String json;
 
-
-
 	public void addLinkProvider(final String providerName) {
 		addStringToList(providerName, this.linkProvider);
 	}
@@ -37,6 +36,7 @@ public class ScholixFlat {
 		addStringToList(linkPublisher, sourceLinkPublisher);
 
 	}
+
 	public void addTargetLinkPublisher(final String linkPublisher) {
 		addStringToList(linkPublisher, targetLinkPublisher);
 
@@ -54,7 +54,6 @@ public class ScholixFlat {
 		addStringToList(pidType, targetPidTypes);
 	}
 
-
 	public void addTargetPid(final String pid) {
 		addStringToList(pid, targetPids);
 	}
@@ -128,7 +127,6 @@ public class ScholixFlat {
 		this.targetLinkPublisher = targetLinkPublisher;
 	}
 
-
 	public List<String> getLinkProvider() {
 		return linkProvider;
 	}
@@ -174,8 +172,6 @@ public class ScholixFlat {
 			source.getPublisher().forEach(p -> flat.addSourceLinkPublisher(p.getName()));
 		}
 
-
-
 		final ScholixResource target = scholix.getSource();
 		flat.setTargetDnetIdentifier(target.getDnetIdentifier());
 		if (target.getIdentifier() != null) {

dhp-workflows/dhp-graph-provision/src/test/scala/eu/dnetlib/dhp/sx/provision/ScholixFlatTest.scala

@@ -8,10 +8,8 @@ import org.junit.jupiter.api.{Test}
 
 class ScholixFlatTest {
 
-
   var spark: SparkSession = null
 
-
   def initSpark(): Unit = {
 
     if (spark != null)
@@ -31,13 +29,10 @@ class ScholixFlatTest{
       .getOrCreate()
   }
 
-
-
   def after(): Unit = {
     spark.stop()
   }
 
-
   @Test
   def testScholixConversion(): Unit = {
     initSpark()
@@ -46,7 +41,6 @@ class ScholixFlatTest{
     val t = spark.read.text(p).count
     println(s"total =$t")
 
-
   }
 
 }

pom.xml

@@ -807,7 +807,7 @@
 		<mockito-core.version>3.3.3</mockito-core.version>
 		<mongodb.driver.version>3.4.2</mongodb.driver.version>
 		<vtd.version>[2.12,3.0)</vtd.version>
-		<dhp-schemas.version>[3.16.0]</dhp-schemas.version>
+		<dhp-schemas.version>[3.16.1-SNAPSHOT]</dhp-schemas.version>
 		<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
 		<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
 		<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>