From 0ea7e861aba3617f11881df67514be94b04b8607 Mon Sep 17 00:00:00 2001 From: "sandro.labruzzo" Date: Fri, 25 Oct 2019 11:56:28 +0200 Subject: [PATCH] added organizations test --- .../eu/dnetlib/dhp/graph/ProtoConverter.java | 7 ++ .../dhp/graph/SparkGraphImporterJob.java | 10 +-- .../dnetlib/dhp/graph/ProtoConverterTest.java | 23 ++++- .../eu/dnetlib/dhp/graph/datasource.json | 73 ++++++++++++++++ .../eu/dnetlib/dhp/graph/organization.json | 84 +++++++++---------- 5 files changed, 147 insertions(+), 50 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/datasource.json diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java index 8ecb5c999..fa30c1fde 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/ProtoConverter.java @@ -192,6 +192,13 @@ public class ProtoConverter implements Serializable { } private static Software createSoftware(OafProtos.Oaf oaf) { + final ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata(); + final Software software = setOaf(new Software(), oaf); + + + + + return new Software(); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java index 76bcc0cf9..2fe3b07aa 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java @@ -1,22 +1,16 @@ package eu.dnetlib.dhp.graph; -import eu.dnetlib.dhp.schema.oaf.Datasource; -import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Organization; -import eu.dnetlib.dhp.schema.oaf.Publication; import org.apache.hadoop.io.Text; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; import scala.Tuple2; -import javax.xml.crypto.Data; - public class SparkGraphImporterJob { @@ -42,6 +36,10 @@ public class SparkGraphImporterJob { .map(item -> new Tuple2<>(item._1.toString(), item._2.toString())); + final String body = inputRDD.filter(s -> s._1().contains("20|") && s._1().split("@")[2].equalsIgnoreCase("body")).map(Tuple2::_2).first(); + + System.out.println(body); + final JavaRDD datasources = inputRDD .filter(s -> s._1().split("@")[2].equalsIgnoreCase("body")) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java index fd0edf573..aa3cce66d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/graph/ProtoConverterTest.java @@ -3,8 +3,11 @@ package eu.dnetlib.dhp.graph; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.Organization; import org.apache.commons.io.IOUtils; + import static org.junit.Assert.*; + import org.junit.Test; public class ProtoConverterTest { @@ -12,10 +15,9 @@ public class ProtoConverterTest { @Test public void convertDatasourceTest() throws Exception { - final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/organization.json")); + final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/datasource.json")); Oaf result = ProtoConverter.convert(json); - assertNotNull(result); assertTrue(result instanceof Datasource); Datasource ds = (Datasource) result; @@ -26,8 +28,25 @@ public class ProtoConverterTest { ObjectMapper mapper = new ObjectMapper(); System.out.println(mapper.writeValueAsString(result)); + } + @Test + public void convertOrganizationTest() throws Exception { + + final String json = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/graph/organization.json")); + + Oaf result = ProtoConverter.convert(json); + assertNotNull(result); + assertTrue(result instanceof Organization); + Organization ds = (Organization) result; + assertNotNull(ds.getId()); + + System.out.println(ds.getId()); + + + ObjectMapper mapper = new ObjectMapper(); + System.out.println(mapper.writeValueAsString(result)); } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/datasource.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/datasource.json new file mode 100644 index 000000000..e2c382c0a --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/datasource.json @@ -0,0 +1,73 @@ +{ + "kind": "entity", + "entity": { + "type": "datasource", + "datasource": { + "metadata": { + "officialname": { + "value": "CRIS UNS (Current Research Information System University of Novi Sad)" + }, + "englishname": { + "value": "CRIS UNS (Current Research Information System University of Novi Sad)" + }, + "websiteurl": { + "value": "https://cris.uns.ac.rs/" + }, + "accessinfopackage": [ + { + "value": "https://cris.uns.ac.rs/OAIHandlerOpenAIRECRIS" + } + ], + "namespaceprefix": { + "value": "CrisUnsNoviS" + }, + "datasourcetype": { + "classid": "crissystem", + "classname": "CRIS System", + "schemeid": "dnet:datasource_typologies", + "schemename": "dnet:datasource_typologies" + }, + "openairecompatibility": { + "classid": "openaire-cris_1.1", + "classname": "OpenAIRE CRIS v1.1", + "schemeid": "dnet:datasourceCompatibilityLevel", + "schemename": "dnet:datasourceCompatibilityLevel" + }, + "latitude": { + "value": "0.0" + }, + "longitude": { + "value": "0.0" + }, + "journal": { + "issnPrinted": "", + "issnOnline": "", + "issnLinking": "" + } + } + }, + "originalId": [ + "CRIS_UNS____::openaire" + ], + "collectedfrom": [ + { + "key": "", + "value": "" + } + ], + "dateofcollection": "2019-04-04", + "id": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556", + "dateoftransformation": "" + }, + "dataInfo": { + "inferred": false, + "deletedbyinference": false, + "trust": "0.9", + "provenanceaction": { + "classid": "sysimport:crosswalk:entityregistry", + "classname": "sysimport:crosswalk:entityregistry", + "schemeid": "dnet:provenance_actions", + "schemename": "dnet:provenance_actions" + } + } +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/organization.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/organization.json index e2c382c0a..1f8ab1950 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/organization.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/graph/organization.json @@ -1,68 +1,68 @@ { "kind": "entity", "entity": { - "type": "datasource", - "datasource": { + "type": "organization", + "organization": { "metadata": { - "officialname": { - "value": "CRIS UNS (Current Research Information System University of Novi Sad)" + "legalname": { + "value": "University of Utrecht" }, - "englishname": { - "value": "CRIS UNS (Current Research Information System University of Novi Sad)" + "eclegalbody": { + "value": "false" }, - "websiteurl": { - "value": "https://cris.uns.ac.rs/" + "eclegalperson": { + "value": "false" }, - "accessinfopackage": [ - { - "value": "https://cris.uns.ac.rs/OAIHandlerOpenAIRECRIS" - } - ], - "namespaceprefix": { - "value": "CrisUnsNoviS" + "ecnonprofit": { + "value": "false" }, - "datasourcetype": { - "classid": "crissystem", - "classname": "CRIS System", - "schemeid": "dnet:datasource_typologies", - "schemename": "dnet:datasource_typologies" + "ecresearchorganization": { + "value": "false" }, - "openairecompatibility": { - "classid": "openaire-cris_1.1", - "classname": "OpenAIRE CRIS v1.1", - "schemeid": "dnet:datasourceCompatibilityLevel", - "schemename": "dnet:datasourceCompatibilityLevel" + "echighereducation": { + "value": "false" }, - "latitude": { - "value": "0.0" + "ecinternationalorganizationeurinterests": { + "value": "false" }, - "longitude": { - "value": "0.0" + "ecinternationalorganization": { + "value": "false" }, - "journal": { - "issnPrinted": "", - "issnOnline": "", - "issnLinking": "" + "ecenterprise": { + "value": "false" + }, + "ecsmevalidated": { + "value": "false" + }, + "ecnutscode": { + "value": "false" + }, + "country": { + "classid": "FI", + "classname": "Finland", + "schemeid": "dnet:countries", + "schemename": "dnet:countries" } } }, "originalId": [ - "CRIS_UNS____::openaire" + "aka_________::f88cc5f874ff27f0fd6e7cb24842e9fb" ], "collectedfrom": [ { - "key": "", - "value": "" + "key": "10|openaire____::6ac933301a3933c8a22ceebea7000326", + "value": "Academy of Finland" } ], - "dateofcollection": "2019-04-04", - "id": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556", - "dateoftransformation": "" + "dateofcollection": "2018-09-28", + "id": "20|aka_________::0070a5080d7092f960fb33c8a9fca016", + "dateoftransformation": "2019-04-16" }, "dataInfo": { - "inferred": false, - "deletedbyinference": false, + "inferred": true, + "deletedbyinference": true, "trust": "0.9", + "inferenceprovenance": "dedup-similarity-organization-simple", "provenanceaction": { "classid": "sysimport:crosswalk:entityregistry", "classname": "sysimport:crosswalk:entityregistry", @@ -70,4 +70,4 @@ "schemename": "dnet:provenance_actions" } } -} \ No newline at end of file +}