From c0d275a1b724aec388773580a9115f8f38f1fc10 Mon Sep 17 00:00:00 2001 From: Alessia Date: Wed, 18 Dec 2024 17:34:25 +0100 Subject: [PATCH 1/5] Test: added missing rdf type --- .../eu/dnetlib/ariadneplus/reader/json/roceeh.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/roceeh.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/roceeh.json index 99bb3ad..55f6163 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/roceeh.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/roceeh.json @@ -165,6 +165,13 @@ "type" : "literal", "datatype" : "http://www.w3.org/2001/XMLSchema#dateTime" } + ], + "https://www.ariadne-infrastructure.eu/property/rdfType" : [ + { + "value" : "Record", + "type" : "literal", + "datatype" : "http://www.w3.org/2001/XMLSchema#string" + } ] } } \ No newline at end of file -- 2.17.1 From 7dd16b74ca30b27dc416368c4ba229749ff10f09 Mon Sep 17 00:00:00 2001 From: Alessia Date: Wed, 18 Dec 2024 19:03:19 +0100 Subject: [PATCH 2/5] Extension for countries (https://support.d4science.org/issues/28575): code, properties, and tests --- dnet-ariadneplus-graphdb-publisher/pom.xml | 2 +- .../elasticsearch/model/AriadnePlusEntry.java | 9 ++++++ .../elasticsearch/model/CountryInfo.java | 31 +++++++++++++++++++ .../src/main/resources/application.properties | 20 ++++++++++++ .../read_collection_data_template.sparql | 7 +++++ .../sparql/read_record_data_template.sparql | 7 +++++ .../GraphDbReaderAndESIndexTest.java | 10 ++++++ .../reader/json/ParseRDFJsonTest.java | 17 ++++++++++ .../ariadneplus/reader/json/collection.json | 15 +++++++++ .../dnetlib/ariadneplus/reader/json/spec.json | 26 ++++++++++++++++ .../reader/json/test-thanados.json | 16 +++++++++- 11 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/CountryInfo.java diff --git a/dnet-ariadneplus-graphdb-publisher/pom.xml b/dnet-ariadneplus-graphdb-publisher/pom.xml index 0d0d39d..495edb2 100644 --- a/dnet-ariadneplus-graphdb-publisher/pom.xml +++ b/dnet-ariadneplus-graphdb-publisher/pom.xml @@ -10,7 +10,7 @@ eu.dnetlib dnet-ariadneplus-graphdb-publisher jar - 4.2.1-SNAPSHOT + 4.2.1-COUNTRY-SNAPSHOT scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-ariadneplus-graphdb-publisher/trunk https://github.com/spring-projects/spring-boot/spring-boot-starter-parent/dnet-ariadneplus-graphdb-publisher diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java index 048e108..f15727b 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java @@ -36,6 +36,7 @@ public class AriadnePlusEntry { private String wasCreated; private List digitalImage; private List otherId; + private List country; private transient String uniqueIsPartOf; private transient String typeURI; @@ -318,4 +319,12 @@ public class AriadnePlusEntry { } this.otherId.add(otherId); } + + public List getCountry() { + return country; + } + + public void setCountry(List country) { + this.country = country; + } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/CountryInfo.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/CountryInfo.java new file mode 100644 index 0000000..8677106 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/CountryInfo.java @@ -0,0 +1,31 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class CountryInfo { + private String id; + private String name; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public CountryInfo() { + } + + public static CountryInfo fromJson(String json){ + return new Gson().fromJson(json, CountryInfo.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 78d79b5..fb2b423 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -308,6 +308,21 @@ class.map.specifications={\ }\ }\ },\ +"CountryInfo": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ +"class_field": "Id",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/name": {\ +"class_field": "Name",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ "AriadnePlusEntry": {\ "class_type": "unique",\ "mappings": {\ @@ -476,6 +491,11 @@ class.map.specifications={\ "class_field": "DigitalImage",\ "substring": "no",\ "external_reference": "DigitalImage"\ +},\ +"https://www.ariadne-infrastructure.eu/property/country": {\ +"class_field": "Country",\ +"substring": "no",\ +"external_reference": "CountryInfo"\ }\ }\ }\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index 797b702..bd89286 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -95,6 +95,9 @@ CONSTRUCT { ?descriptionInfo aoprop:text ?description . ?descriptionInfo aoprop:lang ?descriptionLang . %record aoprop:language ?language . +%record aoprop:country ?country . +?country aoprop:id ?country . +?country aoprop:name ?countryName . } where { graph { @@ -121,6 +124,10 @@ where { optional { %record aocat:has_landing_page / rdfs:label ?landingPage . } + optional { + %record aocat:has_country ?country . + ?country rdfs:label ?countryName . + } BIND (IRI(concat( "https://ariadne-infrastructure.eu/TextLang/",SHA1(STR(?description)))) as ?descriptionInfo) BIND (LANG(?description) as ?descriptionLang) BIND (IRI(concat( "https://ariadne-infrastructure.eu/TextLang/",SHA1(STR(?title)))) as ?titleInfo) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 1da8731..048b3fb 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -100,6 +100,9 @@ CONSTRUCT { %record aoprop:image ?primaryVisualComponent . ?primaryVisualComponent aoprop:primaryVisualComponent ?primaryVisualComponent . ?primaryVisualComponent aoprop:visualComponent ?visualComponent . +%record aoprop:country ?country . +?country aoprop:id ?country . +?country aoprop:name ?countryName . } where { graph { @@ -134,6 +137,10 @@ where { %record aocat:has_spatial_precision / aocat:has_unit / skos:prefLabel ?spatialPrecisionUnit . %record aocat:has_spatial_precision / aocat:has_value ?spatialPrecisionValue . } + optional { + %record aocat:has_country ?country . + ?country rdfs:label ?countryName . + } BIND (IRI(concat( "https://ariadne-infrastructure.eu/TextLang/",SHA1(STR(?description)))) as ?descriptionInfo) BIND (LANG(?description) as ?descriptionLang) BIND (IRI(concat( "https://ariadne-infrastructure.eu/TextLang/",SHA1(STR(?title)))) as ?titleInfo) diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 8d51a50..bc88a16 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -625,6 +625,16 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test + public void uploadWithCountryTest() throws Exception { + boolean isRecord = false; + String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/INFN/8AF9C287-9CBD-32DC-9B18-0C0EFD85BAFB"; + String datasource = "infn"; + String apiId = "firenze"; + readAndIndexTest(isRecord, recordId, datasource, apiId); + + } + private class CollectionData { String URI_ID; String id; diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java index 5c49c96..a61ada8 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java @@ -13,6 +13,7 @@ import org.springframework.core.io.ClassPathResource; import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.util.LinkedHashMap; public class ParseRDFJsonTest { @@ -128,6 +129,22 @@ public class ParseRDFJsonTest { } } + @Test + public void testCountryInCollection() throws ParseException { + String collectionJson = getFromClasspath("eu/dnetlib/ariadneplus/reader/json/collection.json"); + parser.setCollection(true); + int res = parser.parse(collectionJson); + Assert.assertNotEquals(-1, res); + boolean hasCountry = false; + while(parser.hasNextElement()){ + LinkedHashMap element = parser.getNextElement(); + System.out.println(element); + hasCountry = element.containsKey("https://www.ariadne-infrastructure.eu/property/country"); + } + Assert.assertTrue(hasCountry); + + } + private String getFromClasspath(String s) { try { final ClassPathResource resource = new ClassPathResource(s); diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json index 24dd6e6..c250136 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json @@ -111,6 +111,12 @@ "type" : "uri" } ], + "https://www.ariadne-infrastructure.eu/property/country" : [ + { + "value" : "http://www.wikidata.org/entity/Q38", + "type" : "uri" + } + ], "https://www.ariadne-infrastructure.eu/property/resourceType" : [ { "value" : "Collection", @@ -172,5 +178,14 @@ "lang" : "en" } ] + }, + + "http://www.wikidata.org/entity/Q38" : { + "https://www.ariadne-infrastructure.eu/property/name" : [ + { + "value" : "Italy", + "type" : "literal" + } + ] } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json index e0c3e76..4a8fd04 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json @@ -290,6 +290,21 @@ } } }, + "CountryInfo": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/country": { + "class_field": "Id", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/name": { + "class_field": "Name", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, "AriadnePlusEntry": { "class_type": "unique", "mappings": { @@ -423,6 +438,12 @@ "substring": "no", "element_type": "java.lang.String" }, + "https://www.ariadne-infrastructure.eu/property/otherId": { + "class_field": "OtherId", + "class_type": "prototype", + "substring": "no", + "element_type": "java.lang.String" + }, "https://www.ariadne-infrastructure.eu/property/temporal": { "class_field": "Temporal", "substring": "no", @@ -452,6 +473,11 @@ "class_field": "DigitalImage", "substring": "no", "external_reference": "DigitalImage" + }, + "https://www.ariadne-infrastructure.eu/property/country": { + "class_field": "Country", + "substring": "no", + "external_reference": "CountryInfo" } } } diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/test-thanados.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/test-thanados.json index a2cb4ec..0caf17b 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/test-thanados.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/test-thanados.json @@ -204,6 +204,12 @@ "type" : "uri" } ], + "https://www.ariadne-infrastructure.eu/property/country" : [ + { + "value" : "http://www.wikidata.org/entity/Q38", + "type" : "uri" + } + ], "https://www.ariadne-infrastructure.eu/property/rdfType" : [ { "value" : "Collection", @@ -317,6 +323,14 @@ "value" : "https://ariadne-infrastructure.eu/aocat/Time-Span/25162", "type" : "uri" } - ] + ], + "http://www.wikidata.org/entity/Q38" : { + "https://www.ariadne-infrastructure.eu/property/name": [ + { + "value": "Italy", + "type": "literal" + } + ] + } } } \ No newline at end of file -- 2.17.1 From db2aec61b47e4202d6aed908305e47c37c93e56e Mon Sep 17 00:00:00 2001 From: Alessia Date: Wed, 18 Dec 2024 19:06:37 +0100 Subject: [PATCH 3/5] Extension for countries (https://support.d4science.org/issues/28575): code, properties, and tests --- .../ariadneplus/GraphDbReaderAndESIndexTest.java | 12 +----------- .../ariadneplus/reader/json/ParseRDFJsonTest.java | 4 +--- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index bc88a16..7559642 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -8,16 +8,11 @@ import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherException; import eu.dnetlib.ariadneplus.reader.ResourceManager; import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; -import org.apache.commons.compress.utils.Lists; import org.apache.commons.io.IOUtils; import org.eclipse.rdf4j.RDF4JException; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.ValueFactory; -import org.eclipse.rdf4j.query.GraphQuery; -import org.eclipse.rdf4j.query.GraphQueryResult; -import org.eclipse.rdf4j.query.QueryLanguage; -import org.eclipse.rdf4j.query.QueryResults; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.repository.RepositoryResult; @@ -26,18 +21,13 @@ import org.eclipse.rdf4j.repository.util.Repositories; import org.junit.Ignore; import org.junit.Test; import org.springframework.core.io.ClassPathResource; - import java.io.BufferedReader; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Properties; -import java.util.stream.Collectors; -import java.util.stream.Stream; + /** * @author enrico.ottonello diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java index a61ada8..57ca45f 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java @@ -2,12 +2,10 @@ package eu.dnetlib.ariadneplus.reader.json; import eu.dnetlib.ariadneplus.elasticsearch.model.AriadnePlusEntry; import eu.dnetlib.ariadneplus.reader.ResourceManager; -import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; import net.minidev.json.parser.ParseException; import org.apache.commons.io.IOUtils; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.springframework.core.io.ClassPathResource; @@ -104,7 +102,7 @@ public class ParseRDFJsonTest { Assert.assertNotEquals(-1, res); mng.manage(parser); if(mng.hasNext()) - System.out.println(mng.next()); + System.out.println(mng.next()); } @Test -- 2.17.1 From 49b992d1f4e53e6bb1bbe5971f3cb0bc6a05a39b Mon Sep 17 00:00:00 2001 From: Alessia Date: Wed, 18 Dec 2024 19:16:37 +0100 Subject: [PATCH 4/5] Json configuration for class mapping aligned with the tested connfiguration spec.json --- .../src/main/resources/application.properties | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index fb2b423..bd67f05 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -311,7 +311,7 @@ class.map.specifications={\ "CountryInfo": {\ "class_type": "prototype",\ "mappings": {\ -"https://www.ariadne-infrastructure.eu/property/id": {\ +"https://www.ariadne-infrastructure.eu/property/country": {\ "class_field": "Id",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -500,4 +500,3 @@ class.map.specifications={\ }\ }\ } - -- 2.17.1 From 9bc72c34d90eae9e27b7cebb5e8407c6ee75dfcb Mon Sep 17 00:00:00 2001 From: Alessia Date: Wed, 18 Dec 2024 19:16:44 +0100 Subject: [PATCH 5/5] Formatting --- .../dnetlib/ariadneplus/reader/json/spec.json | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json index 4a8fd04..166d8a2 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json @@ -290,21 +290,21 @@ } } }, - "CountryInfo": { - "class_type": "prototype", - "mappings": { - "https://www.ariadne-infrastructure.eu/property/country": { - "class_field": "Id", - "substring": "no", - "element_type": "java.lang.String" - }, - "https://www.ariadne-infrastructure.eu/property/name": { - "class_field": "Name", - "substring": "no", - "element_type": "java.lang.String" - } + "CountryInfo": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/country": { + "class_field": "Id", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/name": { + "class_field": "Name", + "substring": "no", + "element_type": "java.lang.String" } - }, + } + }, "AriadnePlusEntry": { "class_type": "unique", "mappings": { -- 2.17.1