D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Collector Plugin for Thanados and support for collections in collections #3

Merged
enrico.ottonello merged 14 commits from thanados into master 2022-05-06 11:06:39 +02:00
Conversation 0 Commits 14 Files Changed 13 +1015 -351
13 changed files with 1015 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -70,6 +70,15 @@ public class BulkUpload {
uri.setLabel(ace.getTypeLabel());
ace.setHas_type(uri);
String uniqueIsPartOf = ace.getUniqueIsPartOf();
if (uniqueIsPartOf != null) {
ace.setIsPartOf(Arrays.asList(uniqueIsPartOf));
}
if (ace.getContributor() != null) {
ace.getContributor().clear();
ace.setContributor(ace.getCreator());
}
if (ace.getSpatial()!=null) {
ace.getSpatial()
.stream()
@ -162,17 +171,6 @@ public class BulkUpload {
}
}
if (!isCollection) {
String uniqueIsPartOf = ace.getUniqueIsPartOf();
if (uniqueIsPartOf != null) {
ace.setIsPartOf(Arrays.asList(uniqueIsPartOf));
}
if (ace.getContributor() != null) {
ace.getContributor().clear();
ace.setContributor(ace.getCreator());
}
}
String[] splits = ace.getIdentifier().split("/");
log.debug("JSON >>>> "+ace.toJson());

322
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/enrich/THANADOS/ordered_sparql_insert_THANADOS_test.sparql Normal file
View File

@ -0,0 +1,322 @@
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_temporal_coverage ?tcov .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?s aocat:is_part_of ?coll .
?coll aocat:has_temporal_coverage ?tcov .
MINUS {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?s aocat:has_temporal_coverage ?tc .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_type <https://ariadne-infrastructure.eu/aocat/Concept/AO_Type/collection> .
<https://ariadne-infrastructure.eu/aocat/Concept/AO_Type/collection> rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Concept> .
<https://ariadne-infrastructure.eu/aocat/Concept/AO_Type/collection> skos:prefLabel "collection"
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
MINUS {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?s aocat:has_type ?t .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
<https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Agent> .
<https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> rdfs:label "THANADOS" .
<https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> aocat:has_name "THANADOS" .
?s aocat:has_creator <https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> .
?s aocat:has_contributor <https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> .
?s aocat:has_responsible <https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> .
?s aocat:has_owner <https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> .
?s aocat:has_publisher <https://ariadne-infrastructure.eu/aocat/Agent/THANADOS> .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:was_issued "2022"
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?s aocat:was_issued ?smt .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_access_rights "Not provided"
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?s aocat:has_access_rights ?smt .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_spatial_coverage ?scov .
?scov aocat:has_place_name ?placeName .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_spatial_coverage ?scov .
?scov aocat:has_place_name ?placeName .
MINUS {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_spatial_coverage ?sc .
?sc aocat:has_place_name ?name .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_spatial_coverage <https://ariadne-infrastructure.eu/aocat/Place/NOTPROVIDED> .
<https://ariadne-infrastructure.eu/aocat/Place/NOTPROVIDED> rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Spatial_Region> .
<https://ariadne-infrastructure.eu/aocat/Place/NOTPROVIDED> rdfs:label "Name not provided" .
<https://ariadne-infrastructure.eu/aocat/Place/NOTPROVIDED> aocat:has_place_name "Name not provided" .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type aocat:AO_Individual_Data_Resource .
MINUS {
?s rdf:type aocat:AO_Individual_Data_Resource .
?s aocat:has_spatial_coverage ?spatialRegion .
?spatialRegion aocat:has_place_name ?placeName .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_type <https://ariadne-infrastructure.eu/aocat/Concept/AO_Type/provided%20record> .
<https://ariadne-infrastructure.eu/aocat/Concept/AO_Type/provided%20record> rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Concept> .
<https://ariadne-infrastructure.eu/aocat/Concept/AO_Type/provided%20record> skos:prefLabel "provided record"
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type aocat:AO_Individual_Data_Resource .
MINUS {
?s rdf:type aocat:AO_Individual_Data_Resource .
?s aocat:has_type ?t .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_creator ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_creator ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_creator ?x
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_contributor ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_contributor ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_contributor ?x
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_responsible ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_responsible ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_responsible ?x
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_owner ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_owner ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_owner ?x
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_publisher ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_publisher ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_publisher ?x
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:was_issued ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:was_issued ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:was_issued ?x
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s aocat:has_access_rights ?cr .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::thanados::test> {
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:is_part_of ?coll .
?coll aocat:has_access_rights ?cr .
MINUS{
?s rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Individual_Data_Resource> .
?s aocat:has_access_rights ?x
}
}
};

58
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -26,13 +26,12 @@ import java.util.Properties;
* @author enrico.ottonello
*
*/
//@Ignore
@Ignore
public class GraphDbReaderAndESIndexTest {
private RunSPARQLQueryService runSPQRLQuery;
@Test
// @Ignore
public void uploadAMCRFieldworkTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA";
@ -42,7 +41,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadAMCRFieldworkEventOnlyPeriodoTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/849F814F-5892-3408-AD5E-904938B4492A";
@ -52,7 +50,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadAMCRDocumentTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/FC59581D-DC3A-31DA-922A-98DE764F3D76";
@ -62,7 +59,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadAMCRSiteTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/3C7EC936-A7CA-3720-B3DC-413A25754FD4";
@ -72,7 +68,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadAMCRIndividualFindTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/98D717C4-410F-35C6-8072-FABA7686B4A3";
@ -82,7 +77,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadADSRecordWithNativeFromUntilTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF";
@ -92,7 +86,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA";
@ -102,7 +95,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadZbivaRecordSpatialTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479";
@ -112,7 +104,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadHNMCollectionSpatialTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5";
@ -122,7 +113,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadHNMCollectionTemporalTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10";
@ -132,7 +122,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadADSArchivesBoundingBoxTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/90D1C95D-E249-3E74-92D9-B58FDF690CC7";
@ -142,7 +131,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDansSpatialTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD";
@ -152,7 +140,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDansNativePeriodTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B";
@ -162,7 +149,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDansTemporalPolygonTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B";
@ -172,7 +158,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadSndRockartTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A";
@ -182,7 +167,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadNIAMTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1";
@ -192,7 +176,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadADS398Test() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73";
@ -202,7 +185,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadROADAfterFullReloadTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E859C58E-0307-3D7C-BCAB-45483DEA917C";
@ -212,7 +194,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadADS328NativeFromUntilTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559";
@ -222,7 +203,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadFastiMultipleAriadneSubjectTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/FE740BC7-2D9E-3644-A88E-3E8E5A5EA95F";
@ -232,7 +212,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadSNDArchivePolygonTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/A7A2588A-1FE7-3FDD-AD66-C84EACE1860E";
@ -242,7 +221,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadFastiTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/532D492C-B141-30D0-886A-1E69C2C20474";
@ -252,7 +230,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDansClosedAccessTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D1C1D6AD-320D-3616-B75D-F2E2C3DA7F3E";
@ -262,7 +239,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDansRestrictedAccessTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/221AAEC8-CC9B-310C-B800-0DFE34850E0B";
@ -272,7 +248,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDansOpenAccessTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/B251A0FE-EFE0-341B-A8B3-E0813CC82856";
@ -282,7 +257,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadConicetCollectionTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IDACOR%20%28CONICET%2FUNC%29/8B238475-2BA7-3036-88C0-968E9A642E64";
@ -292,7 +266,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadConicetRecordTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CONICET/0CD6BF2E-4AB7-39B6-937E-7DD59092DD3F";
@ -302,7 +275,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadNaraTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/001CA559-D688-31FB-AB48-45CC8DB1E355";
@ -312,7 +284,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadINFNTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/INFN/5AE70226-82A9-3B06-9E25-8FB71B64C51C";
@ -322,7 +293,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
public void uploadADSGreyLiteratureOver300TriplesTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/35AEC455-6CB5-31BB-B73E-30959FF3C4FE";
@ -332,7 +302,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDimeBoundingBoxTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DIME/0E5FB5C7-6FCA-36AD-BEE5-03C9AFF393CD";
@ -342,7 +311,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadNormalizerTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DCDF395A-2CC8-3FC2-B9A5-5A924090DF10";
@ -352,7 +320,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadNormalizerTest2() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/A0D9DC1F-2C62-3423-BAE6-4248D5072585";
@ -362,7 +329,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadNormalizerAllUppercaseTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E644B5AD-3DC2-3FEC-A070-B7FD434B0E90";
@ -372,7 +338,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadNormalizerMinusTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/9F96015D-84FA-3BDE-AE3C-90CF3CC7CEDB";
@ -382,7 +347,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadCeniehTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0F55399D-8001-36E6-8D52-0C32981CAD7E";
@ -392,7 +356,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadDoliaTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/F0E19AC1-AE02-3BE5-AF78-D9EEC1670AC1";
@ -404,7 +367,6 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
// @Ignore
public void uploadIAVPFastCatCollectionWasCreatedTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IAVP/E60F992F-C34D-39B5-99AD-406BA27B090B";
@ -413,6 +375,24 @@ public class GraphDbReaderAndESIndexTest {
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
@Test
public void collectionInCollectionTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/THANADOS/3E6614E4-EA97-3E7D-BA72-B0343A63FA39";
String datasource = "thanados";
String collectionId = "test";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
@Test
public void recordInCollectionTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/THANADOS/46FD94FF-0C91-369A-A64F-E29B7C9A40D9";
String datasource = "thanados";
String collectionId = "test";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
final ClassPathResource resource = new ClassPathResource("application.properties");
Properties appProps = new Properties();

5
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java
View File

@ -18,12 +18,14 @@ import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.RDFWriter;
import org.eclipse.rdf4j.rio.Rio;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.core.io.ClassPathResource;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
@Ignore
public class GraphDbReaderTest {
private static final Log log = LogFactory.getLog(GraphDbReaderTest.class);
@ -86,9 +88,10 @@ public class GraphDbReaderTest {
}
private boolean openConnection() throws Exception {
manager = new RemoteRepositoryManager(getGraphDBUrl());
manager.init();
manager.setUsernameAndPassword(getUsername(), getPwd());
// manager.setUsernameAndPassword(getUsername(), getPwd());
try {
repository = manager.getRepository(getGraphDBRepository());
} catch (UnauthorizedException e) {

2
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java
View File

@ -23,7 +23,7 @@ public class ParseRDFJsonTest {
public void setup(){
parser = new ParseRDFJSON();
parser.setCatalogEntryCollectionJsonPath("$[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Collection')]");
parser.setCatalogEntryJsonPath("$[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Record')]");
parser.setCatalogEntryJsonPath("[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Record')]");
mng = new ResourceManager();
mng.setup("https://www.ariadne-infrastructure.eu/property/resourceType", "eu.dnetlib.ariadneplus.elasticsearch.model.",
"[\"https://www.ariadne-infrastructure.eu/property/resourceType\", \"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\"]",

15
dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json
View File

@ -21,6 +21,19 @@
"lang" : "en"
}
],
"https://www.ariadne-infrastructure.eu/property/rdfType" : [
{
"value" : "Collection",
"type" : "literal",
"datatype" : "http://www.w3.org/2001/XMLSchema#string"
}
],
"https://www.ariadne-infrastructure.eu/property/partOf" : [
{
"value" : "https://ariadne-infrastructure.eu/aocat/Collection/CENIEH/A4A042CF-4FD6-3FB4-B701-CBC8A1653ADA",
"type" : "uri"
}
],
"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType" : [
{
"value" : "https://ariadne-infrastructure.eu/aocat/Concept/AO_Subject/Fieldwork",
@ -100,7 +113,7 @@
],
"https://www.ariadne-infrastructure.eu/property/resourceType" : [
{
"value" : "Dataset Collection",
"value" : "Collection",
"type" : "literal",
"datatype" : "http://www.w3.org/2001/XMLSchema#string"
}

754
dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json
View File

@ -1,300 +1,458 @@
{
"AriadneTemporal": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/from": {
"class_field": "From",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/periodName": {
"class_field": "PeriodName",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/until": {
"class_field": "Until",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/uri": {
"class_field": "Uri",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"AriadneGeoPoint": {
"class_type": "unique",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/lat": {
"class_field": "Lat",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lon": {
"class_field": "Lon",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"Spatial": {
"class_type": "prototype",
"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {
"class_field": "Address",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {
"class_field": "CoordinateSystem",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/country": {
"class_field": "Country",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/location": {
"class_field": "Location",
"external_reference": "AriadneGeoPoint",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/lat": {
"class_field": "Lat",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lon": {
"class_field": "Lon",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/placeName": {
"class_field": "PlaceName",
"substring": "no",
"element_type": "java.lang.String"
}}
},
"ArchaeologicalResourceType": {
"class_type": "unique",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/id": {
"class_field": "Id",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/name": {
"class_field": "Name",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"AgentInfo": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/name": {
"class_field": "Name",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/type": {
"class_field": "Type",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/email": {
"class_field": "Email",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/phone": {
"class_field": "Phone",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"NativeSubject": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/prefLabel": {
"class_field": "PrefLabel",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/rdfAbout": {
"class_field": "RdfAbout",
"substring": "yes",
"element_type": "java.lang.String"
}
}
},
"DerivedSubject": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/prefLabel": {
"class_field": "PrefLabel",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/source": {
"class_field": "Source",
"substring": "yes",
"element_type": "java.lang.String"
}
}
},
"AatSubject": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/id": {
"class_field": "Id",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/label": {
"class_field": "Label",
"substring": "yes",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lang": {
"class_field": "Lang",
"substring": "yes",
"element_type": "java.lang.String"
}
}
},
"AriadneCatalogEntry": {
"class_type": "unique",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/accessPolicy": {
"class_field": "AccessPolicy",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/identifier": {
"class_field": "Identifier",
"substring": "yes",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/publisher": {
"class_field": "Publisher",
"external_reference": "AgentInfo",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {
"class_field": "ArchaeologicalResourceType",
"external_reference": "ArchaeologicalResourceType",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/issued": {
"class_field": "Issued",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/resourceType": {
"class_field": "ResourceType",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/modified": {
"class_field": "Modified",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/nativeSubject": {
"class_field": "NativeSubject",
"substring": "no",
"external_reference": "NativeSubject"
},
"https://www.ariadne-infrastructure.eu/property/derivedSubject": {
"class_field": "DerivedSubject",
"substring": "no",
"external_reference": "DerivedSubject"
},
"https://www.ariadne-infrastructure.eu/property/aatSubjects": {
"class_field": "AatSubjects",
"substring": "no",
"external_reference": "AatSubject"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegion": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/landingPage": {
"class_field": "LandingPage",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/placeName": {
"class_field": "PlaceName",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/title": {
"class_field": "Title",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/accessRights": {
"class_field": "AccessRights",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/description": {
"class_field": "Description",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/creator": {
"class_field": "Creator",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/contributor": {
"class_field": "Contributor",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/legalResponsible": {
"class_field": "LegalResponsible",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/owner": {
"class_field": "Owner",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/originalId": {
"class_field": "OriginalId",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/uri": {
"class_field": "Temporal",
"substring": "no",
"external_reference": "AriadneTemporal"
},
"https://www.ariadne-infrastructure.eu/property/language": {
"class_field": "Language",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/partOf": {
"class_field": "UniqueIsPartOf",
"substring": "no",
"element_type": "java.lang.String"
}
}
}
"TextLang": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/text": {
"class_field": "Text",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lang": {
"class_field": "Language",
"substring": "yes",
"element_type": "java.lang.String"
}
}
},
"DigitalImage": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/primaryVisualComponent": {
"class_field": "ProviderUri",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/visualComponent": {
"class_field": "AriadneUri",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"AriadneResource": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/aboutURILabel": {
"class_field": "Label",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/aboutURI": {
"class_field": "Uri",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"AriadneGeoPoint": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/lat": {
"class_field": "Lat",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lon": {
"class_field": "Lon",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"Temporal": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/from": {
"class_field": "From",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/periodOName": {
"class_field": "PeriodName",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/until": {
"class_field": "Until",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/uri": {
"class_field": "Uri",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"NativePeriod": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/nativeFrom": {
"class_field": "From",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/nativePeriodName": {
"class_field": "PeriodName",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/nativeUntil": {
"class_field": "Until",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"Spatial": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/address": {
"class_field": "Address",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lat": {
"class_field": "Lat",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lon": {
"class_field": "Lon",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/placeName": {
"class_field": "PlaceName",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/boxMaxLat": {
"class_field": "BoundingBoxMaxLat",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/boxMinLat": {
"class_field": "BoundingBoxMinLat",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/boxMaxLon": {
"class_field": "BoundingBoxMaxLon",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/boxMinLon": {
"class_field": "BoundingBoxMinLon",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/precisionUnit": {
"class_field": "SpatialPrecision",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/precisionValue": {
"class_field": "CoordinatePrecision",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/geoPoint": {
"class_field": "PolygonGeoPoints",
"substring": "no",
"element_type": "java.lang.String",
"external_reference": "AriadneGeoPoint"
},
"https://www.ariadne-infrastructure.eu/property/wkt": {
"class_field": "Wkt",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"AriadneSubject": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/name": {
"class_field": "PrefLabel",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"AgentInfo": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/name": {
"class_field": "Name",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/type": {
"class_field": "Type",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/homepage": {
"class_field": "Homepage",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/institution": {
"class_field": "Institution",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/agentIdentifier": {
"class_field": "AgentIdentifier",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"PublisherInfo": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/publisherName": {
"class_field": "Name",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/type": {
"class_field": "Type",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/publisherEmail": {
"class_field": "Email",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/publisherHomepage": {
"class_field": "Homepage",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/publisherInstitution": {
"class_field": "Institution",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/publisherAgentIdentifier": {
"class_field": "AgentIdentifier",
"substring": "no",
"element_type": "java.lang.String"
}
}
},
"NativeSubject": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/prefLabel": {
"class_field": "PrefLabel",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/rdfAbout": {
"class_field": "RdfAbout",
"substring": "yes",
"element_type": "java.lang.String"
}
}
},
"DerivedSubject": {
"class_type": "prototype",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/prefLabel": {
"class_field": "PrefLabel",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/source": {
"class_field": "Source",
"substring": "yes",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/id": {
"class_field": "Id",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/label": {
"class_field": "PrefLabel",
"substring": "yes",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/lang": {
"class_field": "Lang",
"substring": "yes",
"element_type": "java.lang.String"
}
}
},
"AriadnePlusEntry": {
"class_type": "unique",
"mappings": {
"https://www.ariadne-infrastructure.eu/property/typeURI": {
"class_field": "TypeURI",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/typeLabel": {
"class_field": "TypeLabel",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/accessPolicy": {
"class_field": "AccessPolicy",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/identifier": {
"class_field": "Identifier",
"substring": "yes",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/publisher": {
"class_field": "Publisher",
"external_reference": "PublisherInfo",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {
"class_field": "AriadneSubject",
"external_reference": "AriadneSubject",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/issued": {
"class_field": "Issued",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/created": {
"class_field": "WasCreated",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/resourceType": {
"class_field": "ResourceType",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/modified": {
"class_field": "Modified",
"element_type": "java.lang.String",
"substring": "no"
},
"https://www.ariadne-infrastructure.eu/property/nativeSubject": {
"class_field": "NativeSubject",
"substring": "no",
"external_reference": "NativeSubject"
},
"https://www.ariadne-infrastructure.eu/property/derivedSubject": {
"class_field": "DerivedSubject",
"substring": "no",
"external_reference": "DerivedSubject"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegion": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegionBox": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegionPolygon": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/spatialRegionWKTPolygon": {
"class_field": "Spatial",
"substring": "no",
"external_reference": "Spatial"
},
"https://www.ariadne-infrastructure.eu/property/landingPage": {
"class_field": "LandingPage",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/titleInfo": {
"class_field": "Title",
"substring": "no",
"external_reference": "TextLang"
},
"https://www.ariadne-infrastructure.eu/property/accessRights": {
"class_field": "AccessRights",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/descriptionInfo": {
"class_field": "Description",
"substring": "no",
"external_reference": "TextLang"
},
"https://www.ariadne-infrastructure.eu/property/creator": {
"class_field": "Creator",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/contributor": {
"class_field": "Contributor",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/legalResponsible": {
"class_field": "Responsible",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/owner": {
"class_field": "Owner",
"substring": "no",
"external_reference": "AgentInfo"
},
"https://www.ariadne-infrastructure.eu/property/originalId": {
"class_field": "OriginalId",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/temporal": {
"class_field": "Temporal",
"substring": "no",
"external_reference": "Temporal"
},
"https://www.ariadne-infrastructure.eu/property/temporalNative": {
"class_field": "NativePeriod",
"substring": "no",
"external_reference": "NativePeriod"
},
"https://www.ariadne-infrastructure.eu/property/language": {
"class_field": "Language",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/partOf": {
"class_field": "UniqueIsPartOf",
"substring": "no",
"element_type": "java.lang.String"
},
"https://www.ariadne-infrastructure.eu/property/about": {
"class_field": "Is_about",
"substring": "no",
"external_reference": "AriadneResource"
},
"https://www.ariadne-infrastructure.eu/property/image": {
"class_field": "DigitalImage",
"substring": "no",
"external_reference": "DigitalImage"
}
}
}
}

5
dnet-ariadneplus/pom.xml
View File

@ -36,6 +36,11 @@
<artifactId>dnet-msro-service</artifactId>
<version>[7.0.0-SAXONHE-SOLR772-SNAPSHOT, 8.0.0-SAXONHE)</version>
</dependency>
<dependency>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
<version>2.4.8</version>
</dependency>
</dependencies>
<properties>

21
dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosCollectorPlugin.java Normal file
View File

@ -0,0 +1,21 @@
package eu.dnetlib.data.collector.plugins.ariadneplus.thanados;
import eu.dnetlib.data.collector.plugins.httplist.HttpListIterator;
import eu.dnetlib.rmi.data.CollectorServiceException;
import eu.dnetlib.rmi.data.InterfaceDescriptor;
import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import java.util.Date;
public class ThanadosCollectorPlugin extends AbstractCollectorPlugin {
@Override
public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate) {
final String baseUrl = interfaceDescriptor.getBaseUrl();
final String listAddress = interfaceDescriptor.getParams().get("listUrl");
return () -> new ThanadosIterator(baseUrl, listAddress, fromDate);
}
}

111
dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIterator.java Normal file
View File

@ -0,0 +1,111 @@
package eu.dnetlib.data.collector.plugins.ariadneplus.thanados;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import eu.dnetlib.data.collector.ThreadSafeIterator;
import eu.dnetlib.rmi.data.CollectorServiceRuntimeException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.List;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
public class ThanadosIterator extends ThreadSafeIterator {
private static final Log log = LogFactory.getLog(ThanadosIterator.class);
private final CloseableHttpClient client = HttpClients.createDefault();
private String baseUrl;
private List<String> identifiers;
private int counter = 0;
private String urlFormat = "%s/%s?format=xml";
private String fromDate;
public ThanadosIterator(final String baseUrl, final String listAddress, final String fromDate) {
try {
this.baseUrl = baseUrl;
this.identifiers = downloadIdentifierList(listAddress, fromDate);
this.counter = 0;
this.fromDate = fromDate;
} catch (Exception e) {
throw new CollectorServiceRuntimeException("Error creating iterator", e);
}
}
@Override
public boolean doHasNext() {
return counter < this.identifiers.size();
}
@Override
public String doNext() {
try {
if (counter < this.identifiers.size()) {
String toDownload = String.format(urlFormat, baseUrl, identifiers.get(counter));
log.debug("Downloading "+toDownload);
System.out.println("Downloading "+toDownload);
return download(toDownload);
} else {
throw new CollectorServiceRuntimeException("Iterator has reached the end");
}
} finally {
counter++;
}
}
protected List<String> downloadIdentifierList(final String listUrl, final String fromDate) throws ParseException {
String urlToListItems = listUrl;
if(StringUtils.isNotBlank(fromDate))
urlToListItems = listUrl+"/"+fromDate;
log.info("Getting list of items from "+urlToListItems);
String response = download(urlToListItems);
JSONObject map = (JSONObject)(new JSONParser(JSONParser.MODE_PERMISSIVE).parse(response));
final String sites = map.getAsString("\"sites\"");
Gson converter = new Gson();
Type type = new TypeToken<List<String>>(){}.getType();
return converter.fromJson(sites, type );
}
private String download(final String url) {
log.debug("download: "+url);
final HttpGet method = new HttpGet(url);
try(CloseableHttpResponse response = client.execute(method)) {
int statusCode = response.getStatusLine().getStatusCode();
if (HttpStatus.SC_OK == statusCode) {
return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()), "utf-8");
} else {
throw new CollectorServiceRuntimeException("Error " + statusCode + " dowloading url: " + url);
}
} catch (IOException e) {
throw new CollectorServiceRuntimeException("Error downloading url: " + url);
}
}
public List<String> getIdentifiers() {
return identifiers;
}
public void setIdentifiers(List<String> identifiers) {
this.identifiers = identifiers;
}
}

3
dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/VocabularyDSResourceType/dnet_api_protocols.xml
View File

@ -59,6 +59,9 @@
<TERM code="ads" encoding="ARIADNE" english_name="ads" native_name="ads">
<SYNONYMS/>
</TERM>
<TERM code="thanados" encoding="ARIADNE" english_name="thanados" native_name="thanados">
<SYNONYMS/>
</TERM>
<TERM code="UNKNOWN" encoding="DNET" english_name="UNKNOWN" native_name="UNKNOWN">
<SYNONYMS/>
</TERM>

11
dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml
View File

@ -59,4 +59,15 @@
</bean>
</property>
</bean>
<bean id="thanadosCollectorPlugin" class="eu.dnetlib.data.collector.plugins.ariadneplus.thanados.ThanadosCollectorPlugin">
<property name="protocolDescriptor">
<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="thanados">
<property name="params">
<list>
<bean class="eu.dnetlib.rmi.data.ProtocolParameter" p:name="listUrl"/>
</list>
</property>
</bean>
</property>
</bean>
</beans>

39
dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIteratorTest.java Normal file
View File

@ -0,0 +1,39 @@
package eu.dnetlib.data.collector.plugins.ariadneplus.thanados;
import org.junit.Ignore;
import org.junit.Test;
@Ignore
public class ThanadosIteratorTest {
private String sitelist = "https://thanados.net/sites/sitelist";
ThanadosIterator it;
@Test
public void testDownloadList(){
it = new ThanadosIterator("", sitelist, "");
it.getIdentifiers().stream().forEach(id -> System.out.println(id));
System.out.println(it.getIdentifiers().size());
}
@Test
public void testDownload(){
int count = 0;
it = new ThanadosIterator("https://thanados.openatlas.eu/api/0.3/subunits", sitelist, "");
while(it.hasNext()){
it.next();
count++;
System.out.println(count);
}
System.out.println("DONE");
}
@Test
public void testIncremental(){
it = new ThanadosIterator("", sitelist, "2022-03-12");
it.getIdentifiers().stream().forEach(id -> System.out.println(id));
System.out.println(it.getIdentifiers().size());
}
}