diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 508b1dc..be7d918 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -70,6 +70,15 @@ public class BulkUpload { uri.setLabel(ace.getTypeLabel()); ace.setHas_type(uri); + String uniqueIsPartOf = ace.getUniqueIsPartOf(); + if (uniqueIsPartOf != null) { + ace.setIsPartOf(Arrays.asList(uniqueIsPartOf)); + } + if (ace.getContributor() != null) { + ace.getContributor().clear(); + ace.setContributor(ace.getCreator()); + } + if (ace.getSpatial()!=null) { ace.getSpatial() .stream() @@ -162,17 +171,6 @@ public class BulkUpload { } } - if (!isCollection) { - String uniqueIsPartOf = ace.getUniqueIsPartOf(); - if (uniqueIsPartOf != null) { - ace.setIsPartOf(Arrays.asList(uniqueIsPartOf)); - } - if (ace.getContributor() != null) { - ace.getContributor().clear(); - ace.setContributor(ace.getCreator()); - } - } - String[] splits = ace.getIdentifier().split("/"); log.debug("JSON >>>> "+ace.toJson()); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/enrich/THANADOS/ordered_sparql_insert_THANADOS_test.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/enrich/THANADOS/ordered_sparql_insert_THANADOS_test.sparql new file mode 100644 index 0000000..540a4ce --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/enrich/THANADOS/ordered_sparql_insert_THANADOS_test.sparql @@ -0,0 +1,322 @@ +PREFIX aocat: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?s aocat:has_temporal_coverage ?tcov . + } +} +WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_temporal_coverage ?tcov . + MINUS { + ?s rdf:type . + ?s aocat:has_temporal_coverage ?tc . + } + } +}; + +PREFIX aocat: +PREFIX rdf: +PREFIX skos: +INSERT { + GRAPH { + ?s aocat:has_type . + rdf:type . + skos:prefLabel "collection" + } + } + WHERE { + GRAPH { + ?s rdf:type . + MINUS { + ?s rdf:type . + ?s aocat:has_type ?t . + } + } +}; + +PREFIX aocat: +PREFIX skos: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + rdf:type . + rdfs:label "THANADOS" . + aocat:has_name "THANADOS" . + ?s aocat:has_creator . + ?s aocat:has_contributor . + ?s aocat:has_responsible . + ?s aocat:has_owner . + ?s aocat:has_publisher . + } + } + WHERE { + GRAPH { + ?s rdf:type . + } +}; + +PREFIX aocat: +PREFIX skos: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?s aocat:was_issued "2022" + } + } + WHERE { + GRAPH { + ?s rdf:type . + MINUS{ + ?s rdf:type . + ?s aocat:was_issued ?smt . + } + } +}; + +PREFIX aocat: +PREFIX skos: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?s aocat:has_access_rights "Not provided" + } + } + WHERE { + GRAPH { + ?s rdf:type . + MINUS{ + ?s rdf:type . + ?s aocat:has_access_rights ?smt . + } + } +}; + +PREFIX aocat: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?s aocat:has_spatial_coverage ?scov . + ?scov aocat:has_place_name ?placeName . + } +} +WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_spatial_coverage ?scov . + ?scov aocat:has_place_name ?placeName . + MINUS { + ?s rdf:type . + ?s aocat:has_spatial_coverage ?sc . + ?sc aocat:has_place_name ?name . + } + } +}; +PREFIX aocat: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?s aocat:has_spatial_coverage . + rdf:type . + rdfs:label "Name not provided" . + aocat:has_place_name "Name not provided" . + } +} +WHERE { + GRAPH { + ?s rdf:type aocat:AO_Individual_Data_Resource . + MINUS { + ?s rdf:type aocat:AO_Individual_Data_Resource . + ?s aocat:has_spatial_coverage ?spatialRegion . + ?spatialRegion aocat:has_place_name ?placeName . + } + } +}; + +PREFIX aocat: +PREFIX rdf: +PREFIX rdfs: +PREFIX skos: +INSERT { + GRAPH { + ?s aocat:has_type . + rdf:type . + skos:prefLabel "provided record" + } + } + WHERE { + GRAPH { + ?s rdf:type aocat:AO_Individual_Data_Resource . + MINUS { + ?s rdf:type aocat:AO_Individual_Data_Resource . + ?s aocat:has_type ?t . + } + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:has_creator ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_creator ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:has_creator ?x + } + + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:has_contributor ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_contributor ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:has_contributor ?x + } + + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:has_responsible ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_responsible ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:has_responsible ?x + } + + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:has_owner ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_owner ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:has_owner ?x + } + + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:has_publisher ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_publisher ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:has_publisher ?x + } + + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:was_issued ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:was_issued ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:was_issued ?x + } + + } +}; + +PREFIX aocat: +PREFIX rdf: +INSERT { + GRAPH { + + ?s aocat:has_access_rights ?cr . + + } + } + WHERE { + GRAPH { + ?s rdf:type . + ?s aocat:is_part_of ?coll . + ?coll aocat:has_access_rights ?cr . + MINUS{ + ?s rdf:type . + ?s aocat:has_access_rights ?x + } + + } +}; + + + + diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 14228f7..31762ad 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -26,13 +26,12 @@ import java.util.Properties; * @author enrico.ottonello * */ -//@Ignore +@Ignore public class GraphDbReaderAndESIndexTest { private RunSPARQLQueryService runSPQRLQuery; @Test -// @Ignore public void uploadAMCRFieldworkTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA"; @@ -42,7 +41,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadAMCRFieldworkEventOnlyPeriodoTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/849F814F-5892-3408-AD5E-904938B4492A"; @@ -52,7 +50,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadAMCRDocumentTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/FC59581D-DC3A-31DA-922A-98DE764F3D76"; @@ -62,7 +59,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadAMCRSiteTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/3C7EC936-A7CA-3720-B3DC-413A25754FD4"; @@ -72,7 +68,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadAMCRIndividualFindTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/98D717C4-410F-35C6-8072-FABA7686B4A3"; @@ -82,7 +77,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadADSRecordWithNativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF"; @@ -92,7 +86,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA"; @@ -102,7 +95,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadZbivaRecordSpatialTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479"; @@ -112,7 +104,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadHNMCollectionSpatialTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5"; @@ -122,7 +113,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadHNMCollectionTemporalTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10"; @@ -132,7 +122,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadADSArchivesBoundingBoxTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/90D1C95D-E249-3E74-92D9-B58FDF690CC7"; @@ -142,7 +131,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDansSpatialTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD"; @@ -152,7 +140,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDansNativePeriodTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B"; @@ -162,7 +149,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDansTemporalPolygonTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B"; @@ -172,7 +158,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadSndRockartTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A"; @@ -182,7 +167,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadNIAMTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1"; @@ -192,7 +176,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadADS398Test() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73"; @@ -202,7 +185,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadROADAfterFullReloadTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E859C58E-0307-3D7C-BCAB-45483DEA917C"; @@ -212,7 +194,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadADS328NativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559"; @@ -222,7 +203,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadFastiMultipleAriadneSubjectTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/FE740BC7-2D9E-3644-A88E-3E8E5A5EA95F"; @@ -232,7 +212,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadSNDArchivePolygonTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/A7A2588A-1FE7-3FDD-AD66-C84EACE1860E"; @@ -242,7 +221,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadFastiTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/532D492C-B141-30D0-886A-1E69C2C20474"; @@ -252,7 +230,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDansClosedAccessTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D1C1D6AD-320D-3616-B75D-F2E2C3DA7F3E"; @@ -262,7 +239,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDansRestrictedAccessTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/221AAEC8-CC9B-310C-B800-0DFE34850E0B"; @@ -272,7 +248,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDansOpenAccessTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/B251A0FE-EFE0-341B-A8B3-E0813CC82856"; @@ -282,7 +257,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadConicetCollectionTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IDACOR%20%28CONICET%2FUNC%29/8B238475-2BA7-3036-88C0-968E9A642E64"; @@ -292,7 +266,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadConicetRecordTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CONICET/0CD6BF2E-4AB7-39B6-937E-7DD59092DD3F"; @@ -302,7 +275,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadNaraTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/001CA559-D688-31FB-AB48-45CC8DB1E355"; @@ -312,7 +284,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadINFNTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/INFN/5AE70226-82A9-3B06-9E25-8FB71B64C51C"; @@ -322,7 +293,6 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore public void uploadADSGreyLiteratureOver300TriplesTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/35AEC455-6CB5-31BB-B73E-30959FF3C4FE"; @@ -332,7 +302,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDimeBoundingBoxTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DIME/0E5FB5C7-6FCA-36AD-BEE5-03C9AFF393CD"; @@ -342,7 +311,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadNormalizerTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DCDF395A-2CC8-3FC2-B9A5-5A924090DF10"; @@ -352,7 +320,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadNormalizerTest2() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/A0D9DC1F-2C62-3423-BAE6-4248D5072585"; @@ -362,7 +329,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadNormalizerAllUppercaseTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E644B5AD-3DC2-3FEC-A070-B7FD434B0E90"; @@ -372,7 +338,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadNormalizerMinusTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/9F96015D-84FA-3BDE-AE3C-90CF3CC7CEDB"; @@ -382,7 +347,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadCeniehTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0F55399D-8001-36E6-8D52-0C32981CAD7E"; @@ -392,7 +356,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadDoliaTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/F0E19AC1-AE02-3BE5-AF78-D9EEC1670AC1"; @@ -404,7 +367,6 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore public void uploadIAVPFastCatCollectionWasCreatedTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IAVP/E60F992F-C34D-39B5-99AD-406BA27B090B"; @@ -413,6 +375,24 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test + public void collectionInCollectionTest() throws Exception { + boolean isRecord = false; + String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/THANADOS/3E6614E4-EA97-3E7D-BA72-B0343A63FA39"; + String datasource = "thanados"; + String collectionId = "test"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + + @Test + public void recordInCollectionTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/THANADOS/46FD94FF-0C91-369A-A64F-E29B7C9A40D9"; + String datasource = "thanados"; + String collectionId = "test"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java index 6f26351..24f8e7a 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java @@ -18,12 +18,14 @@ import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.RDFWriter; import org.eclipse.rdf4j.rio.Rio; +import org.junit.Ignore; import org.junit.Test; import org.springframework.core.io.ClassPathResource; import java.io.StringWriter; import java.nio.charset.StandardCharsets; +@Ignore public class GraphDbReaderTest { private static final Log log = LogFactory.getLog(GraphDbReaderTest.class); @@ -86,9 +88,10 @@ public class GraphDbReaderTest { } private boolean openConnection() throws Exception { + manager = new RemoteRepositoryManager(getGraphDBUrl()); manager.init(); - manager.setUsernameAndPassword(getUsername(), getPwd()); + // manager.setUsernameAndPassword(getUsername(), getPwd()); try { repository = manager.getRepository(getGraphDBRepository()); } catch (UnauthorizedException e) { diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java index 883bea9..98e45c5 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJsonTest.java @@ -23,7 +23,7 @@ public class ParseRDFJsonTest { public void setup(){ parser = new ParseRDFJSON(); parser.setCatalogEntryCollectionJsonPath("$[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Collection')]"); - parser.setCatalogEntryJsonPath("$[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Record')]"); + parser.setCatalogEntryJsonPath("[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Record')]"); mng = new ResourceManager(); mng.setup("https://www.ariadne-infrastructure.eu/property/resourceType", "eu.dnetlib.ariadneplus.elasticsearch.model.", "[\"https://www.ariadne-infrastructure.eu/property/resourceType\", \"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\"]", diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json index 0448b02..24dd6e6 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/collection.json @@ -21,6 +21,19 @@ "lang" : "en" } ], + "https://www.ariadne-infrastructure.eu/property/rdfType" : [ + { + "value" : "Collection", + "type" : "literal", + "datatype" : "http://www.w3.org/2001/XMLSchema#string" + } + ], + "https://www.ariadne-infrastructure.eu/property/partOf" : [ + { + "value" : "https://ariadne-infrastructure.eu/aocat/Collection/CENIEH/A4A042CF-4FD6-3FB4-B701-CBC8A1653ADA", + "type" : "uri" + } + ], "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType" : [ { "value" : "https://ariadne-infrastructure.eu/aocat/Concept/AO_Subject/Fieldwork", @@ -100,7 +113,7 @@ ], "https://www.ariadne-infrastructure.eu/property/resourceType" : [ { - "value" : "Dataset Collection", + "value" : "Collection", "type" : "literal", "datatype" : "http://www.w3.org/2001/XMLSchema#string" } diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json index f365663..e0c3e76 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/eu/dnetlib/ariadneplus/reader/json/spec.json @@ -1,300 +1,458 @@ { -"AriadneTemporal": { -"class_type": "prototype", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/from": { -"class_field": "From", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/periodName": { -"class_field": "PeriodName", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/until": { -"class_field": "Until", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/uri": { -"class_field": "Uri", -"substring": "no", -"element_type": "java.lang.String" -} -} -}, -"AriadneGeoPoint": { -"class_type": "unique", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/lat": { -"class_field": "Lat", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/lon": { -"class_field": "Lon", -"substring": "no", -"element_type": "java.lang.String" -} -} -}, -"Spatial": { -"class_type": "prototype", -"mappings":{"https://www.ariadne-infrastructure.eu/property/address": { -"class_field": "Address", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/coordinateSystem": { -"class_field": "CoordinateSystem", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/country": { -"class_field": "Country", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/location": { -"class_field": "Location", -"external_reference": "AriadneGeoPoint", -"substring": "no" -}, -"https://www.ariadne-infrastructure.eu/property/lat": { -"class_field": "Lat", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/lon": { -"class_field": "Lon", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/placeName": { -"class_field": "PlaceName", -"substring": "no", -"element_type": "java.lang.String" -}} -}, -"ArchaeologicalResourceType": { -"class_type": "unique", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/id": { -"class_field": "Id", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/name": { -"class_field": "Name", -"substring": "no", -"element_type": "java.lang.String" -} -} -}, -"AgentInfo": { -"class_type": "prototype", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/name": { -"class_field": "Name", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/type": { -"class_field": "Type", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/email": { -"class_field": "Email", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/phone": { -"class_field": "Phone", -"substring": "no", -"element_type": "java.lang.String" -} -} -}, -"NativeSubject": { -"class_type": "prototype", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/prefLabel": { -"class_field": "PrefLabel", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/rdfAbout": { -"class_field": "RdfAbout", -"substring": "yes", -"element_type": "java.lang.String" -} -} -}, -"DerivedSubject": { -"class_type": "prototype", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/prefLabel": { -"class_field": "PrefLabel", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/source": { -"class_field": "Source", -"substring": "yes", -"element_type": "java.lang.String" -} -} -}, -"AatSubject": { -"class_type": "prototype", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/id": { -"class_field": "Id", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/label": { -"class_field": "Label", -"substring": "yes", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/lang": { -"class_field": "Lang", -"substring": "yes", -"element_type": "java.lang.String" -} -} -}, -"AriadneCatalogEntry": { -"class_type": "unique", -"mappings": { -"https://www.ariadne-infrastructure.eu/property/accessPolicy": { -"class_field": "AccessPolicy", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/identifier": { -"class_field": "Identifier", -"substring": "yes", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/publisher": { -"class_field": "Publisher", -"external_reference": "AgentInfo", -"substring": "no" -}, -"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": { -"class_field": "ArchaeologicalResourceType", -"external_reference": "ArchaeologicalResourceType", -"substring": "no" -}, -"https://www.ariadne-infrastructure.eu/property/issued": { -"class_field": "Issued", -"element_type": "java.lang.String", -"substring": "no" -}, -"https://www.ariadne-infrastructure.eu/property/resourceType": { -"class_field": "ResourceType", -"element_type": "java.lang.String", -"substring": "no" -}, -"https://www.ariadne-infrastructure.eu/property/modified": { -"class_field": "Modified", -"element_type": "java.lang.String", -"substring": "no" -}, -"https://www.ariadne-infrastructure.eu/property/nativeSubject": { -"class_field": "NativeSubject", -"substring": "no", -"external_reference": "NativeSubject" -}, -"https://www.ariadne-infrastructure.eu/property/derivedSubject": { -"class_field": "DerivedSubject", -"substring": "no", -"external_reference": "DerivedSubject" -}, -"https://www.ariadne-infrastructure.eu/property/aatSubjects": { -"class_field": "AatSubjects", -"substring": "no", -"external_reference": "AatSubject" -}, -"https://www.ariadne-infrastructure.eu/property/spatialRegion": { -"class_field": "Spatial", -"substring": "no", -"external_reference": "Spatial" -}, -"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": { -"class_field": "Spatial", -"substring": "no", -"external_reference": "Spatial" -}, -"https://www.ariadne-infrastructure.eu/property/landingPage": { -"class_field": "LandingPage", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/placeName": { -"class_field": "PlaceName", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/title": { -"class_field": "Title", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/accessRights": { -"class_field": "AccessRights", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/description": { -"class_field": "Description", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/creator": { -"class_field": "Creator", -"substring": "no", -"external_reference": "AgentInfo" -}, -"https://www.ariadne-infrastructure.eu/property/contributor": { -"class_field": "Contributor", -"substring": "no", -"external_reference": "AgentInfo" -}, -"https://www.ariadne-infrastructure.eu/property/legalResponsible": { -"class_field": "LegalResponsible", -"substring": "no", -"external_reference": "AgentInfo" -}, -"https://www.ariadne-infrastructure.eu/property/owner": { -"class_field": "Owner", -"substring": "no", -"external_reference": "AgentInfo" -}, -"https://www.ariadne-infrastructure.eu/property/originalId": { -"class_field": "OriginalId", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/uri": { -"class_field": "Temporal", -"substring": "no", -"external_reference": "AriadneTemporal" -}, -"https://www.ariadne-infrastructure.eu/property/language": { -"class_field": "Language", -"substring": "no", -"element_type": "java.lang.String" -}, -"https://www.ariadne-infrastructure.eu/property/partOf": { -"class_field": "UniqueIsPartOf", -"substring": "no", -"element_type": "java.lang.String" -} -} -} + "TextLang": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/text": { + "class_field": "Text", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/lang": { + "class_field": "Language", + "substring": "yes", + "element_type": "java.lang.String" + } + } + }, + "DigitalImage": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/primaryVisualComponent": { + "class_field": "ProviderUri", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/visualComponent": { + "class_field": "AriadneUri", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "AriadneResource": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/aboutURILabel": { + "class_field": "Label", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/aboutURI": { + "class_field": "Uri", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "AriadneGeoPoint": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/lat": { + "class_field": "Lat", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/lon": { + "class_field": "Lon", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "Temporal": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/from": { + "class_field": "From", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/periodOName": { + "class_field": "PeriodName", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/until": { + "class_field": "Until", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/uri": { + "class_field": "Uri", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "NativePeriod": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/nativeFrom": { + "class_field": "From", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/nativePeriodName": { + "class_field": "PeriodName", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/nativeUntil": { + "class_field": "Until", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "Spatial": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/address": { + "class_field": "Address", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/lat": { + "class_field": "Lat", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/lon": { + "class_field": "Lon", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/placeName": { + "class_field": "PlaceName", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/boxMaxLat": { + "class_field": "BoundingBoxMaxLat", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/boxMinLat": { + "class_field": "BoundingBoxMinLat", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/boxMaxLon": { + "class_field": "BoundingBoxMaxLon", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/boxMinLon": { + "class_field": "BoundingBoxMinLon", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/precisionUnit": { + "class_field": "SpatialPrecision", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/precisionValue": { + "class_field": "CoordinatePrecision", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/geoPoint": { + "class_field": "PolygonGeoPoints", + "substring": "no", + "element_type": "java.lang.String", + "external_reference": "AriadneGeoPoint" + }, + "https://www.ariadne-infrastructure.eu/property/wkt": { + "class_field": "Wkt", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "AriadneSubject": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/name": { + "class_field": "PrefLabel", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "AgentInfo": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/name": { + "class_field": "Name", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/type": { + "class_field": "Type", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/homepage": { + "class_field": "Homepage", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/institution": { + "class_field": "Institution", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/agentIdentifier": { + "class_field": "AgentIdentifier", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "PublisherInfo": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/publisherName": { + "class_field": "Name", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/type": { + "class_field": "Type", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/publisherEmail": { + "class_field": "Email", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/publisherHomepage": { + "class_field": "Homepage", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/publisherInstitution": { + "class_field": "Institution", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/publisherAgentIdentifier": { + "class_field": "AgentIdentifier", + "substring": "no", + "element_type": "java.lang.String" + } + } + }, + "NativeSubject": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/prefLabel": { + "class_field": "PrefLabel", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/rdfAbout": { + "class_field": "RdfAbout", + "substring": "yes", + "element_type": "java.lang.String" + } + } + }, + "DerivedSubject": { + "class_type": "prototype", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/prefLabel": { + "class_field": "PrefLabel", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/source": { + "class_field": "Source", + "substring": "yes", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/id": { + "class_field": "Id", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/label": { + "class_field": "PrefLabel", + "substring": "yes", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/lang": { + "class_field": "Lang", + "substring": "yes", + "element_type": "java.lang.String" + } + } + }, + "AriadnePlusEntry": { + "class_type": "unique", + "mappings": { + "https://www.ariadne-infrastructure.eu/property/typeURI": { + "class_field": "TypeURI", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/typeLabel": { + "class_field": "TypeLabel", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/accessPolicy": { + "class_field": "AccessPolicy", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/identifier": { + "class_field": "Identifier", + "substring": "yes", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/publisher": { + "class_field": "Publisher", + "external_reference": "PublisherInfo", + "substring": "no" + }, + "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": { + "class_field": "AriadneSubject", + "external_reference": "AriadneSubject", + "substring": "no" + }, + "https://www.ariadne-infrastructure.eu/property/issued": { + "class_field": "Issued", + "element_type": "java.lang.String", + "substring": "no" + }, + "https://www.ariadne-infrastructure.eu/property/created": { + "class_field": "WasCreated", + "element_type": "java.lang.String", + "substring": "no" + }, + "https://www.ariadne-infrastructure.eu/property/resourceType": { + "class_field": "ResourceType", + "element_type": "java.lang.String", + "substring": "no" + }, + "https://www.ariadne-infrastructure.eu/property/modified": { + "class_field": "Modified", + "element_type": "java.lang.String", + "substring": "no" + }, + "https://www.ariadne-infrastructure.eu/property/nativeSubject": { + "class_field": "NativeSubject", + "substring": "no", + "external_reference": "NativeSubject" + }, + "https://www.ariadne-infrastructure.eu/property/derivedSubject": { + "class_field": "DerivedSubject", + "substring": "no", + "external_reference": "DerivedSubject" + }, + "https://www.ariadne-infrastructure.eu/property/spatialRegion": { + "class_field": "Spatial", + "substring": "no", + "external_reference": "Spatial" + }, + "https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": { + "class_field": "Spatial", + "substring": "no", + "external_reference": "Spatial" + }, + "https://www.ariadne-infrastructure.eu/property/spatialRegionBox": { + "class_field": "Spatial", + "substring": "no", + "external_reference": "Spatial" + }, + "https://www.ariadne-infrastructure.eu/property/spatialRegionPolygon": { + "class_field": "Spatial", + "substring": "no", + "external_reference": "Spatial" + }, + "https://www.ariadne-infrastructure.eu/property/spatialRegionWKTPolygon": { + "class_field": "Spatial", + "substring": "no", + "external_reference": "Spatial" + }, + "https://www.ariadne-infrastructure.eu/property/landingPage": { + "class_field": "LandingPage", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/titleInfo": { + "class_field": "Title", + "substring": "no", + "external_reference": "TextLang" + }, + "https://www.ariadne-infrastructure.eu/property/accessRights": { + "class_field": "AccessRights", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/descriptionInfo": { + "class_field": "Description", + "substring": "no", + "external_reference": "TextLang" + }, + "https://www.ariadne-infrastructure.eu/property/creator": { + "class_field": "Creator", + "substring": "no", + "external_reference": "AgentInfo" + }, + "https://www.ariadne-infrastructure.eu/property/contributor": { + "class_field": "Contributor", + "substring": "no", + "external_reference": "AgentInfo" + }, + "https://www.ariadne-infrastructure.eu/property/legalResponsible": { + "class_field": "Responsible", + "substring": "no", + "external_reference": "AgentInfo" + }, + "https://www.ariadne-infrastructure.eu/property/owner": { + "class_field": "Owner", + "substring": "no", + "external_reference": "AgentInfo" + }, + "https://www.ariadne-infrastructure.eu/property/originalId": { + "class_field": "OriginalId", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/temporal": { + "class_field": "Temporal", + "substring": "no", + "external_reference": "Temporal" + }, + "https://www.ariadne-infrastructure.eu/property/temporalNative": { + "class_field": "NativePeriod", + "substring": "no", + "external_reference": "NativePeriod" + }, + "https://www.ariadne-infrastructure.eu/property/language": { + "class_field": "Language", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/partOf": { + "class_field": "UniqueIsPartOf", + "substring": "no", + "element_type": "java.lang.String" + }, + "https://www.ariadne-infrastructure.eu/property/about": { + "class_field": "Is_about", + "substring": "no", + "external_reference": "AriadneResource" + }, + "https://www.ariadne-infrastructure.eu/property/image": { + "class_field": "DigitalImage", + "substring": "no", + "external_reference": "DigitalImage" + } + } + } } \ No newline at end of file diff --git a/dnet-ariadneplus/pom.xml b/dnet-ariadneplus/pom.xml index 3abf3a6..ce4e2e4 100644 --- a/dnet-ariadneplus/pom.xml +++ b/dnet-ariadneplus/pom.xml @@ -36,6 +36,11 @@ dnet-msro-service [7.0.0-SAXONHE-SOLR772-SNAPSHOT, 8.0.0-SAXONHE) + + net.minidev + json-smart + 2.4.8 + diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosCollectorPlugin.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosCollectorPlugin.java new file mode 100644 index 0000000..946e0a3 --- /dev/null +++ b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosCollectorPlugin.java @@ -0,0 +1,21 @@ +package eu.dnetlib.data.collector.plugins.ariadneplus.thanados; + +import eu.dnetlib.data.collector.plugins.httplist.HttpListIterator; +import eu.dnetlib.rmi.data.CollectorServiceException; +import eu.dnetlib.rmi.data.InterfaceDescriptor; +import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; + +import java.util.Date; + +public class ThanadosCollectorPlugin extends AbstractCollectorPlugin { + + @Override + public Iterable collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate) { + final String baseUrl = interfaceDescriptor.getBaseUrl(); + final String listAddress = interfaceDescriptor.getParams().get("listUrl"); + + return () -> new ThanadosIterator(baseUrl, listAddress, fromDate); + } +} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIterator.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIterator.java new file mode 100644 index 0000000..1199a39 --- /dev/null +++ b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIterator.java @@ -0,0 +1,111 @@ +package eu.dnetlib.data.collector.plugins.ariadneplus.thanados; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import eu.dnetlib.data.collector.ThreadSafeIterator; +import eu.dnetlib.rmi.data.CollectorServiceRuntimeException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.List; + +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; + +public class ThanadosIterator extends ThreadSafeIterator { + + private static final Log log = LogFactory.getLog(ThanadosIterator.class); + private final CloseableHttpClient client = HttpClients.createDefault(); + + private String baseUrl; + private List identifiers; + private int counter = 0; + private String urlFormat = "%s/%s?format=xml"; + private String fromDate; + + + public ThanadosIterator(final String baseUrl, final String listAddress, final String fromDate) { + try { + this.baseUrl = baseUrl; + this.identifiers = downloadIdentifierList(listAddress, fromDate); + this.counter = 0; + this.fromDate = fromDate; + } catch (Exception e) { + throw new CollectorServiceRuntimeException("Error creating iterator", e); + } + } + + @Override + public boolean doHasNext() { + return counter < this.identifiers.size(); + } + + @Override + public String doNext() { + try { + if (counter < this.identifiers.size()) { + String toDownload = String.format(urlFormat, baseUrl, identifiers.get(counter)); + log.debug("Downloading "+toDownload); + System.out.println("Downloading "+toDownload); + return download(toDownload); + } else { + throw new CollectorServiceRuntimeException("Iterator has reached the end"); + } + } finally { + counter++; + } + } + + protected List downloadIdentifierList(final String listUrl, final String fromDate) throws ParseException { + String urlToListItems = listUrl; + if(StringUtils.isNotBlank(fromDate)) + urlToListItems = listUrl+"/"+fromDate; + log.info("Getting list of items from "+urlToListItems); + String response = download(urlToListItems); + JSONObject map = (JSONObject)(new JSONParser(JSONParser.MODE_PERMISSIVE).parse(response)); + final String sites = map.getAsString("\"sites\""); + Gson converter = new Gson(); + Type type = new TypeToken>(){}.getType(); + return converter.fromJson(sites, type ); + + } + + private String download(final String url) { + log.debug("download: "+url); + final HttpGet method = new HttpGet(url); + + try(CloseableHttpResponse response = client.execute(method)) { + int statusCode = response.getStatusLine().getStatusCode(); + + if (HttpStatus.SC_OK == statusCode) { + return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()), "utf-8"); + } else { + throw new CollectorServiceRuntimeException("Error " + statusCode + " dowloading url: " + url); + } + + } catch (IOException e) { + throw new CollectorServiceRuntimeException("Error downloading url: " + url); + } + } + + + public List getIdentifiers() { + return identifiers; + } + + public void setIdentifiers(List identifiers) { + this.identifiers = identifiers; + } +} diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/VocabularyDSResourceType/dnet_api_protocols.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/VocabularyDSResourceType/dnet_api_protocols.xml index c3d1972..4f27b7c 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/VocabularyDSResourceType/dnet_api_protocols.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/VocabularyDSResourceType/dnet_api_protocols.xml @@ -59,6 +59,9 @@ + + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml index dcb36f2..1be5de1 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml @@ -59,4 +59,15 @@ + + + + + + + + + + + diff --git a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIteratorTest.java b/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIteratorTest.java new file mode 100644 index 0000000..ec2bac3 --- /dev/null +++ b/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIteratorTest.java @@ -0,0 +1,39 @@ +package eu.dnetlib.data.collector.plugins.ariadneplus.thanados; + +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class ThanadosIteratorTest { + + private String sitelist = "https://thanados.net/sites/sitelist"; + ThanadosIterator it; + + @Test + public void testDownloadList(){ + it = new ThanadosIterator("", sitelist, ""); + it.getIdentifiers().stream().forEach(id -> System.out.println(id)); + System.out.println(it.getIdentifiers().size()); + } + + @Test + public void testDownload(){ + int count = 0; + it = new ThanadosIterator("https://thanados.openatlas.eu/api/0.3/subunits", sitelist, ""); + while(it.hasNext()){ + it.next(); + count++; + System.out.println(count); + } + System.out.println("DONE"); + } + + + @Test + public void testIncremental(){ + it = new ThanadosIterator("", sitelist, "2022-03-12"); + it.getIdentifiers().stream().forEach(id -> System.out.println(id)); + System.out.println(it.getIdentifiers().size()); + } + +}