From aa1c6c2fda9a4debf6c7f5248f2256bb6bb19e39 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 9 Jun 2020 11:39:56 +0200 Subject: [PATCH 1/8] added query to generate aat and periodo enriched graphs --- .../ariadneplus/gen_aatplus_graph.sparql | 33 ++++ .../ariadneplus/gen_periodoplus_graph.sparql | 23 +++ .../ariadneplus/read_record_data.sparql | 164 ++++++++++++++++++ .../parthenos/sparql/D14_Software.sparql | 106 ----------- .../sparql/E29_Design_or_Procedure.sparql | 51 ------ .../dnetlib/parthenos/sparql/E39_Actor.sparql | 68 -------- .../parthenos/sparql/E78_Collection.sparql | 73 -------- .../parthenos/sparql/PE18_Dataset.sparql | 138 --------------- .../parthenos/sparql/PE1_Service.sparql | 148 ---------------- .../parthenos/sparql/PE35_Project.sparql | 62 ------- .../parthenos/sparql/rdfs_rule_set_isql.txt | 76 -------- .../src/main/resources/query.txt | 51 +++--- 12 files changed, 246 insertions(+), 747 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_aatplus_graph.sparql create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_periodoplus_graph.sparql create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/read_record_data.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_aatplus_graph.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_aatplus_graph.sparql new file mode 100644 index 0000000..8565b35 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_aatplus_graph.sparql @@ -0,0 +1,33 @@ +PREFIX aocat: +PREFIX skos: +PREFIX time: +PREFIX ariadneplus: +PREFIX rdf: +INSERT { + GRAPH { + ?record aocat:has_derived_subject ?aat . + } +} +USING +USING +WHERE { + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:exactMatch ?aat . + } + union + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:broadMatch ?aat . + } + union + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:closeMatch ?aat . + } + union + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:narrowMatch ?aat . + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_periodoplus_graph.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_periodoplus_graph.sparql new file mode 100644 index 0000000..77284a8 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_periodoplus_graph.sparql @@ -0,0 +1,23 @@ +PREFIX aocat: +PREFIX skos: +PREFIX time: +PREFIX ariadneplus: +INSERT { + GRAPH { + ?temporal aocat:has_period ?periodO . + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } +} +WHERE { + ?temporal aocat:has_native_period ?native_period . + ?native_period skos:prefLabel ?native_label . + optional { + ?periodO skos:altLabel ?native_label . + ?periodO skos:inScheme . + ?periodO time:intervalStartedBy ?intervalStartedBy . + ?intervalStartedBy skos:prefLabel ?temporalFrom . + ?periodO time:intervalFinishedBy ?intervalFinishedBy . + ?intervalFinishedBy skos:prefLabel ?temporalUntil . + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/read_record_data.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/read_record_data.sparql new file mode 100644 index 0000000..2e94223 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/read_record_data.sparql @@ -0,0 +1,164 @@ +PREFIX aocat: +PREFIX rdfs: +PREFIX skos: +PREFIX aoprop: +PREFIX onto: +PREFIX ariadneplus: +PREFIX time: + +CONSTRUCT { +?record aoprop:identifier ?record . +?record aoprop:originalId ?originalId . +?record aoprop:issued ?issued . +?record aoprop:modified ?modified . +?record aoprop:partOf ?partOf . +?record aoprop:creator ?creator . +?creator aoprop:name ?creatorName . +?creator aoprop:email ?creatorEmail . +?record aoprop:contributor ?contributor . +?contributor aoprop:name ?contributorName . +?contributor aoprop:email ?contributorEmail . +?record aoprop:legalResponsible ?legalResponsible . +?legalResponsible aoprop:name ?legalResponsibleName . +?legalResponsible aoprop:email ?legalResponsibleEmail . +?record aoprop:owner ?owner . +?owner aoprop:name ?ownerName . +?owner aoprop:email ?ownerEmail . +?record aoprop:publisher ?publisher . +?publisher aoprop:name ?publisherName . +?publisher aoprop:email ?publisherEmail . +?record aoprop:accessPolicy ?accessPolicy . +?record aoprop:accessRights ?accessRights . +?record aoprop:landingPage ?landingPage . +?record aoprop:spatialRegion ?spatialRegion . +?spatialRegion aoprop:placeName ?spatialPlaceName . +?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . +?record aoprop:spatialRegionPoint ?spatialRegionPoint . +?spatialRegionPoint aoprop:lat ?spatialLocationLat . +?spatialRegionPoint aoprop:lon ?spatialLocationLon . +?record aoprop:spatialRegionBox ?spatialRegionBox . +?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat . +?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . +?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . +?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +?record aoprop:uri ?temporal . +?temporal aoprop:periodName ?temporalPeriodName . +?temporal aoprop:from ?temporalFrom . +?temporal aoprop:until ?temporalUntil . +?record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . +?record aoprop:archeologicalResourceType ?archeologicalResourceType . +?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . +?record aoprop:resourceType ?resourceType . +?record aoprop:nativeSubject ?nativeSubject . +?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . +?nativeSubject aoprop:rdfAbout ?nativeSubject . +?record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +?record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?record aoprop:title ?title . +?record aoprop:description ?description . +?record aoprop:language ?language . +} +from +from +from +from +from +where { + ?record aocat:has_language / skos:prefLabel ?language . + ?record aocat:has_original_id ?originalId . + ?record aocat:is_part_of ?partOf . + ?record aocat:has_creator ?creator . + ?creator aocat:has_name ?creatorName . + ?record aocat:has_title ?title . + ?record aocat:has_type / skos:prefLabel ?resourceType . + ?record aocat:has_native_subject ?nativeSubject . + ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . + optional { + ?record aocat:has_derived_subject ?derivedSubject . + ?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel . + } + optional { + ?creator aocat:has_email ?creatorEmail . + } + optional { + ?record aocat:has_description ?description . + } + optional { + ?record aocat:has_access_policy / rdfs:label ?accessPolicy . + } + optional { + ?record aocat:has_landing_page / rdfs:label ?landingPage . + } + optional { + ?record aocat:has_temporal_coverage ?temporalNative . + ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . + } + optional { + ?record aocat:has_temporal_coverage ?temporal . + ?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName . + optional { + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } + } + + { + select * + where { + ?record aocat:is_part_of ?collection . + ?collection aocat:was_issued ?issued . + ?collection aocat:was_modified ?modified . + ?collection aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + ?collection aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + ?collection aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + ?collection aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + ?collection aocat:has_access_rights ?accessRights . + ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } + } + } + { + select * + where { + ?record aocat:has_spatial_coverage ?spatialRegion . + ?spatialRegion aocat:has_place_name ?spatialPlaceName . + optional { + ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem . + } + optional { + ?record aocat:has_spatial_coverage ?spatialRegionPoint . + ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; + aocat:has_longitude ?spatialLocationLon . + } + optional { + ?record aocat:has_spatial_coverage ?spatialRegionBox . + ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; + aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ; + aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ; + aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ; + } + } + } + +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql deleted file mode 100644 index f8fa1bd..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql +++ /dev/null @@ -1,106 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - - <${subjectURL}> crm:P102_has_title ?title . - <${subjectURL}> crm:P3_has_note ?description . - <${subjectURL}> a ?type . - <${subjectURL}> rdfs:label ?label . - - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - - <${subjectURL}> crm:P2_has_type ?e55TypeLabel . - - <${subjectURL}> crmpe:PP7i_is_software_object_hosted_by ?host . - <${subjectURL}> crmpe:PP32i_is_curated_by ?curatingService . - - <${subjectURL}> crmpe:PP17_has_snapshot ?snapshot1 . - <${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshot2 . - - <${subjectURL}> crmpe:PP21_has_software_part ?sw1 . - <${subjectURL}> crmpe:PP21i_is_software_part_of ?sw2 . - <${subjectURL}> crmpe:PP22_has_release ?sw3 . - <${subjectURL}> crmpe:PP22i_is_release_of ?sw4 . - - <${subjectURL}> ?adopter . - <${subjectURL}> ?creationtime . -} -WHERE { - <${subjectURL}> a crmdig:D14_Software . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} -UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . -} - UNION{ - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - } - UNION{<${subjectURL}> crmpe:PP7i_is_software_object_hosted_by ?host . } - UNION{?host crmpe:PP7_hosts_software_object <${subjectURL}> . } - UNION{<${subjectURL}> crmpe:PP32i_is_curated_by ?curatingService .} - UNION{?curatingService crmpe:PP32_curates <${subjectURL}> .} - UNION{ - <${subjectURL}> crmpe:PP32i_is_curated_by ?curatingService . - ?curatingService crmpe:PP31_uses_curation_plan ?curationplan . - } -UNION{ - ?curatingService crmpe:PP32_curates <${subjectURL}> . - ?curatingService crmpe:PP31_uses_curation_plan ?curationplan . -} - UNION{<${subjectURL}> crmpe:PP17_has_snapshot ?snapshot1 .} - UNION{ ?snapshot1 crmpe:PP17i_is_snapshot_of <${subjectURL}> .} - UNION{<${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshot2 .} - UNION{ ?snapshot2 crmpe:PP17_has_snapshot <${subjectURL}> .} - - UNION{<${subjectURL}> crmpe:PP21_has_software_part ?sw1 .} - UNION{?sw1 crmpe:PP21i_is_software_part_of <${subjectURL}>.} - - UNION{ ?sw2 crmpe:PP21_has_software_part <${subjectURL}> .} - UNION{<${subjectURL}> crmpe:PP21i_is_software_part_of ?sw2 .} - - UNION{<${subjectURL}> crmpe:PP22_has_release ?sw3 .} - UNION{?sw3 crmpe:PP22i_is_release_of <${subjectURL}> .} - - UNION{<${subjectURL}> crmpe:PP22i_is_release_of ?sw4 .} - UNION{?sw4 crmpe:PP22_has_release <${subjectURL}> .} - - UNION{ - ?adopter crm:P94i_was_created_by ?event . - ?event crmdig:L23_used_software_or_firmware <${subjectURL}> . - } - UNION { - <${subjectURL}> crmdig:L23i_was_software_or_firmware_used_by ?event . - ?adopter crm:P94i_was_created_by ?event . - } - UNION{ - <${subjectURL}> crm:P94i_was_created_by ?event1 . - ?event1 crm:P2_has_type . - ?event1 crm:P4_has_time-span ?timespan . - ?timespan crm:P81_ongoing_throughout ?creationtime . - } -} -LIMIT 1000 \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql deleted file mode 100644 index 5ef1d16..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql +++ /dev/null @@ -1,51 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . -<${subjectURL}> crm:P3_has_note ?description . -<${subjectURL}> a ?type . -<${subjectURL}> rdfs:label ?label . - -<${subjectURL}> crm:P2_has_type ?e55TypeLabel . - -<${subjectURL}> ?adopter . -<${subjectURL}> ?creationtime . -} -WHERE { - <${subjectURL}> a crm:E29_Design_or_Procedure . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} -UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . -} -UNION { - ?adopter crmpe:PP31_uses_curation_plan <${subjectURL}> . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?event1 . - ?event1 crm:P2_has_type . - ?event1 crm:P4_has_time-span ?timespan . - ?timespan crm:P81_ongoing_throughout ?creationtime . -} -} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql deleted file mode 100644 index dfbad19..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql +++ /dev/null @@ -1,68 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { -<${subjectURL}> rdfs:label ?label . -<${subjectURL}> a ?type . -<${subjectURL}> crm:P2_has_type ?e55TypeLabel . - -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . -<${subjectURL}> crm:P3_has_note ?description . - -<${subjectURL}> crm:P76_has_contact_point ?contactpoint . -?contactpoint crm:P2_has_type ?contactpointType . -?contactpoint rdfs:label ?contactpointLabel . -?contactpointType rdfs:label ?contactpointTypeLabel . - -<${subjectURL}> crm:P107_has_current_or_former_member ?member1 . -<${subjectURL}> crm:P107i_is_current_or_former_member_of ?team . -<${subjectURL}> crmpe:PP2i_provides ?service . - -<${subjectURL}> crmpe:PP44i_is_maintaining_team_of ?maintained . -} -WHERE { - <${subjectURL}> a crm:E39_Actor . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ <${subjectURL}> crm:P3_has_note ?description .} -UNION{ <${subjectURL}> rdfs:label ?label .} -UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . -} -UNION{ - <${subjectURL}> crm:P76_has_contact_point ?contactpoint . -} - UNION{ - <${subjectURL}> crm:P76_has_contact_point ?contactpoint . - ?contactpoint rdfs:label ?contactpointLabel . -} -UNION{ - <${subjectURL}> crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpointType rdfs:label ?contactpointTypeLabel . - } - UNION{ <${subjectURL}> crm:P107_has_current_or_former_member ?member1 .} - UNION{ ?member1 crm:P107_has_current_or_former_member <${subjectURL}> .} - UNION{ <${subjectURL}> crm:P107i_is_current_or_former_member_of ?team .} - UNION{ ?team crm:P107_has_current_or_former_member <${subjectURL}> .} - - UNION{ <${subjectURL}> crmpe:PP2i_provides ?service .} - UNION{ ?service crmpe:PP2_provided_by <${subjectURL}> .} - - UNION{ <${subjectURL}> crmpe:PP44i_is_maintaining_team_of ?maintained .} - UNION{ ?maintained crmpe:PP44_has_maintaining_team <${subjectURL}> .} -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql deleted file mode 100644 index 98dd6ae..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql +++ /dev/null @@ -1,73 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - - <${subjectURL}> crm:P102_has_title ?title . - <${subjectURL}> crm:P3_has_note ?description . - <${subjectURL}> a ?type . - <${subjectURL}> rdfs:label ?label . - - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - - <${subjectURL}> crm:P2_has_type ?e55TypeLabel . - - <${subjectURL}> crm:P46_is_composed_of ?Col_Part . - - <${subjectURL}> crmpe:PP4i_is_object_hosted_by ?host_serv . - <${subjectURL}> crmpe:PP32i_is_curated_by ?curator . - <${subjectURL}> ?creator . - - <${subjectURL}> crm:P72_has_language ?languageLabel . -} -WHERE { - <${subjectURL}> a crm:E78_Collection . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} -UNION{ -<${subjectURL}> crm:P129_is_about ?about . -?about a ?aboutType . -?about rdfs:label ?aboutLabel . -} -UNION { -<${subjectURL}> crm:P2_has_type ?e55Type . -?e55Type rdfs:label ?e55TypeLabel . -} -UNION{ <${subjectURL}> crm:P46_is_composed_of ?Col_Part .} -UNION{ ?Col_Part crm:P46i_forms_part_of <${subjectURL}> .} - -UNION{ <${subjectURL}> crmpe:PP4i_is_object_hosted_by ?host_serv .} -UNION{ <${subjectURL}> crmpe:PP32i_is_curated_by ?curator .} -UNION{ ?host_serv crmpe:PP4_hosts_object <${subjectURL}> .} -UNION{ ?curator crmpe:PP32_curates <${subjectURL}> .} - -UNION { - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P14_carried_out_by ?creator . -} -UNION { - <${subjectURL}> crm:P72_has_language ?language . - ?language rdfs:label ?languageLabel . -} - -} LIMIT 30000 \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql deleted file mode 100644 index 1012e27..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql +++ /dev/null @@ -1,138 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - - <${subjectURL}> crm:P102_has_title ?title . - <${subjectURL}> crm:P3_has_note ?description . - <${subjectURL}> a ?type . - <${subjectURL}> rdfs:label ?label . - - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - - <${subjectURL}> crm:P2_has_type ?e55TypeLabel . - - <${subjectURL}> crmpe:PP8i_is_dataset_hosted_by ?host . - <${subjectURL}> crmpe:PP32i_is_curated_by ?curator . - - <${subjectURL}> crm:P106i_forms_part_of ?container . - <${subjectURL}> crm:P106_is_composed_of ?d3 . - - <${subjectURL}> crmpe:PP39_is_metadata_for ?anotherdataset . - <${subjectURL}> crmpe:PP39i_has_metadata ?metadata . - - <${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshotted . - <${subjectURL}> crmpe:PP17_has_snapshot ?snapshot . - - <${subjectURL}> ?encoding_label . - <${subjectURL}> ?sw . - <${subjectURL}> ?creator . - <${subjectURL}> ?creationtime . - <${subjectURL}> ?updatetime . - <${subjectURL}> ?custodytime . - <${subjectURL}> ?custodialhistoryNote . - <${subjectURL}> ?curationtime . - <${subjectURL}> crm:P72_has_language ?languageLabel . -} -WHERE { - <${subjectURL}> a crmpe:PE18_Dataset . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} - UNION{ - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - } - UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . - } - UNION{ <${subjectURL}> crmpe:PP8i_is_dataset_hosted_by ?host . } - UNION{ ?host crmpe:PP8_hosts_dataset <${subjectURL}> . } - UNION{ <${subjectURL}> crmpe:PP32i_is_curated_by ?curator .} - UNION{ ?curator crmpe:PP32_curates <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP32i_is_curated_by ?curator .} - UNION{ ?curator crmpe:PP32_curates <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP31_uses_curation_plan ?curationplan .} - UNION{ <${subjectURL}> crm:P106i_forms_part_of ?container .} - UNION{ ?container crm:P106_is_composed_of <${subjectURL}> .} - UNION{ <${subjectURL}> crm:P106_is_composed_of ?d3 .} - UNION{ ?d3 crm:P106i_forms_part_of <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP39_is_metadata_for ?anotherdataset .} - UNION{ ?anotherdataset crmpe:PP39i_has_metadata <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP39i_has_metadata ?metadata .} - UNION{ ?metadata crmpe:PP39_is_metadata_for <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP17_has_snapshot ?snapshot .} - UNION{ ?snapshot crmpe:PP17i_is_snapshot_of <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshotted .} - UNION{ ?snapshotted crmpe:PP17_has_snapshot <${subjectURL}> .} - UNION{ - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P33_used_specific_technique ?technique . - ?technique crm:P2_has_type ?techniquetype . - ?techniquetype rdfs:label ?encoding_label . - } -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P2_has_type . - ?Creation_event crmdig:L23_used_software_or_firmware ?sw . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P2_has_type . - ?Creation_event crm:P14_carried_out_by ?creator . - } -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?event1 . - ?event1 crm:P2_has_type . - ?event1 crm:P4_has_time-span ?timespan . - ?timespan crm:P81_ongoing_throughout ?creationtime . - } -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?updateEvent . - ?updateEvent crm:P2_has_type . - ?updateEvent crm:P4_has_time-span ?timespanUpdate . - ?timespanUpdate crm:P82_at_some_time_within ?updatetime . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?custodialhistory . - ?custodialhistory crm:P2_has_type . - ?custodialhistory crm:P3_has_note ?custodialhistoryNote . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?custodialhistory . - ?custodialhistory crm:P2_has_type . - ?custodialhistory crm:P4_has_time-span ?timespanCustody . - ?timespanCustody crm:P82_at_some_time_within ?custodytime . -} -UNION { - <${subjectURL}> ?hasCurationActivity ?curationActivity . - ?curationActivity a crm:E87_Curation_Activity . - ?curationActivity crm:P4_has_time-span ?curationActivitytimespan . - ?curationActivitytimespan crm:P82_at_some_time_within ?curationtime . -} -UNION { - <${subjectURL}> crm:P72_has_language ?language . - ?language rdfs:label ?languageLabel . -} -} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql deleted file mode 100644 index 873ea06..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql +++ /dev/null @@ -1,148 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . -<${subjectURL}> crm:P3_has_note ?description . -<${subjectURL}> rdfs:label ?label . -<${subjectURL}> a ?type . -<${subjectURL}> crm:P2_has_type ?atypeLabel . - -<${subjectURL}> crmpe:PP2_provided_by ?provider . -?provider crm:P1_is_identified_by ?providerTitle . -?provider rdfs:label ?providerLabel . -?provider crm:P3_has_note ?providerDescr . -?provider crm:P76_has_contact_point ?contactpoint . -?contactpoint crm:P2_has_type ?contactpointType . -?contactpoint rdfs:label ?contactpointLabel . -?contactpointType rdfs:label ?contactpointTypeLabel . - -<${subjectURL}> crmpe:PP4_hosts_object ?object . - -<${subjectURL}> crmpe:PP28_has_designated_access_point ?accesspoint . -<${subjectURL}> crmpe:PP1i_is_currently_offered_by ?offerer . -<${subjectURL}> crmpe:PP45_has_competency ?competenceLabel . - -<${subjectURL}> crmpe:PP31_uses_curation_plan ?curationplan . - -<${subjectURL}> crmpe:PP29_uses_access_protocol ?protocol . -?protocol rdfs:label ?protocolLabel . - -<${subjectURL}> crmpe:PP32_curates ?object2 . - -<${subjectURL}> crmpe:PP15_delivers_on_request ?delivered . -<${subjectURL}> crmpe:PP14_runs_on_request ?ran . - -<${subjectURL}> crmpe:PP42_has_declarative_time ?declTime . - -<${subjectURL}> ?availabilityLabel . -<${subjectURL}> ?activitytypeLabel . - - <${subjectURL}> crm:P16_used_specific_object ?Conditions_of_Use . - ?Conditions_of_Use a crm:E30_Right . - ?Conditions_of_Use crm:P3_has_note ?Conditions_of_Use_Note . - ?Conditions_of_Use crm:P2_has_type ?Conditions_of_Use_Type . - ?Conditions_of_Use_Type rdfs:label ?Conditions_of_Use_Type_label . - -} -WHERE { - <${subjectURL}> a crmpe:PE1_Service . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . - FILTER NOT EXISTS { ?titleRes a crm:E51_Contact_Point } -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} - UNION{ - <${subjectURL}> crm:P2_has_type ?atype . - ?atype rdfs:label ?atypeLabel . -} -UNION{ - <${subjectURL}> crm:P2_has_type ?availability . - ?availability rdfs:label ?availabilityLabel . - FILTER(STRSTARTS(STR(?availability), "http://ariadneplus.d4science.org/handle/AriadnePlus/REG/Concept/availabilitytype")) -} -UNION{ - <${subjectURL}> crm:P2_has_type ?activitytype . - ?activitytype rdfs:label ?activitytypeLabel . - FILTER(STRSTARTS(STR(?activitytype), "http://ariadneplus.d4science.org/handle/AriadnePlus/REG/Concept/activitytype")) -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P1_is_identified_by ?providerTitleRes . - ?providerTitleRes rdfs:label ?providerTitle . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P3_has_note ?providerDescr . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider rdfs:label ?providerLabel . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P76_has_contact_point ?contactpoint . -?contactpoint rdfs:label ?contactpointLabel . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpointType rdfs:label ?contactpointTypeLabel .} -UNION { <${subjectURL}> crmpe:PP28_has_designated_access_point ?accesspoint .} -UNION { ?accesspoint crmpe:PP28i_is_designated_access_point_of <${subjectURL}> .} -UNION { <${subjectURL}> crmpe:PP1i_is_currently_offered_by ?offerer .} -UNION { ?offerer crmpe:PP1_currently_offers <${subjectURL}> .} -UNION { - <${subjectURL}> crmpe:PP45_has_competency ?competence . - ?competence rdfs:label ?competenceLabel .} -UNION { <${subjectURL}> crmpe:PP4_hosts_object ?object .} -UNION { ?object crmpe:PP4i_is_object_hosted_by <${subjectURL}> .} - -UNION {<${subjectURL}> crmpe:PP31_uses_curation_plan ?curationplan .} -UNION { ?curationplan crmpe:PP31i_is_curation_plan_used_by <${subjectURL}> .} -UNION { - <${subjectURL}> crmpe:PP29_uses_access_protocol ?protocol . - ?protocol rdfs:label ?protocolLabel .} -UNION { - ?protocol crmpe:PP29i_is_access_protocol_used_by <${subjectURL}> . - ?protocol rdfs:label ?protocolLabel .} -UNION { <${subjectURL}> crmpe:PP32_curates ?object2 .} -UNION { ?object2 crmpe:PP32i_is_curated_by <${subjectURL}> .} -UNION { <${subjectURL}> crmpe:PP15_delivers_on_request ?delivered .} -UNION { ?delivered crmpe:PP15i_is_delivered_by <${subjectURL}>.} -UNION { <${subjectURL}> crmpe:PP14_runs_on_request ?ran .} -UNION { ?ran crmpe:PP14i_is_run_by <${subjectURL}>.} - UNION { - <${subjectURL}> crmpe:PP42_has_declarative_time ?declTime .} -UNION { - <${subjectURL}> crm:P16_used_specific_object ?Conditions_of_Use . - ?Conditions_of_Use a crm:E30_Right . - ?Conditions_of_Use crm:P3_has_note ?Conditions_of_Use_Note . -} -UNION { - <${subjectURL}> crm:P16_used_specific_object ?Conditions_of_Use . - ?Conditions_of_Use a crm:E30_Right . - ?Conditions_of_Use crm:P2_has_type ?Conditions_of_Use_Type . - ?Conditions_of_Use_Type rdfs:label ?Conditions_of_Use_Type_label . -} - -} - diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql deleted file mode 100644 index 3695c2a..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql +++ /dev/null @@ -1,62 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> rdfs:label ?label . - -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . - - <${subjectURL}> crm:P3_has_note ?description . - - <${subjectURL}> a ?type . - - <${subjectURL}> crmpe:PP1_currently_offers ?offeredservice . - -<${subjectURL}> crmpe:PP44_has_maintaining_team ?team . - ?team crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpoint rdfs:label ?contactpointLabel . - ?contactpointType rdfs:label ?contactpointTypeLabel . - - <${subjectURL}> crm:P4_has_time-span ?time_span . - ?time_span crm:P82a_begin_of_the_begin ?time_actual . -} -WHERE { - <${subjectURL}> a crmpe:PE35_Project . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION - { - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . - } -UNION{ <${subjectURL}> crm:P3_has_note ?description .} -UNION{ <${subjectURL}> rdfs:label ?label .} - -UNION{ - <${subjectURL}> crmpe:PP1_currently_offers ?offeredservice . -} -UNION{ - <${subjectURL}> crmpe:PP44_has_maintaining_team ?team . - } - UNION{ - <${subjectURL}> crmpe:PP44_has_maintaining_team ?team . - ?team crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpoint rdfs:label ?contactpointLabel . - ?contactpointType rdfs:label ?contactpointTypeLabel . - } -UNION { - <${subjectURL}> crm:P4_has_time-span ?time_span . - ?time_span crm:P82a_begin_of_the_begin ?time_actual . - } - -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt deleted file mode 100644 index e505fd8..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt +++ /dev/null @@ -1,76 +0,0 @@ -To check available rules: SELECT * from DB.DBA.SYS_RDF_SCHEMA; - -#### ARIADNEPLUS RULES #### - -### ORIGINAL WITH OFFICIALLY PUBLISHED SCHEMAS - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.cidoc-crm.org/cidoc-crm/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://ariadneplus.d4science.org/CRMext/CRMpe.rdfs/') ; - - -### USING SCHEMAS AVAILABLE FROM THE WORKSPACE - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.cidoc-crm.org/cidoc-crm/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://ariadneplus.d4science.org/CRMext/CRMpe.rdfs/') ; - -#### EXTENDED RULES #### - -# CRMext4SKOSandLabel_v1.4___23-03-2016180729___3031___20-09-2016163145___12893.rdfs -sparql load into ; -# CRMpc_v1.1.1.rdfs -sparql load into ; - -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/') ; -rdfs_rule_set ('extended_rules', 'http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/') ; -rdfs_rule_set ('extended_rules', 'http://ariadneplus.d4science.org/CRMext/CRMpe.rdfs/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'https://vocabs.dariah.eu/bbt/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/CRMarchaeo/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/CRMgeo/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/CRMsci/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://iflastandards.info/ns/fr/frbr/frbroo/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.w3.org/2004/02/skos/core#') ; - - - - - - - - - diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt b/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt index 268fafe..0e669bd 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt @@ -45,21 +45,29 @@ CONSTRUCT { ?temporal aoprop:periodName ?temporalPeriodName . ?temporal aoprop:from ?temporalFrom . ?temporal aoprop:until ?temporalUntil . +?record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . ?record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . ?record aoprop:resourceType ?resourceType . ?record aoprop:nativeSubject ?nativeSubject . ?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . ?nativeSubject aoprop:rdfAbout ?nativeSubject . -?nativeSubject aoprop:aatSubjectsId ?aatSubjectsId . +?record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +?record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . ?record aoprop:title ?title . ?record aoprop:description ?description . ?record aoprop:language ?language . } from -from -from -from +from +from +from +from where { ?record aocat:has_language / skos:prefLabel ?language . ?record aocat:has_original_id ?originalId . @@ -68,19 +76,15 @@ where { ?creator aocat:has_name ?creatorName . ?record aocat:has_title ?title . ?record aocat:has_type / skos:prefLabel ?resourceType . + ?record aocat:has_native_subject ?nativeSubject . + ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . + optional { + ?record aocat:has_derived_subject ?derivedSubject . + ?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel . + } optional { ?creator aocat:has_email ?creatorEmail . } - optional { - ?record aocat:has_native_subject ?nativeSubject . - ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . - optional { - ?nativeSubject skos:exactMatch ?aatSubjectsId . - } - optional { - ?nativeSubject skos:closeMatch ?aatSubjectsId . - } - } optional { ?record aocat:has_description ?description . } @@ -90,22 +94,19 @@ where { optional { ?record aocat:has_landing_page / rdfs:label ?landingPage . } - optional { - ?record aocat:has_temporal_coverage ?temporal . - ?temporal aocat:has_period / rdfs:label ?temporalPeriodName . + optional { + ?record aocat:has_temporal_coverage ?temporalNative . + ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . } optional { ?record aocat:has_temporal_coverage ?temporal . - ?temporal aocat:has_native_period ?nativePeriod . - ?nativePeriod skos:prefLabel ?temporalPeriodName . + ?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName . optional { - ?nativePeriod skos:exactMatch ?periodORef . - ?periodORef time:intervalStartedBy ?intervalStartedBy . - ?intervalStartedBy skos:prefLabel ?temporalFrom . - ?periodORef time:intervalFinishedBy ?intervalFinishedBy . - ?intervalFinishedBy skos:prefLabel ?temporalUntil . - } + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } } + { select * where { From af2eb7bb91db3ee068930f2776ab1ffc70aff1ae Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 9 Jun 2020 11:44:48 +0200 Subject: [PATCH 2/8] added sparql query folder --- .../eu/dnetlib/ariadneplus/{ => sparql}/gen_aatplus_graph.sparql | 0 .../dnetlib/ariadneplus/{ => sparql}/gen_periodoplus_graph.sparql | 0 .../eu/dnetlib/ariadneplus/{ => sparql}/read_record_data.sparql | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/{ => sparql}/gen_aatplus_graph.sparql (100%) rename dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/{ => sparql}/gen_periodoplus_graph.sparql (100%) rename dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/{ => sparql}/read_record_data.sparql (100%) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_aatplus_graph.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_aatplus_graph.sparql similarity index 100% rename from dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_aatplus_graph.sparql rename to dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_aatplus_graph.sparql diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_periodoplus_graph.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_periodoplus_graph.sparql similarity index 100% rename from dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/gen_periodoplus_graph.sparql rename to dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_periodoplus_graph.sparql diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/read_record_data.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql similarity index 100% rename from dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/read_record_data.sparql rename to dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql From fa1d268b3627394eb2c27e512cf53f7fe10d80c7 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Wed, 10 Jun 2020 11:53:37 +0200 Subject: [PATCH 3/8] added query to link data from collection level to records level --- .../sparql/collection_to_records_data.sparql | 44 +++++++++++++++++++ .../sparql/read_record_data.sparql | 26 ++++++----- 2 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql new file mode 100644 index 0000000..40d936f --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql @@ -0,0 +1,44 @@ +PREFIX aocat: +PREFIX skos: +PREFIX time: +PREFIX ariadneplus: +INSERT { + GRAPH { + ?record aocat:was_issued ?issued . + ?record aocat:was_modified ?modified . + ?record aocat:has_contributor ?contributor . + ?record aocat:has_responsible ?legalResponsible . + ?record aocat:has_owner ?owner . + ?record aocat:has_publisher ?publisher . + ?record aocat:has_access_rights ?accessRights . + ?record aocat:has_ARIADNE_subject ?archeologicalResourceType . + } +} +WHERE { + ?record aocat:is_part_of ?collection . + ?collection aocat:was_issued ?issued . + ?collection aocat:was_modified ?modified . + ?collection aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + ?collection aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + ?collection aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + ?collection aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + ?collection aocat:has_access_rights ?accessRights . + ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql index 2e94223..84b4730 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql @@ -58,7 +58,8 @@ CONSTRUCT { ?derivedSubject aoprop:source "Getty AAT" . ?record aoprop:aatSubjects ?derivedSubject . ?derivedSubject aoprop:id ?derivedSubject . -?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:lang ?aatSubjectsLang . ?record aoprop:title ?title . ?record aoprop:description ?description . ?record aoprop:language ?language . @@ -68,7 +69,7 @@ from from from from -where { +where { ?record aocat:has_language / skos:prefLabel ?language . ?record aocat:has_original_id ?originalId . ?record aocat:is_part_of ?partOf . @@ -111,18 +112,18 @@ where { select * where { ?record aocat:is_part_of ?collection . - ?collection aocat:was_issued ?issued . - ?collection aocat:was_modified ?modified . - ?collection aocat:has_contributor ?contributor . + ?record aocat:was_issued ?issued . + ?record aocat:was_modified ?modified . + ?record aocat:has_contributor ?contributor . ?contributor aocat:has_name ?contributorName . - ?collection aocat:has_responsible ?legalResponsible . + ?record aocat:has_responsible ?legalResponsible . ?legalResponsible aocat:has_name ?legalResponsibleName . - ?collection aocat:has_owner ?owner . + ?record aocat:has_owner ?owner . ?owner aocat:has_name ?ownerName . - ?collection aocat:has_publisher ?publisher . + ?record aocat:has_publisher ?publisher . ?publisher aocat:has_name ?publisherName . - ?collection aocat:has_access_rights ?accessRights . - ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?record aocat:has_access_rights ?accessRights . + ?record aocat:has_ARIADNE_subject ?archeologicalResourceType . ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . optional { ?contributor aocat:has_email ?contributorEmail . @@ -138,6 +139,7 @@ where { } } } + { select * where { @@ -160,5 +162,5 @@ where { } } } - -} + BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang) +} \ No newline at end of file From b80274fff3bd74fe7786d08a54d70ee68f5e82aa Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Wed, 10 Jun 2020 19:39:53 +0200 Subject: [PATCH 4/8] added graphDB reader and ES indexer --- .../ariadneplus/elasticsearch/BulkUpload.java | 27 +- .../elasticsearch/model/AgentInfo.java | 8 +- .../model/ArcheologicalResourceType.java | 4 +- .../model/AriadneCatalogEntry.java | 63 ++-- .../elasticsearch/model/AriadneGeoPoint.java | 4 +- .../elasticsearch/model/AriadneTemporal.java | 8 +- .../elasticsearch/model/NativeSubject.java | 4 +- .../elasticsearch/model/Spatial.java | 54 +-- .../ariadneplus/reader/ResourceManager.java | 33 +- .../reader/RunSPARQLQueryService.java | 304 ++++++++++++++--- .../ariadneplus/reader/json/ParseRDFJSON.java | 11 +- .../ariadneplus/reader/utils/ESUtils.java | 14 + .../reader/utils/PropertiesMap.java | 22 +- .../src/main/resources/application.properties | 277 +++++++-------- .../GraphDbReaderAndESIndexTest.java | 55 +++ .../ariadneplus/GraphDbReaderTest.java | 40 --- .../test/resources/application.properties | 319 ++++++++++++++++++ 17 files changed, 890 insertions(+), 357 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java create mode 100644 dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java create mode 100644 dnet-ariadneplus-graphdb-publisher/test/resources/application.properties diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 2d5c6d9..c18a5bd 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -5,6 +5,9 @@ import java.lang.reflect.InvocationTargetException; import javax.annotation.PostConstruct; +import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo; +import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; +import eu.dnetlib.ariadneplus.reader.ResourceManager; import org.apache.http.HttpHost; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -16,9 +19,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; -import eu.dnetlib.ariadneplus.reader.ResourceManager; - @Service public class BulkUpload { @@ -28,22 +28,33 @@ public class BulkUpload { private RestHighLevelClient client; @PostConstruct - private void init() throws IOException { + public void init() throws IOException { client = new RestHighLevelClient( RestClient.builder( - new HttpHost("localhost",9200,"http"))); + new HttpHost("elastic-test.ariadne.d4science.org",9200,"http"))); } public void index(ResourceManager manager) { BulkRequest request = new BulkRequest(); while (manager.hasNext()){ + try { - AriadneCatalogEntry ace = ((AriadneCatalogEntry) manager.next()); - request.add(new IndexRequest("prova_via_code").id(ace.getOriginalId()) + Object next = manager.next(); + AriadneCatalogEntry ace = ((AriadneCatalogEntry) next); + AgentInfo testPublisher = new AgentInfo(); + testPublisher.setName("TEST"); + ace.getPublisher().add(testPublisher); + String[] splits = ace.getIdentifier().split("/"); + request.add(new IndexRequest("catalog_test").id(splits[splits.length-1]) .source(ace.toJson(),XContentType.JSON)); - System.out.println("indexing to ES record "+ace.getOriginalId()); + System.out.println("indexing to ES record "+ace.toJson()); BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + System.out.println("indexing to ES record completed "+bulkResponse.status()); + if (bulkResponse.hasFailures()) { + System.out.println("FailureMessage: "+bulkResponse.buildFailureMessage()); + } + } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (NoSuchMethodException e) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java index 9a885a1..b05a8bb 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java @@ -55,16 +55,16 @@ public class AgentInfo { AgentInfo pi = new AgentInfo(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/name" : + case "https://www.ariadne-infrastructure.eu/property/name" : pi.setName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/type": + case "https://www.ariadne-infrastructure.eu/property/type": pi.setType(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/email": + case "https://www.ariadne-infrastructure.eu/property/email": pi.setEmail(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/phone": + case "https://www.ariadne-infrastructure.eu/property/phone": pi.setPhone(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java index 80ea18c..62fa44d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java @@ -16,10 +16,10 @@ public class ArcheologicalResourceType { ArcheologicalResourceType art = new ArcheologicalResourceType(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/id" : + case "https://www.ariadne-infrastructure.eu/property/id" : art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/name": + case "https://www.ariadne-infrastructure.eu/property/name": String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); art.setName(tmp); break; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java index 938868f..8d754d7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -1,8 +1,9 @@ package eu.dnetlib.ariadneplus.elasticsearch.model; import com.google.gson.Gson; -import java.util.List; +import eu.dnetlib.ariadneplus.reader.utils.ESUtils; +import java.util.List; public class AriadneCatalogEntry { private List aatSubjects; @@ -21,7 +22,7 @@ public class AriadneCatalogEntry { private List hasItemMetadataStructure; private List hasMetadataRecord; private String identifier; - private String partOf; + private String isPartOf; private String issued; private List keyword; private String landingPage; @@ -40,6 +41,8 @@ public class AriadneCatalogEntry { private String rights; private List scientificResponsible; private List spatial; +// private List spatialRegion; +// private List spatialRegionPoint; private List technicalResponsible; private List temporal; @@ -173,12 +176,12 @@ public class AriadneCatalogEntry { this.identifier = identifier; } - public String getPartOf() { - return partOf; + public String getIsPartOf() { + return isPartOf; } - public void setPartOf(String partOf) { - this.partOf = partOf; + public void setIsPartOf(String isPartOf) { + this.isPartOf = isPartOf; } public String getIssued() { @@ -186,7 +189,7 @@ public class AriadneCatalogEntry { } public void setIssued(String issued) { - this.issued = issued; + this.issued = ESUtils.getESFormatDate(issued); } public List getKeyword() { @@ -226,7 +229,7 @@ public class AriadneCatalogEntry { } public void setModified(String modified) { - this.modified = modified; + this.modified = ESUtils.getESFormatDate(modified); } public List getNativeSubject() { @@ -322,7 +325,12 @@ public class AriadneCatalogEntry { } public void setSpatial(List spatial) { - this.spatial = spatial; + if (this.spatial==null) { + this.spatial = spatial; + } + else { + this.spatial.addAll(spatial); + } } public List getTechnicalResponsible() { @@ -357,13 +365,13 @@ public class AriadneCatalogEntry { // JsonObject content = json.getAsJsonObject(); // for (Map.Entry stringJsonElementEntry : content.entrySet()) { // switch (stringJsonElementEntry.getKey()){ -// case "http://www.myprefix/accessPolicy": +// case "https://www.ariadne-infrastructure.eu/property/accessPolicy": // acim.setAccessPolicy(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/accessRights": +// case "https://www.ariadne-infrastructure.eu/property/accessRights": // acim.setAccessRights(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/contributor": +// case "https://www.ariadne-infrastructure.eu/property/contributor": // JsonArray contributor_array = stringJsonElementEntry.getValue().getAsJsonArray(); // List contributor_list = new ArrayList(); // for (int i = 0; i < contributor_array.size() ; i++ ){ @@ -373,25 +381,25 @@ public class AriadneCatalogEntry { // } // acim.setContributor(contributor_list); // break; -// case "http://www.myprefix/description": +// case "https://www.ariadne-infrastructure.eu/property/description": // acim.setDescription(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/isPartOf": +// case "https://www.ariadne-infrastructure.eu/property/isPartOf": // acim.setPartOf(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/issued": +// case "https://www.ariadne-infrastructure.eu/property/issued": // acim.setIssued(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/landingPage": +// case "https://www.ariadne-infrastructure.eu/property/landingPage": // acim.setLandingPage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/language": +// case "https://www.ariadne-infrastructure.eu/property/language": // acim.setLanguage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/modified": +// case "https://www.ariadne-infrastructure.eu/property/modified": // acim.setModified(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/nativeSubject": +// case "https://www.ariadne-infrastructure.eu/property/nativeSubject": // JsonArray nativeSubject_array = stringJsonElementEntry.getValue().getAsJsonArray(); // List nativeSubject_list = new ArrayList(); // for (int i = 0; i < nativeSubject_array.size() ; i++ ){ @@ -401,13 +409,13 @@ public class AriadneCatalogEntry { // } // acim.setNativeSubject(nativeSubject_list); // break; -// case "http://www.myprefix/originalId": +// case "https://www.ariadne-infrastructure.eu/property/originalId": // acim.setOriginalId(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/resourceType": +// case "https://www.ariadne-infrastructure.eu/property/resourceType": // acim.setResourceType(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/spatial": +// case "https://www.ariadne-infrastructure.eu/property/spatial": // JsonArray spatial_array = stringJsonElementEntry.getValue().getAsJsonArray(); // List spatial_list = new ArrayList(); // for (int i = 0; i < spatial_array.size() ; i++ ){ @@ -417,7 +425,7 @@ public class AriadneCatalogEntry { // } // acim.setSpatial(spatial_list); // break; -// case "http://www.myprefix/temporal": +// case "https://www.ariadne-infrastructure.eu/property/temporal": // JsonArray temporal_array = stringJsonElementEntry.getValue().getAsJsonArray(); // List temporal_list = new ArrayList<>(); // for(int i=0; i < temporal_array.size(); i++){ @@ -426,10 +434,10 @@ public class AriadneCatalogEntry { // } // acim.setTemporal(temporal_list); // break; -// case "http://www.myprefix/title": +// case "https://www.ariadne-infrastructure.eu/property/title": // acim.setTitle(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); // break; -// case "http://www.myprefix/publisher": +// case "https://www.ariadne-infrastructure.eu/property/publisher": // JsonArray publisher_array = stringJsonElementEntry.getValue().getAsJsonArray(); // List publisher_list = new ArrayList(); // for (int i = 0; i < publisher_array.size() ; i++ ){ @@ -439,7 +447,7 @@ public class AriadneCatalogEntry { // } // acim.setPublisher(publisher_list); // break; -// case "http://www.myprefix/archeologicalResourceType": +// case "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": // acim.setArcheologicalResourceType(ArcheologicalResourceType.fromRDFJson(map.get(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()))); // } // @@ -449,5 +457,4 @@ public class AriadneCatalogEntry { // System.out.println(acim.toJson()); // return acim; // } -} -//https://ariadne-infrastructure.eu/aocat \ No newline at end of file +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java index 2b6b295..c8ae99a 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java @@ -36,10 +36,10 @@ public class AriadneGeoPoint { AriadneGeoPoint agp = new AriadneGeoPoint(); for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { switch (stringJsonElementEntry.getKey()){ - case "http://www.myprefix/lat": + case "https://www.ariadne-infrastructure.eu/property/lat": agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/lon": + case "https://www.ariadne-infrastructure.eu/property/lon": agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java index 115b4fa..7dd30cc 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java @@ -15,16 +15,16 @@ public class AriadneTemporal { AriadneTemporal at = new AriadneTemporal(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/from" : + case "https://www.ariadne-infrastructure.eu/property/from" : at.setFrom(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/periodName": + case "https://www.ariadne-infrastructure.eu/property/periodName": at.setPeriodName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/until": + case "https://www.ariadne-infrastructure.eu/property/until": at.setUntil(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/uri": + case "https://www.ariadne-infrastructure.eu/property/uri": at.setUri(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java index 324eca8..a211d37 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java @@ -38,10 +38,10 @@ public class NativeSubject { NativeSubject pi = new NativeSubject(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/prefLabel" : + case "https://www.ariadne-infrastructure.eu/property/prefLabel" : pi.setPrefLabel(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/rdfAbout": + case "https://www.ariadne-infrastructure.eu/property/rdfAbout": pi.setRdfAbout(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index 0587975..d1d02b7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -16,44 +16,8 @@ public class Spatial { private String country; private AriadneGeoPoint location; private String placeName; - - public static Spatial fromRDFJson(JsonElement json, Map map) { - Spatial pi = new Spatial(); - for (Map.Entry entry : json.getAsJsonObject().entrySet()){ - switch (entry.getKey()){ - case "http://www.myprefix/address" : - pi.setAddress(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMaxLat": - pi.setBoundingBoxMaxLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMaxLon": - pi.setBoundingBoxMaxLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMinLat": - pi.setBoundingBoxMinLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMinLon": - pi.setBoundingBoxMinLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/country": - pi.setCountry(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/coordinateSystem": - pi.setCoordinateSystem(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/location": - String map_key = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); - pi.setLocation(AriadneGeoPoint.fromRDFJson(map.get(map_key))); - break; - case "http://www.myprefix/placeName": - pi.setPlaceName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - } - } - - return pi; - } + private String lat; + private String lon; public String getAddress() { return address; @@ -127,6 +91,20 @@ public class Spatial { this.placeName = placeName; } + public void setLat(String lat) { + if (this.getLocation()==null) { + this.setLocation(new AriadneGeoPoint()); + } + this.getLocation().setLat(lat); + } + + public void setLon(String lon) { + if (this.getLocation()==null) { + this.setLocation(new AriadneGeoPoint()); + } + this.getLocation().setLon(lon); + } + public Spatial() { } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java index 732fe78..6eaabc1 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -10,16 +10,16 @@ import java.util.Set; import javax.annotation.PostConstruct; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import eu.dnetlib.ariadneplus.reader.utils.ClassSpec; +import eu.dnetlib.ariadneplus.reader.utils.Mappings; +import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; -import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; -import eu.dnetlib.ariadneplus.reader.utils.ClassSpec; -import eu.dnetlib.ariadneplus.reader.utils.Mappings; -import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; import net.minidev.json.JSONArray; import net.minidev.json.JSONObject; @@ -32,11 +32,23 @@ public class ResourceManager { private String general_classpath; @Value("${exclude.predicates:[]}") private String exclude_predicates; + @Value("${class.map.specifications:undefined}") + private String spec; private List not_parsable; private ParseRDFJSON parser; + private PropertiesMap propertiesMap; + + public void setup(String type_path, String general_classpath, String exclude_predicates, String spec) { + this.type_path = type_path; + this.general_classpath = general_classpath; + this.exclude_predicates = exclude_predicates; + propertiesMap = new PropertiesMap(); + propertiesMap.fill(spec); + } + @PostConstruct public void init(){ Type listType = new TypeToken>(){}.getType(); @@ -69,11 +81,20 @@ public class ResourceManager { if(entry instanceof LinkedHashMap){ LinkedHashMap tmp = (LinkedHashMap)((JSONArray)((LinkedHashMap)entry).get(type_path)).get(0); class_name = (String)tmp.get("value"); + if (class_name.equals("provided record")) { + class_name = "AriadneCatalogEntry"; + } } } + if (entry == null) { + System.out.println("entry NULL " ); + return null; + + } + System.out.println("class_name: " + class_name); Class c = Class.forName(general_classpath + class_name); Object class_instance = c.newInstance(); - ClassSpec class_spec = PropertiesMap.get(class_name); + ClassSpec class_spec = propertiesMap.get(class_name); Set keySet; if(entry instanceof LinkedHashMap) @@ -101,7 +122,7 @@ public class ResourceManager { setField.invoke(class_instance, getFieldValue(values.get(0))); } else{ - if(PropertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){ + if(propertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){ List value_list = new ArrayList<>(); for(Object value: values){ value_list.add(manage(ParseRDFJSON.get(getFieldValue(value)), map.getExternal_reference())); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 749ec67..7746b36 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -1,13 +1,10 @@ package eu.dnetlib.ariadneplus.reader; -import java.io.StringWriter; - -import org.eclipse.rdf4j.model.IRI; -import org.eclipse.rdf4j.model.Literal; +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Resource; -import org.eclipse.rdf4j.model.ValueFactory; -import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.query.GraphQuery; import org.eclipse.rdf4j.query.GraphQueryResult; import org.eclipse.rdf4j.query.QueryLanguage; @@ -21,14 +18,14 @@ import org.eclipse.rdf4j.rio.Rio; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; -import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import java.io.StringWriter; +import java.util.Calendar; @Service public class RunSPARQLQueryService { - @Value("${sparql.query3:undefined}") - private String query3; + @Value("${sparql.query:undefined}") + private String query; @Value("${repository.url:undefined}") private String repository_url; @@ -41,12 +38,23 @@ public class RunSPARQLQueryService { private ResourceManager resourceManager; private BulkUpload bulkUpload; - private void openConnection(){ - manager = new RemoteRepositoryManager(repository_url); + private static String username = null; + private static String pwd = null; + private static String graphDBUrl = null; + private static String graphDBRepository = null; + + public void setupConnection(String username, String pwd, String graphDbUrl, String graphDbRepository) { + setUsername(username); + setPwd(pwd); + setGraphDBUrl(graphDbUrl); + setGraphDBRepository(graphDbRepository); + } + + private void openConnection(){ + manager = new RemoteRepositoryManager(getGraphDBUrl()); manager.init(); - manager.setUsernameAndPassword("writer", "Writer01"); -// repository = manager.getRepository("test01"); - repository = manager.getRepository("ariadneplus-ts01"); + manager.setUsernameAndPassword(getUsername(), getPwd()); + repository = manager.getRepository(getGraphDBRepository()); connection = repository.getConnection(); } @@ -58,58 +66,213 @@ public class RunSPARQLQueryService { } public String executeQueryGraph(){ - if (query3.equals("undefined")) + query = "PREFIX aocat: \n" + + "PREFIX rdfs: \n" + + "PREFIX skos: \n" + + "PREFIX aoprop: \n" + + "PREFIX onto: \n" + + "PREFIX ariadneplus: \n" + + "PREFIX time: \n" + + "\n" + + "CONSTRUCT { \n" + + "%s aoprop:identifier %s . \n" + + "%s aoprop:originalId ?originalId . \n" + + "%s aoprop:issued ?issued .\n" + + "%s aoprop:modified ?modified .\n" + + "%s aoprop:partOf ?partOf .\n" + + "%s aoprop:creator ?creator . \n" + + "?creator aoprop:name ?creatorName .\n" + + "?creator aoprop:email ?creatorEmail . \n" + + "%s aoprop:contributor ?contributor . \n" + + "?contributor aoprop:name ?contributorName .\n" + + "?contributor aoprop:email ?contributorEmail .\n" + + "%s aoprop:legalResponsible ?legalResponsible . \n" + + "?legalResponsible aoprop:name ?legalResponsibleName .\n" + + "?legalResponsible aoprop:email ?legalResponsibleEmail . \n" + + "%s aoprop:owner ?owner . \n" + + "?owner aoprop:name ?ownerName .\n" + + "?owner aoprop:email ?ownerEmail . \n" + + "%s aoprop:publisher ?publisher . \n" + + "?publisher aoprop:name ?publisherName . \n" + + "?publisher aoprop:email ?publisherEmail . \n" + + "%s aoprop:accessPolicy ?accessPolicy . \n" + + "%s aoprop:accessRights ?accessRights .\n" + + "%s aoprop:landingPage ?landingPage .\n" + + "%s aoprop:spatialRegion ?spatialRegion . \n" + + "?spatialRegion aoprop:placeName ?spatialPlaceName .\n" + + "?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . \n" + + "%s aoprop:spatialRegionPoint ?spatialRegionPoint . \n" + + "?spatialRegionPoint aoprop:lat ?spatialLocationLat . \n" + + "?spatialRegionPoint aoprop:lon ?spatialLocationLon .\n" + + "%s aoprop:spatialRegionBox ?spatialRegionBox . \n" + + "?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat .\n" + + "?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon .\n" + + "?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat .\n" + + "?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon .\n" + + "%s aoprop:uri ?temporal .\n" + + "?temporal aoprop:periodName ?temporalPeriodName .\n" + + "?temporal aoprop:from ?temporalFrom .\n" + + "?temporal aoprop:until ?temporalUntil .\n" + + "%s aoprop:uri ?temporalNative .\n" + + "?temporalNative aoprop:periodName ?temporalNativePeriodName .\n" + + "%s aoprop:archeologicalResourceType ?archeologicalResourceType . \n" + + "?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . \n" + + "%s aoprop:resourceType ?resourceType . \n" + + "%s aoprop:nativeSubject ?nativeSubject . \n" + + "?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel .\n" + + "?nativeSubject aoprop:rdfAbout ?nativeSubject .\n" + + "%s aoprop:derivedSubject ?derivedSubject .\n" + + "?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel .\n" + + "?derivedSubject aoprop:source \"Getty AAT\" .\n" + + "%s aoprop:aatSubjects ?derivedSubject .\n" + + "?derivedSubject aoprop:id ?derivedSubject . \n" + + "?derivedSubject aoprop:label ?derivedSubjectPrefLabel .\n" + + "?derivedSubject aoprop:lang ?aatSubjectsLang .\n" + + "%s aoprop:title ?title . \n" + + "%s aoprop:description ?description . \n" + + "%s aoprop:language ?language . \n" + + "}\n" + + "from \n" + + "from \n" + + "from \n" + + "from \n" + + "from \n" + + "where { \n" + + " \t%s\taocat:has_language / skos:prefLabel ?language .\n" + + " \t%s\taocat:has_original_id ?originalId .\n" + + " \t%s aocat:is_part_of ?partOf .\n" + + " \t%s\taocat:has_creator ?creator .\n" + + " ?creator aocat:has_name ?creatorName .\n" + + "\t %s aocat:has_title ?title .\n" + + " \t%s aocat:has_type / skos:prefLabel ?resourceType .\t\n" + + " \t%s aocat:has_native_subject ?nativeSubject .\n" + + " ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel .\n" + + " optional {\n" + + "\t\t\t%s aocat:has_derived_subject ?derivedSubject .\n" + + " \t?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel .\n" + + " }\n" + + " optional {\n" + + "\t ?creator aocat:has_email ?creatorEmail .\n" + + " \t}\n" + + " optional {\n" + + " %s aocat:has_description ?description .\n" + + " }\n" + + " optional {\n" + + " %s aocat:has_access_policy / rdfs:label ?accessPolicy . \n" + + " }\n" + + " optional {\n" + + " \t%s aocat:has_landing_page / rdfs:label ?landingPage .\n" + + " }\n" + + " \toptional {\n" + + " \t%s aocat:has_temporal_coverage ?temporalNative .\n" + + " \t?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName .\n" + + " }\n" + + " optional {\n" + + " \t%s aocat:has_temporal_coverage ?temporal .\n" + + " \t?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName .\n" + + " \toptional {\n" + + " ?temporal aocat:from ?temporalFrom .\n" + + " ?temporal aocat:until ?temporalUntil .\n" + + " \t}\n" + + " }\n" + + " \t\n" + + " \t{ \n" + + "\t\tselect * \n" + + " where {\n" + + " %s aocat:is_part_of ?collection .\n" + + " %s aocat:was_issued ?issued .\n" + + " %s aocat:was_modified ?modified .\n" + + " %s aocat:has_contributor ?contributor .\n" + + " ?contributor aocat:has_name ?contributorName .\n" + + " %s aocat:has_responsible ?legalResponsible .\n" + + " ?legalResponsible aocat:has_name ?legalResponsibleName .\n" + + " %s aocat:has_owner ?owner .\n" + + " ?owner aocat:has_name ?ownerName .\n" + + " %s aocat:has_publisher ?publisher . \n" + + " ?publisher aocat:has_name ?publisherName .\n" + + " %s aocat:has_access_rights ?accessRights .\n" + + " %s aocat:has_ARIADNE_subject ?archeologicalResourceType . \n" + + " ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .\n" + + " optional {\n" + + " ?contributor aocat:has_email ?contributorEmail .\n" + + " }\n" + + " optional {\n" + + " ?legalResponsible aocat:has_email ?legalResponsibleEmail .\n" + + " }\n" + + " optional {\n" + + " ?owner aocat:has_email ?ownerEmail . \n" + + " }\n" + + " optional {\n" + + " ?publisher aocat:has_email ?publisherEmail .\n" + + " }\n" + + " \t}\n" + + " \t}\n" + + " \n" + + " \t{\n" + + " select * \n" + + " where {\n" + + " \t%s\taocat:has_spatial_coverage ?spatialRegion .\n" + + " ?spatialRegion aocat:has_place_name ?spatialPlaceName .\n" + + " optional {\n" + + " ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem .\n" + + " }\n" + + " optional {\n" + + " %s\taocat:has_spatial_coverage ?spatialRegionPoint .\n" + + " ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; \n" + + " \t\t aocat:has_longitude ?spatialLocationLon . \n" + + " }\n" + + " optional {\n" + + " %s\taocat:has_spatial_coverage ?spatialRegionBox .\n" + + " ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; \n" + + " \t\t aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ;\n" + + " \t\t aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ;\n" + + " \t aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ;\n" + + " }\n" + + " }\t\n" + + " \t}\n" + + " BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang)\n" + + "}"; + + if (query.equals("undefined")) return null; - openConnection(); + query = query.replaceAll("%s", ""); + openConnection(); StringWriter recordWriter = null; - Model model = null; + Model resultsModel = null; String jsonRecord = null; try { - GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query3); + System.out.println("Start connection Time: "+Calendar.getInstance().getTime().toString()); + GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query); GraphQueryResult graphQueryResult = graphQuery.evaluate(); - System.out.println("as model ..."); - model = QueryResults.asModel(graphQueryResult); + resultsModel = QueryResults.asModel(graphQueryResult); graphQueryResult.close(); - int resourceCount = 0; - System.out.println("counting resources ... " ); - ValueFactory factory = SimpleValueFactory.getInstance(); - IRI iri = factory.createIRI("http://www.myprefix/resourcetype"); - Literal value = factory.createLiteral("AriadneCatalogEntry"); - for (Resource record: model.filter(null, iri, value).subjects()) { - System.out.println(); - Model recordModel = null; -// RDFWriter rdfRecordWriter = null; - resourceCount+=1; -// if (resourceCount==12) { -// break; -// } - recordModel = model.filter(record, null, null); - if (recordModel!=null && !recordModel.isEmpty()) { - recordWriter = new StringWriter(); - RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); - Rio.write(recordModel, rdfRecordWriter); -// System.out.println("record json: "+ recordWriter.toString()); - parser.parse(recordWriter.toString()); - resourceManager.manage(parser); - bulkUpload.index(resourceManager); - } - } - System.out.println(); - System.out.println("count resources: "+ resourceCount); - System.out.println("count statements: " + model.size()); - System.out.println("index to Elastic Search completed"); - }catch(Exception e){ + System.out.println("End connection Time: "+Calendar.getInstance().getTime().toString()); + System.out.println("count statements: " + resultsModel.size()); + recordWriter = new StringWriter(); + RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); + Rio.write(resultsModel, rdfRecordWriter); + System.out.println("RDF > json record: "+recordWriter.toString()); + parser.parse(recordWriter.toString()); + resourceManager.manage(parser); + bulkUpload.index(resourceManager); + } catch(Exception e){ e.printStackTrace(); - }finally{ + } finally{ closeConnection(); - if (model!=null) { - model.clear(); + if (resultsModel!=null) { + resultsModel.clear(); } } return jsonRecord; } - + private void dumpModel(Model model) { + System.out.print(" [ dump model ] "); + for (Statement stmt: model) { + System.out.println(stmt.toString()); + } + } + public ParseRDFJSON getParser() { return parser; } @@ -139,4 +302,35 @@ public class RunSPARQLQueryService { this.bulkUpload = bulkUpload; } + public static String getUsername() { + return username; + } + + public static String getPwd() { + return pwd; + } + + public static String getGraphDBUrl() { + return graphDBUrl; + } + + public static String getGraphDBRepository() { + return graphDBRepository; + } + + public static void setUsername(String username) { + RunSPARQLQueryService.username = username; + } + + public static void setPwd(String pwd) { + RunSPARQLQueryService.pwd = pwd; + } + + public static void setGraphDBUrl(String graphDBUrl) { + RunSPARQLQueryService.graphDBUrl = graphDBUrl; + } + + public static void setGraphDBRepository(String graphDBRepository) { + RunSPARQLQueryService.graphDBRepository = graphDBRepository; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java index 89ca81a..41d377d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java @@ -20,8 +20,7 @@ public class ParseRDFJSON { static JSONObject map ; - @Value("${catalog.entry.path:undefined}") - private String query; + private String catalogEntryJsonPath; private String json; @@ -45,7 +44,7 @@ public class ParseRDFJSON { setJson(json); fillMap(); DocumentContext jsonContext = JsonPath.parse(json); - JSONArray entries = jsonContext.read(query); + JSONArray entries = jsonContext.read(getCatalogEntryJsonPath()); int size = entries.size(); it = entries.iterator(); } @@ -62,5 +61,11 @@ public class ParseRDFJSON { return (JSONObject) map.get(key); } + public String getCatalogEntryJsonPath() { + return catalogEntryJsonPath; + } + public void setCatalogEntryJsonPath(String catalogEntryJsonPath) { + this.catalogEntryJsonPath = catalogEntryJsonPath; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java new file mode 100644 index 0000000..fb8b0df --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java @@ -0,0 +1,14 @@ +package eu.dnetlib.ariadneplus.reader.utils; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; + +public class ESUtils { + private static DateTimeFormatter originalRecordDateFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy"); + private static DateTimeFormatter elasticSearchDateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + public static String getESFormatDate(String originalDate) { + LocalDate parsedDate = LocalDate.parse(originalDate, originalRecordDateFormatter); + return parsedDate.format(elasticSearchDateFormatter); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java index 897f3e4..7c7372d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java @@ -1,31 +1,29 @@ package eu.dnetlib.ariadneplus.reader.utils; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; -import javax.annotation.PostConstruct; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; - @Component public class PropertiesMap { - @Value("${class.map.specifications:undefined}") - private String spec; +// @Value("${class.map.specifications:undefined}") +// private String spec; private static Map map; - public static ClassSpec get(String key){ + public ClassSpec get(String key){ return map.get(key); } @PostConstruct - public void fill(){ + public void fill(String spec){ Type mapType = new TypeToken>(){}.getType(); map = new Gson().fromJson(spec,mapType); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index e367fe5..d3dcac8 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -5,147 +5,38 @@ server.port=8281 graphdb.serverUrl=http://localhost:7200/ graphdb.writer.user=writer -graphdb.writer.pwd=writer01 -graphdb.repository=ariadneprova +graphdb.writer.pwd=**** +graphdb.repository=ariadneplus-ts01 graphdb.sparqlUrl = http://localhost:7200/sparql graphdb.baseURI=https://ariadne-infrastructure.eu/ -#repository.url=http://localhost:7200 repository.url=http://graphdb-test.ariadne.d4science.org:7200 -sparql.query2=PREFIX skos: PREFIX rdfs: PREFIX aocat: PREFIX mine: CONSTRUCT { ?archeologicalResourceType a mine:archeologicalResourceType . ?record mine:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType mine:name ?archeologicalResourceTypeName . } WHERE { GRAPH { ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . } } - -sparql.query=PREFIX aocat: \ -PREFIX rdfs: \ -PREFIX skos: \ -PREFIX mine: \ -CONSTRUCT { \ - ?record mine:resourcetype "AriadneCatalogEntry" . \ - ?record mine:identifier ?record .\ - ?record mine:publisher ?x . \ - ?x a mine:AgentInfo . \ - ?x mine:name ?publisherName . \ - ?x mine:type ?publisherType . \ - ?archeologicalResourceType a mine:archeologicalResourceType . \ -?record mine:archeologicalResourceType ?archeologicalResourceType . \ -?archeologicalResourceType mine:name ?archeologicalResourceTypeName . \ -?record mine:issued ?issued . \ -?record mine:resourceType ?resourceType . \ -?record mine:modified ?modified . \ -?record mine:nativeSubject ?ns . \ - ?ns a mine:NativeSubject . \ -?ns mine:prefLabel ?nativeSbj . \ - ?spatial a mine:Spatial . \ -?record mine:spatial ?spatial . \ -?spatial mine:placeName ?spatialPlaceName . \ -?spatial mine:location ?blocation . \ - ?blocation a mine:AriadneGeoPoint . \ -?blocation mine:lat ?spatialLocationLat . \ -?blocation mine:lon ?spatialLocationLon . \ -?record mine:accessPolicy ?accessPolicy . \ -?record mine:landingPage ?landingPage . \ -?record mine:title ?title . \ -?record mine:accessRights ?accessRights . \ -?record mine:description ?description . \ -?record mine:contributor ?contributor . \ - ?contributor a mine:AgentInfo . \ -?contributor mine:name ?contributorName . \ -?contributor mine:type ?contributorType . \ -?record mine:originalId ?originalId . \ - ?record mine:temporal ?t .\ - ?t a mine:AriadneTemporal . \ -?t mine:periodName ?temporalPeriodName . \ -?record mine:language ?language . \ -} \ -WHERE { \ -GRAPH { \ -?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . \ -?x rdfs:label ?publisherName . \ -?record aocat:has_native_subject ?ns . \ -?ns skos:prefLabel ?nativeSbj . \ -?type skos:prefLabel ?resourceType . \ -?record aocat:has_spatial_coverage ?spatial . \ -?spatial aocat:has_place_name ?spatialPlaceName . \ -?spatial aocat:has_latitude ?spatialLocationLat . \ -?spatial aocat:has_longitude ?spatialLocationLon . \ -?record aocat:has_access_policy ?accessPolicy . \ -?record aocat:has_landing_page ?z . \ -?z rdfs:label ?landingPage . \ -?record aocat:has_title ?title . \ -?record aocat:has_access_policy ?accessRights . \ -?record aocat:has_description ?description . \ -?record aocat:has_creator ?contributor . \ -?contributor rdfs:label ?contributorName . \ -?record aocat:has_original_id ?originalId . \ -?record aocat:has_temporal_coverage ?t . \ -?t aocat:has_period ?p . \ -?p skos:prefLabel ?temporalPeriodName . \ -?l skos:prefLabel ?language . \ -BIND("organization" as ?contributorType) . \ -BIND("organization" as ?publisherType) . \ -BIND(uri(concat('http://www.myprefix/location/', md5(concat(str(?spatialLocationLat), str(?spatialLocationLon))))) as ?blocation) \ -} \ -} - - -sparql.query3=PREFIX aocat: \ -PREFIX rdfs: \ -PREFIX skos: \ -PREFIX mine: \ -PREFIX onto: \ -CONSTRUCT {\ - ?record mine:resourcetype "AriadneCatalogEntry" .\ - ?record mine:identifier ?record .\ - ?record mine:publisher ?x .\ - ?x mine:type ?publisherType .\ -?record mine:issued ?issued .\ -?record mine:modified ?modified .\ -?record mine:accessPolicy ?accessPolicy .\ -?record mine:landingPage ?landingPage .\ -?record mine:title ?title .\ -?record mine:accessRights ?accessRights .\ -?record mine:description ?description .\ -?record mine:contributor ?contributor .\ -?record mine:originalId ?originalId .\ -}\ -WHERE {\ -?record aocat:has_access_policy ?accessPolicy .\ -?record aocat:has_landing_page ?z .\ -?z rdfs:label ?landingPage .\ -?record aocat:has_title ?title .\ -?record aocat:has_access_policy ?accessRights .\ -?record aocat:has_description ?description .\ -?record aocat:has_original_id ?originalId .\ -BIND("organization" as ?contributorType) .\ -BIND("organization" as ?publisherType) .\ -}\ -limit 20 - -catalog.entry.path=$[*][?(@['http://www.myprefix/resourcetype'][0]['value']=='AriadneCatalogEntry')] -general.classpath=ariadneplus.elasticsearch.model. -type.path=http://www.myprefix/resourcetype -exclude.predicates=["http://www.myprefix/resourcetype", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] +catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')] +general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. +type.path=https://www.ariadne-infrastructure.eu/property/resourceType +exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] class.map.specifications={\ "AriadneTemporal": {\ "class_type": "prototype",\ "mappings": {\ -"http://www.myprefix/from": {\ +"https://www.ariadne-infrastructure.eu/property/from": {\ "class_field": "From",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/periodName": {\ +"https://www.ariadne-infrastructure.eu/property/periodName": {\ "class_field": "PeriodName",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/until": {\ +"https://www.ariadne-infrastructure.eu/property/until": {\ "class_field": "Until",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/uri": {\ +"https://www.ariadne-infrastructure.eu/property/uri": {\ "class_field": "Uri",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -155,12 +46,12 @@ class.map.specifications={\ "AriadneGeoPoint": {\ "class_type": "unique",\ "mappings": {\ -"http://www.myprefix/lat": {\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ "class_field": "Lat",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/lon": {\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ "class_field": "Lon",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -169,27 +60,37 @@ class.map.specifications={\ },\ "Spatial": {\ "class_type": "prototype",\ -"mappings":{"http://www.myprefix/address": {\ +"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {\ "class_field": "Address",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/coordinateSystem": {\ +"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {\ "class_field": "CoordinateSystem",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/country": {\ +"https://www.ariadne-infrastructure.eu/property/country": {\ "class_field": "Country",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/location": {\ +"https://www.ariadne-infrastructure.eu/property/location": {\ "class_field": "Location",\ "external_reference": "AriadneGeoPoint",\ "substring": "no"\ },\ -"http://www.myprefix/placeName": {\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ +"class_field": "Lat",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ +"class_field": "Lon",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/placeName": {\ "class_field": "PlaceName",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -198,12 +99,12 @@ class.map.specifications={\ "ArcheologicalResourceType": {\ "class_type": "unique",\ "mappings": {\ -"http://www.myprefix/id": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ "class_field": "Id",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/name": {\ +"https://www.ariadne-infrastructure.eu/property/name": {\ "class_field": "Name",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -213,22 +114,22 @@ class.map.specifications={\ "AgentInfo": {\ "class_type": "prototype",\ "mappings": {\ -"http://www.myprefix/name": {\ +"https://www.ariadne-infrastructure.eu/property/name": {\ "class_field": "Name",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/type": {\ +"https://www.ariadne-infrastructure.eu/property/type": {\ "class_field": "Type",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/email": {\ +"https://www.ariadne-infrastructure.eu/property/email": {\ "class_field": "Email",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/phone": {\ +"https://www.ariadne-infrastructure.eu/property/phone": {\ "class_field": "Phone",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -238,108 +139,178 @@ class.map.specifications={\ "NativeSubject": {\ "class_type": "prototype",\ "mappings": {\ -"http://www.myprefix/prefLabel": {\ +"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ "class_field": "PrefLabel",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/rdfAbout": {\ +"https://www.ariadne-infrastructure.eu/property/rdfAbout": {\ "class_field": "RdfAbout",\ "substring": "yes",\ "element_type": "java.lang.String"\ }\ }\ },\ +"DerivedSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ +"class_field": "PrefLabel",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/source": {\ +"class_field": "Source",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AatSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ +"class_field": "Id",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/label": {\ +"class_field": "Label",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lang": {\ +"class_field": "Lang",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ "AriadneCatalogEntry": {\ "class_type": "unique",\ "mappings": {\ -"http://www.myprefix/accessPolicy": {\ +"https://www.ariadne-infrastructure.eu/property/accessPolicy": {\ "class_field": "AccessPolicy",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/identifier": {\ +"https://www.ariadne-infrastructure.eu/property/identifier": {\ "class_field": "Identifier",\ "substring": "yes",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/publisher": {\ +"https://www.ariadne-infrastructure.eu/property/publisher": {\ "class_field": "Publisher",\ "external_reference": "AgentInfo",\ "substring": "no"\ },\ -"http://www.myprefix/archeologicalResourceType": {\ +"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\ "class_field": "ArcheologicalResourceType",\ "external_reference": "ArcheologicalResourceType",\ "substring": "no"\ },\ -"http://www.myprefix/issued": {\ +"https://www.ariadne-infrastructure.eu/property/issued": {\ "class_field": "Issued",\ "element_type": "java.lang.String",\ "substring": "no"\ },\ -"http://www.myprefix/resourceType": {\ +"https://www.ariadne-infrastructure.eu/property/resourceType": {\ "class_field": "ResourceType",\ "element_type": "java.lang.String",\ "substring": "no"\ },\ -"http://www.myprefix/modified": {\ +"https://www.ariadne-infrastructure.eu/property/modified": {\ "class_field": "Modified",\ "element_type": "java.lang.String",\ "substring": "no"\ },\ -"http://www.myprefix/nativeSubject": {\ +"https://www.ariadne-infrastructure.eu/property/nativeSubject": {\ "class_field": "NativeSubject",\ "substring": "no",\ "external_reference": "NativeSubject"\ },\ -"http://www.myprefix/spatial": {\ +"https://www.ariadne-infrastructure.eu/property/derivedSubject": {\ +"class_field": "DerivedSubject",\ +"substring": "no",\ +"external_reference": "DerivedSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\ +"class_field": "AatSubjects",\ +"substring": "no",\ +"external_reference": "AatSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/spatialRegion": {\ "class_field": "Spatial",\ "substring": "no",\ "external_reference": "Spatial"\ },\ -"http://www.myprefix/landingPage": {\ +"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ +"https://www.ariadne-infrastructure.eu/property/landingPage": {\ "class_field": "LandingPage",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/title": {\ +"https://www.ariadne-infrastructure.eu/property/placeName": {\ +"class_field": "PlaceName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/title": {\ "class_field": "Title",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/accessRights": {\ +"https://www.ariadne-infrastructure.eu/property/accessRights": {\ "class_field": "AccessRights",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/description": {\ +"https://www.ariadne-infrastructure.eu/property/description": {\ "class_field": "Description",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/contributor": {\ +"https://www.ariadne-infrastructure.eu/property/creator": {\ +"class_field": "Creator",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/contributor": {\ "class_field": "Contributor",\ "substring": "no",\ "external_reference": "AgentInfo"\ },\ -"http://www.myprefix/originalId": {\ +"https://www.ariadne-infrastructure.eu/property/legalResponsible": {\ +"class_field": "LegalResponsible",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/owner": {\ +"class_field": "Owner",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/originalId": {\ "class_field": "OriginalId",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/temporal": {\ +"https://www.ariadne-infrastructure.eu/property/uri": {\ "class_field": "Temporal",\ "substring": "no",\ "external_reference": "AriadneTemporal"\ },\ -"http://www.myprefix/language": {\ +"https://www.ariadne-infrastructure.eu/property/language": {\ "class_field": "Language",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/partOf": {\ -"class_field": "PartOf",\ +"https://www.ariadne-infrastructure.eu/property/partOf": {\ +"class_field": "IsPartOf",\ "substring": "no",\ "element_type": "java.lang.String"\ }\ diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java new file mode 100644 index 0000000..1d17ba3 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -0,0 +1,55 @@ +/** + * + */ +package eu.dnetlib.ariadneplus; + +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import org.junit.Ignore; +import org.junit.Test; +import org.springframework.core.io.ClassPathResource; + +import java.util.Properties; + +/** + * @author enrico.ottonello + * + */ +public class GraphDbReaderAndESIndexTest { + + private RunSPARQLQueryService runSPQRLQuery; + + @Test + @Ignore + public void readAndIndexTest() throws Exception { + final ClassPathResource resource = new ClassPathResource("application.properties"); + Properties appProps = new Properties(); + appProps.load(resource.getInputStream()); + + runSPQRLQuery = new RunSPARQLQueryService(); + runSPQRLQuery.setupConnection( + appProps.getProperty("graphdb.writer.user"), + appProps.getProperty("graphdb.writer.pwd"), + appProps.getProperty("repository.url"), + appProps.getProperty("graphdb.repository")); + ParseRDFJSON parseRDFJSON = new ParseRDFJSON(); + parseRDFJSON.setCatalogEntryJsonPath(appProps.getProperty("catalog.entry.path")); + runSPQRLQuery.setParser(parseRDFJSON); + ResourceManager resourceManager = new ResourceManager(); + resourceManager.setup( + appProps.getProperty("type.path"), + appProps.getProperty("general.classpath"), + appProps.getProperty("exclude.predicates"), + appProps.getProperty("class.map.specifications") + ); + resourceManager.init(); + runSPQRLQuery.setResourceManager(resourceManager); + BulkUpload bulkUpload = new BulkUpload(); + bulkUpload.init(); + runSPQRLQuery.setBulkUpload(bulkUpload); + runSPQRLQuery.executeQueryGraph(); + + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java deleted file mode 100644 index ae7cd20..0000000 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * - */ -package eu.dnetlib.ariadneplus; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.springframework.beans.factory.annotation.Autowired; - -import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; -import eu.dnetlib.ariadneplus.reader.ResourceManager; -import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; -import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; - -/** - * @author enrico.ottonello - * - */ -@RunWith(JUnit4.class) -public class GraphDbReaderTest { - @Autowired - private RunSPARQLQueryService runSPQRLQuery; - @Autowired - private ParseRDFJSON parser; - @Autowired - private ResourceManager resourceManager; - @Autowired - private BulkUpload bulkUpload; - - @Test - public void readTest() throws Exception { - - runSPQRLQuery.setParser(parser); - runSPQRLQuery.setResourceManager(resourceManager); - runSPQRLQuery.setBulkUpload(bulkUpload); - runSPQRLQuery.executeQueryGraph(); - - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/test/resources/application.properties new file mode 100644 index 0000000..b75655c --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/test/resources/application.properties @@ -0,0 +1,319 @@ +#server.contextPath=/ariadneplus-graphdb +#update due to springboot 2.1.3 +server.servlet.context-path=/ariadneplus-graphdb +server.port=8281 + +graphdb.serverUrl=http://localhost:7200/ +graphdb.writer.user=writer +graphdb.writer.pwd=***** +graphdb.repository=ariadneplus-ts01 +graphdb.sparqlUrl = http://localhost:7200/sparql + +graphdb.baseURI=https://ariadne-infrastructure.eu/ + +repository.url=http://graphdb-test.ariadne.d4science.org:7200 + +catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')] +general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. +type.path=https://www.ariadne-infrastructure.eu/property/resourceType +exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] +class.map.specifications={\ +"AriadneTemporal": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/from": {\ +"class_field": "From",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/periodName": {\ +"class_field": "PeriodName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/until": {\ +"class_field": "Until",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/uri": {\ +"class_field": "Uri",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AriadneGeoPoint": {\ +"class_type": "unique",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ +"class_field": "Lat",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ +"class_field": "Lon",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"Spatial": {\ +"class_type": "prototype",\ +"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {\ +"class_field": "Address",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {\ +"class_field": "CoordinateSystem",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/country": {\ +"class_field": "Country",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/location": {\ +"class_field": "Location",\ +"external_reference": "AriadneGeoPoint",\ +"substring": "no"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ +"class_field": "Lat",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ +"class_field": "Lon",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/placeName": {\ +"class_field": "PlaceName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}}\ +},\ +"ArcheologicalResourceType": {\ +"class_type": "unique",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ +"class_field": "Id",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/name": {\ +"class_field": "Name",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AgentInfo": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/name": {\ +"class_field": "Name",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/type": {\ +"class_field": "Type",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/email": {\ +"class_field": "Email",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/phone": {\ +"class_field": "Phone",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"NativeSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ +"class_field": "PrefLabel",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/rdfAbout": {\ +"class_field": "RdfAbout",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"DerivedSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ +"class_field": "PrefLabel",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/source": {\ +"class_field": "Source",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AatSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ +"class_field": "Id",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/label": {\ +"class_field": "Label",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lang": {\ +"class_field": "Lang",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AriadneCatalogEntry": {\ +"class_type": "unique",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/accessPolicy": {\ +"class_field": "AccessPolicy",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/identifier": {\ +"class_field": "Identifier",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/publisher": {\ +"class_field": "Publisher",\ +"external_reference": "AgentInfo",\ +"substring": "no"\ +},\ +"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\ +"class_field": "ArcheologicalResourceType",\ +"external_reference": "ArcheologicalResourceType",\ +"substring": "no"\ +},\ +"https://www.ariadne-infrastructure.eu/property/issued": {\ +"class_field": "Issued",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ +"https://www.ariadne-infrastructure.eu/property/resourceType": {\ +"class_field": "ResourceType",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ +"https://www.ariadne-infrastructure.eu/property/modified": {\ +"class_field": "Modified",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ +"https://www.ariadne-infrastructure.eu/property/nativeSubject": {\ +"class_field": "NativeSubject",\ +"substring": "no",\ +"external_reference": "NativeSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/derivedSubject": {\ +"class_field": "DerivedSubject",\ +"substring": "no",\ +"external_reference": "DerivedSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\ +"class_field": "AatSubjects",\ +"substring": "no",\ +"external_reference": "AatSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/spatialRegion": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ +"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ +"https://www.ariadne-infrastructure.eu/property/landingPage": {\ +"class_field": "LandingPage",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/placeName": {\ +"class_field": "PlaceName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/title": {\ +"class_field": "Title",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/accessRights": {\ +"class_field": "AccessRights",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/description": {\ +"class_field": "Description",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/creator": {\ +"class_field": "Creator",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/contributor": {\ +"class_field": "Contributor",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/legalResponsible": {\ +"class_field": "LegalResponsible",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/owner": {\ +"class_field": "Owner",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/originalId": {\ +"class_field": "OriginalId",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/uri": {\ +"class_field": "Temporal",\ +"substring": "no",\ +"external_reference": "AriadneTemporal"\ +},\ +"https://www.ariadne-infrastructure.eu/property/language": {\ +"class_field": "Language",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/partOf": {\ +"class_field": "IsPartOf",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +}\ +} \ No newline at end of file From 064f29832943ef1be346e74a8312cff670027649 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Thu, 11 Jun 2020 18:20:42 +0200 Subject: [PATCH 5/8] added query to retrieve record ids collection; datasource, collection id and record id are now paramters on a query template from file --- .../reader/RunSPARQLQueryService.java | 230 +++---------- .../src/main/resources/application.properties | 2 +- .../sparql/read_record_data_template.sparql | 166 +++++++++ .../GraphDbReaderAndESIndexTest.java | 39 ++- .../test/resources/application.properties | 319 ------------------ 5 files changed, 259 insertions(+), 497 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql delete mode 100644 dnet-ariadneplus-graphdb-publisher/test/resources/application.properties diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 7746b36..7186e9a 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -5,10 +5,7 @@ import eu.dnetlib.ariadneplus.reader.ResourceManager; import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; import org.eclipse.rdf4j.model.Model; import org.eclipse.rdf4j.model.Statement; -import org.eclipse.rdf4j.query.GraphQuery; -import org.eclipse.rdf4j.query.GraphQueryResult; -import org.eclipse.rdf4j.query.QueryLanguage; -import org.eclipse.rdf4j.query.QueryResults; +import org.eclipse.rdf4j.query.*; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; @@ -19,7 +16,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.StringWriter; +import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; +import java.util.List; @Service public class RunSPARQLQueryService { @@ -65,177 +65,19 @@ public class RunSPARQLQueryService { manager.shutDown(); } - public String executeQueryGraph(){ - query = "PREFIX aocat: \n" + - "PREFIX rdfs: \n" + - "PREFIX skos: \n" + - "PREFIX aoprop: \n" + - "PREFIX onto: \n" + - "PREFIX ariadneplus: \n" + - "PREFIX time: \n" + - "\n" + - "CONSTRUCT { \n" + - "%s aoprop:identifier %s . \n" + - "%s aoprop:originalId ?originalId . \n" + - "%s aoprop:issued ?issued .\n" + - "%s aoprop:modified ?modified .\n" + - "%s aoprop:partOf ?partOf .\n" + - "%s aoprop:creator ?creator . \n" + - "?creator aoprop:name ?creatorName .\n" + - "?creator aoprop:email ?creatorEmail . \n" + - "%s aoprop:contributor ?contributor . \n" + - "?contributor aoprop:name ?contributorName .\n" + - "?contributor aoprop:email ?contributorEmail .\n" + - "%s aoprop:legalResponsible ?legalResponsible . \n" + - "?legalResponsible aoprop:name ?legalResponsibleName .\n" + - "?legalResponsible aoprop:email ?legalResponsibleEmail . \n" + - "%s aoprop:owner ?owner . \n" + - "?owner aoprop:name ?ownerName .\n" + - "?owner aoprop:email ?ownerEmail . \n" + - "%s aoprop:publisher ?publisher . \n" + - "?publisher aoprop:name ?publisherName . \n" + - "?publisher aoprop:email ?publisherEmail . \n" + - "%s aoprop:accessPolicy ?accessPolicy . \n" + - "%s aoprop:accessRights ?accessRights .\n" + - "%s aoprop:landingPage ?landingPage .\n" + - "%s aoprop:spatialRegion ?spatialRegion . \n" + - "?spatialRegion aoprop:placeName ?spatialPlaceName .\n" + - "?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . \n" + - "%s aoprop:spatialRegionPoint ?spatialRegionPoint . \n" + - "?spatialRegionPoint aoprop:lat ?spatialLocationLat . \n" + - "?spatialRegionPoint aoprop:lon ?spatialLocationLon .\n" + - "%s aoprop:spatialRegionBox ?spatialRegionBox . \n" + - "?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat .\n" + - "?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon .\n" + - "?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat .\n" + - "?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon .\n" + - "%s aoprop:uri ?temporal .\n" + - "?temporal aoprop:periodName ?temporalPeriodName .\n" + - "?temporal aoprop:from ?temporalFrom .\n" + - "?temporal aoprop:until ?temporalUntil .\n" + - "%s aoprop:uri ?temporalNative .\n" + - "?temporalNative aoprop:periodName ?temporalNativePeriodName .\n" + - "%s aoprop:archeologicalResourceType ?archeologicalResourceType . \n" + - "?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . \n" + - "%s aoprop:resourceType ?resourceType . \n" + - "%s aoprop:nativeSubject ?nativeSubject . \n" + - "?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel .\n" + - "?nativeSubject aoprop:rdfAbout ?nativeSubject .\n" + - "%s aoprop:derivedSubject ?derivedSubject .\n" + - "?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel .\n" + - "?derivedSubject aoprop:source \"Getty AAT\" .\n" + - "%s aoprop:aatSubjects ?derivedSubject .\n" + - "?derivedSubject aoprop:id ?derivedSubject . \n" + - "?derivedSubject aoprop:label ?derivedSubjectPrefLabel .\n" + - "?derivedSubject aoprop:lang ?aatSubjectsLang .\n" + - "%s aoprop:title ?title . \n" + - "%s aoprop:description ?description . \n" + - "%s aoprop:language ?language . \n" + - "}\n" + - "from \n" + - "from \n" + - "from \n" + - "from \n" + - "from \n" + - "where { \n" + - " \t%s\taocat:has_language / skos:prefLabel ?language .\n" + - " \t%s\taocat:has_original_id ?originalId .\n" + - " \t%s aocat:is_part_of ?partOf .\n" + - " \t%s\taocat:has_creator ?creator .\n" + - " ?creator aocat:has_name ?creatorName .\n" + - "\t %s aocat:has_title ?title .\n" + - " \t%s aocat:has_type / skos:prefLabel ?resourceType .\t\n" + - " \t%s aocat:has_native_subject ?nativeSubject .\n" + - " ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel .\n" + - " optional {\n" + - "\t\t\t%s aocat:has_derived_subject ?derivedSubject .\n" + - " \t?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel .\n" + - " }\n" + - " optional {\n" + - "\t ?creator aocat:has_email ?creatorEmail .\n" + - " \t}\n" + - " optional {\n" + - " %s aocat:has_description ?description .\n" + - " }\n" + - " optional {\n" + - " %s aocat:has_access_policy / rdfs:label ?accessPolicy . \n" + - " }\n" + - " optional {\n" + - " \t%s aocat:has_landing_page / rdfs:label ?landingPage .\n" + - " }\n" + - " \toptional {\n" + - " \t%s aocat:has_temporal_coverage ?temporalNative .\n" + - " \t?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName .\n" + - " }\n" + - " optional {\n" + - " \t%s aocat:has_temporal_coverage ?temporal .\n" + - " \t?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName .\n" + - " \toptional {\n" + - " ?temporal aocat:from ?temporalFrom .\n" + - " ?temporal aocat:until ?temporalUntil .\n" + - " \t}\n" + - " }\n" + - " \t\n" + - " \t{ \n" + - "\t\tselect * \n" + - " where {\n" + - " %s aocat:is_part_of ?collection .\n" + - " %s aocat:was_issued ?issued .\n" + - " %s aocat:was_modified ?modified .\n" + - " %s aocat:has_contributor ?contributor .\n" + - " ?contributor aocat:has_name ?contributorName .\n" + - " %s aocat:has_responsible ?legalResponsible .\n" + - " ?legalResponsible aocat:has_name ?legalResponsibleName .\n" + - " %s aocat:has_owner ?owner .\n" + - " ?owner aocat:has_name ?ownerName .\n" + - " %s aocat:has_publisher ?publisher . \n" + - " ?publisher aocat:has_name ?publisherName .\n" + - " %s aocat:has_access_rights ?accessRights .\n" + - " %s aocat:has_ARIADNE_subject ?archeologicalResourceType . \n" + - " ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .\n" + - " optional {\n" + - " ?contributor aocat:has_email ?contributorEmail .\n" + - " }\n" + - " optional {\n" + - " ?legalResponsible aocat:has_email ?legalResponsibleEmail .\n" + - " }\n" + - " optional {\n" + - " ?owner aocat:has_email ?ownerEmail . \n" + - " }\n" + - " optional {\n" + - " ?publisher aocat:has_email ?publisherEmail .\n" + - " }\n" + - " \t}\n" + - " \t}\n" + - " \n" + - " \t{\n" + - " select * \n" + - " where {\n" + - " \t%s\taocat:has_spatial_coverage ?spatialRegion .\n" + - " ?spatialRegion aocat:has_place_name ?spatialPlaceName .\n" + - " optional {\n" + - " ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem .\n" + - " }\n" + - " optional {\n" + - " %s\taocat:has_spatial_coverage ?spatialRegionPoint .\n" + - " ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; \n" + - " \t\t aocat:has_longitude ?spatialLocationLon . \n" + - " }\n" + - " optional {\n" + - " %s\taocat:has_spatial_coverage ?spatialRegionBox .\n" + - " ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; \n" + - " \t\t aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ;\n" + - " \t\t aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ;\n" + - " \t aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ;\n" + - " }\n" + - " }\t\n" + - " \t}\n" + - " BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang)\n" + - "}"; + public String executeMultipleQueryGraph(String queryTemplate, List recordIds, String datasource, String collectionId){ + if (queryTemplate==null) + return null; + final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); + recordIds.forEach(recordId -> { + executeQueryGraph(selectQueryTemplate, recordId); + }); + return "ok"; + } - if (query.equals("undefined")) - return null; - query = query.replaceAll("%s", ""); + private String executeQueryGraph(String selectQueryTemplate, String recordId){ + query = selectQueryTemplate.replaceAll("%record", recordId); + System.out.println("execute query "+query); openConnection(); StringWriter recordWriter = null; Model resultsModel = null; @@ -248,7 +90,10 @@ public class RunSPARQLQueryService { graphQueryResult.close(); System.out.println("End connection Time: "+Calendar.getInstance().getTime().toString()); System.out.println("count statements: " + resultsModel.size()); - recordWriter = new StringWriter(); + if (resultsModel.size()==0) { + return "noresult"; + } + recordWriter = new StringWriter(); RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); Rio.write(resultsModel, rdfRecordWriter); System.out.println("RDF > json record: "+recordWriter.toString()); @@ -333,4 +178,39 @@ public class RunSPARQLQueryService { public static void setGraphDBRepository(String graphDBRepository) { RunSPARQLQueryService.graphDBRepository = graphDBRepository; } + + public List selectRecordIds(){ + query = "PREFIX rdf: \n" + + "select * \n" + + "from \n" + + "where { \n" + + "\t?recordId rdf:type .\n" + + "} \n"; + if (query.equals("undefined")) + return null; +// query = query.replaceAll("%s", ""); + openConnection(); + String jsonRecord = null; + List recordIds = new ArrayList<>(); + try { + System.out.println("Start connection Time: "+Calendar.getInstance().getTime().toString()); + TupleQuery selectQuery = connection.prepareTupleQuery(QueryLanguage.SPARQL, query); + TupleQueryResult selectQueryResult = selectQuery.evaluate(); + int counter = 0; + while (selectQueryResult.hasNext()) { + BindingSet recordSet = selectQueryResult.next(); + org.eclipse.rdf4j.model.Value recordIdValue = recordSet.getValue("recordId"); + System.out.println(recordIdValue.stringValue()); + recordIds.add(recordIdValue.stringValue()); + counter++; + } + System.out.println("Total records: "+counter); + System.out.println("End connection Time: "+Calendar.getInstance().getTime().toString()); + } catch(Exception e){ + e.printStackTrace(); + } finally{ + closeConnection(); + } + return recordIds; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index d3dcac8..3d2e298 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -5,7 +5,7 @@ server.port=8281 graphdb.serverUrl=http://localhost:7200/ graphdb.writer.user=writer -graphdb.writer.pwd=**** +graphdb.writer.pwd=Writer01 graphdb.repository=ariadneplus-ts01 graphdb.sparqlUrl = http://localhost:7200/sparql diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql new file mode 100644 index 0000000..798e5cb --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -0,0 +1,166 @@ +PREFIX aocat: +PREFIX rdfs: +PREFIX skos: +PREFIX aoprop: +PREFIX onto: +PREFIX ariadneplus: +PREFIX time: + +CONSTRUCT { +%record aoprop:identifier %record . +%record aoprop:originalId ?originalId . +%record aoprop:issued ?issued . +%record aoprop:modified ?modified . +%record aoprop:partOf ?partOf . +%record aoprop:creator ?creator . +?creator aoprop:name ?creatorName . +?creator aoprop:email ?creatorEmail . +%record aoprop:contributor ?contributor . +?contributor aoprop:name ?contributorName . +?contributor aoprop:email ?contributorEmail . +%record aoprop:legalResponsible ?legalResponsible . +?legalResponsible aoprop:name ?legalResponsibleName . +?legalResponsible aoprop:email ?legalResponsibleEmail . +%record aoprop:owner ?owner . +?owner aoprop:name ?ownerName . +?owner aoprop:email ?ownerEmail . +%record aoprop:publisher ?publisher . +?publisher aoprop:name ?publisherName . +?publisher aoprop:email ?publisherEmail . +%record aoprop:accessPolicy ?accessPolicy . +%record aoprop:accessRights ?accessRights . +%record aoprop:landingPage ?landingPage . +%record aoprop:spatialRegion ?spatialRegion . +?spatialRegion aoprop:placeName ?spatialPlaceName . +?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . +%record aoprop:spatialRegionPoint ?spatialRegionPoint . +?spatialRegionPoint aoprop:lat ?spatialLocationLat . +?spatialRegionPoint aoprop:lon ?spatialLocationLon . +%record aoprop:spatialRegionBox ?spatialRegionBox . +?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat . +?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . +?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . +?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +%record aoprop:uri ?temporal . +?temporal aoprop:periodName ?temporalPeriodName . +?temporal aoprop:from ?temporalFrom . +?temporal aoprop:until ?temporalUntil . +%record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . +%record aoprop:archeologicalResourceType ?archeologicalResourceType . +?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . +%record aoprop:resourceType ?resourceType . +%record aoprop:nativeSubject ?nativeSubject . +?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . +?nativeSubject aoprop:rdfAbout ?nativeSubject . +%record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +%record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:lang ?aatSubjectsLang . +%record aoprop:title ?title . +%record aoprop:description ?description . +%record aoprop:language ?language . +} +from +from +from +from +from +where { + %record aocat:has_language / skos:prefLabel ?language . + %record aocat:has_original_id ?originalId . + %record aocat:is_part_of ?partOf . + %record aocat:has_creator ?creator . + ?creator aocat:has_name ?creatorName . + %record aocat:has_title ?title . + %record aocat:has_type / skos:prefLabel ?resourceType . + %record aocat:has_native_subject ?nativeSubject . + ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . + optional { + %record aocat:has_derived_subject ?derivedSubject . + ?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel . + } + optional { + ?creator aocat:has_email ?creatorEmail . + } + optional { + %record aocat:has_description ?description . + } + optional { + %record aocat:has_access_policy / rdfs:label ?accessPolicy . + } + optional { + %record aocat:has_landing_page / rdfs:label ?landingPage . + } + optional { + %record aocat:has_temporal_coverage ?temporalNative . + ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . + } + optional { + %record aocat:has_temporal_coverage ?temporal . + ?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName . + optional { + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } + } + + { + select * + where { + %record aocat:is_part_of ?collection . + %record aocat:was_issued ?issued . + %record aocat:was_modified ?modified . + %record aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + %record aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + %record aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + %record aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + %record aocat:has_access_rights ?accessRights . + %record aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } + } + } + + { + select * + where { + %record aocat:has_spatial_coverage ?spatialRegion . + ?spatialRegion aocat:has_place_name ?spatialPlaceName . + optional { + ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem . + } + optional { + %record aocat:has_spatial_coverage ?spatialRegionPoint . + ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; + aocat:has_longitude ?spatialLocationLon . + } + optional { + %record aocat:has_spatial_coverage ?spatialRegionBox . + ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; + aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ; + aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ; + aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ; + } + } + } + BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang) +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 1d17ba3..b49f9b7 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -7,10 +7,14 @@ import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; import eu.dnetlib.ariadneplus.reader.ResourceManager; import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import org.apache.commons.io.IOUtils; import org.junit.Ignore; import org.junit.Test; import org.springframework.core.io.ClassPathResource; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import java.util.Properties; /** @@ -22,7 +26,7 @@ public class GraphDbReaderAndESIndexTest { private RunSPARQLQueryService runSPQRLQuery; @Test - @Ignore +// @Ignore public void readAndIndexTest() throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); @@ -49,7 +53,38 @@ public class GraphDbReaderAndESIndexTest { BulkUpload bulkUpload = new BulkUpload(); bulkUpload.init(); runSPQRLQuery.setBulkUpload(bulkUpload); - runSPQRLQuery.executeQueryGraph(); + String recordId = ""; + String datasource = "ads"; + String collectionId = "271"; + List recordIds = Arrays.asList(recordId); + final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); + runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId); + + } + + @Test + @Ignore + public void selectRecordsTest() throws Exception { + final ClassPathResource resource = new ClassPathResource("application.properties"); + Properties appProps = new Properties(); + appProps.load(resource.getInputStream()); + + runSPQRLQuery = new RunSPARQLQueryService(); + runSPQRLQuery.setupConnection( + appProps.getProperty("graphdb.writer.user"), + appProps.getProperty("graphdb.writer.pwd"), + appProps.getProperty("repository.url"), + appProps.getProperty("graphdb.repository")); + runSPQRLQuery.selectRecordIds(); + } + + @Test + @Ignore + public void loadQueryTest() throws Exception { + final ClassPathResource resource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(resource.getInputStream(), StandardCharsets.UTF_8.name()); + System.out.println(queryTemplate); } } diff --git a/dnet-ariadneplus-graphdb-publisher/test/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/test/resources/application.properties deleted file mode 100644 index b75655c..0000000 --- a/dnet-ariadneplus-graphdb-publisher/test/resources/application.properties +++ /dev/null @@ -1,319 +0,0 @@ -#server.contextPath=/ariadneplus-graphdb -#update due to springboot 2.1.3 -server.servlet.context-path=/ariadneplus-graphdb -server.port=8281 - -graphdb.serverUrl=http://localhost:7200/ -graphdb.writer.user=writer -graphdb.writer.pwd=***** -graphdb.repository=ariadneplus-ts01 -graphdb.sparqlUrl = http://localhost:7200/sparql - -graphdb.baseURI=https://ariadne-infrastructure.eu/ - -repository.url=http://graphdb-test.ariadne.d4science.org:7200 - -catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')] -general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. -type.path=https://www.ariadne-infrastructure.eu/property/resourceType -exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] -class.map.specifications={\ -"AriadneTemporal": {\ -"class_type": "prototype",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/from": {\ -"class_field": "From",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/periodName": {\ -"class_field": "PeriodName",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/until": {\ -"class_field": "Until",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/uri": {\ -"class_field": "Uri",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"AriadneGeoPoint": {\ -"class_type": "unique",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/lat": {\ -"class_field": "Lat",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/lon": {\ -"class_field": "Lon",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"Spatial": {\ -"class_type": "prototype",\ -"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {\ -"class_field": "Address",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {\ -"class_field": "CoordinateSystem",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/country": {\ -"class_field": "Country",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/location": {\ -"class_field": "Location",\ -"external_reference": "AriadneGeoPoint",\ -"substring": "no"\ -},\ -"https://www.ariadne-infrastructure.eu/property/lat": {\ -"class_field": "Lat",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/lon": {\ -"class_field": "Lon",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/placeName": {\ -"class_field": "PlaceName",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -}}\ -},\ -"ArcheologicalResourceType": {\ -"class_type": "unique",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/id": {\ -"class_field": "Id",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/name": {\ -"class_field": "Name",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"AgentInfo": {\ -"class_type": "prototype",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/name": {\ -"class_field": "Name",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/type": {\ -"class_field": "Type",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/email": {\ -"class_field": "Email",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/phone": {\ -"class_field": "Phone",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"NativeSubject": {\ -"class_type": "prototype",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ -"class_field": "PrefLabel",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/rdfAbout": {\ -"class_field": "RdfAbout",\ -"substring": "yes",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"DerivedSubject": {\ -"class_type": "prototype",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ -"class_field": "PrefLabel",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/source": {\ -"class_field": "Source",\ -"substring": "yes",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"AatSubject": {\ -"class_type": "prototype",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/id": {\ -"class_field": "Id",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/label": {\ -"class_field": "Label",\ -"substring": "yes",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/lang": {\ -"class_field": "Lang",\ -"substring": "yes",\ -"element_type": "java.lang.String"\ -}\ -}\ -},\ -"AriadneCatalogEntry": {\ -"class_type": "unique",\ -"mappings": {\ -"https://www.ariadne-infrastructure.eu/property/accessPolicy": {\ -"class_field": "AccessPolicy",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/identifier": {\ -"class_field": "Identifier",\ -"substring": "yes",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/publisher": {\ -"class_field": "Publisher",\ -"external_reference": "AgentInfo",\ -"substring": "no"\ -},\ -"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\ -"class_field": "ArcheologicalResourceType",\ -"external_reference": "ArcheologicalResourceType",\ -"substring": "no"\ -},\ -"https://www.ariadne-infrastructure.eu/property/issued": {\ -"class_field": "Issued",\ -"element_type": "java.lang.String",\ -"substring": "no"\ -},\ -"https://www.ariadne-infrastructure.eu/property/resourceType": {\ -"class_field": "ResourceType",\ -"element_type": "java.lang.String",\ -"substring": "no"\ -},\ -"https://www.ariadne-infrastructure.eu/property/modified": {\ -"class_field": "Modified",\ -"element_type": "java.lang.String",\ -"substring": "no"\ -},\ -"https://www.ariadne-infrastructure.eu/property/nativeSubject": {\ -"class_field": "NativeSubject",\ -"substring": "no",\ -"external_reference": "NativeSubject"\ -},\ -"https://www.ariadne-infrastructure.eu/property/derivedSubject": {\ -"class_field": "DerivedSubject",\ -"substring": "no",\ -"external_reference": "DerivedSubject"\ -},\ -"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\ -"class_field": "AatSubjects",\ -"substring": "no",\ -"external_reference": "AatSubject"\ -},\ -"https://www.ariadne-infrastructure.eu/property/spatialRegion": {\ -"class_field": "Spatial",\ -"substring": "no",\ -"external_reference": "Spatial"\ -},\ -"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {\ -"class_field": "Spatial",\ -"substring": "no",\ -"external_reference": "Spatial"\ -},\ -"https://www.ariadne-infrastructure.eu/property/landingPage": {\ -"class_field": "LandingPage",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/placeName": {\ -"class_field": "PlaceName",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/title": {\ -"class_field": "Title",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/accessRights": {\ -"class_field": "AccessRights",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/description": {\ -"class_field": "Description",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/creator": {\ -"class_field": "Creator",\ -"substring": "no",\ -"external_reference": "AgentInfo"\ -},\ -"https://www.ariadne-infrastructure.eu/property/contributor": {\ -"class_field": "Contributor",\ -"substring": "no",\ -"external_reference": "AgentInfo"\ -},\ -"https://www.ariadne-infrastructure.eu/property/legalResponsible": {\ -"class_field": "LegalResponsible",\ -"substring": "no",\ -"external_reference": "AgentInfo"\ -},\ -"https://www.ariadne-infrastructure.eu/property/owner": {\ -"class_field": "Owner",\ -"substring": "no",\ -"external_reference": "AgentInfo"\ -},\ -"https://www.ariadne-infrastructure.eu/property/originalId": {\ -"class_field": "OriginalId",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/uri": {\ -"class_field": "Temporal",\ -"substring": "no",\ -"external_reference": "AriadneTemporal"\ -},\ -"https://www.ariadne-infrastructure.eu/property/language": {\ -"class_field": "Language",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/partOf": {\ -"class_field": "IsPartOf",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -}\ -}\ -}\ -} \ No newline at end of file From 06995feefae2ed91a9598a10c6fe1a95e90edc5e Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Fri, 12 Jun 2020 18:14:41 +0200 Subject: [PATCH 6/8] added rest method indexOnES(datasource,collectionId) to retrieve records data from graphDB and index to ES --- .../ariadneplus/elasticsearch/BulkUpload.java | 30 +++--- .../model/AriadneCatalogEntry.java | 99 ------------------- .../ariadneplus/graphdb/GraphDBClient.java | 54 +++++++++- .../graphdb/GraphDBClientFactory.java | 7 +- .../AriadnePlusPublisherController.java | 4 + .../publisher/AriadnePlusPublisherHelper.java | 17 ++++ .../ariadneplus/reader/ResourceManager.java | 40 +++++++- .../reader/RunSPARQLQueryService.java | 55 ++++------- .../ariadneplus/reader/json/ParseRDFJSON.java | 1 + .../reader/utils/PropertiesMap.java | 12 ++- .../src/main/resources/application.properties | 9 +- .../GraphDbReaderAndESIndexTest.java | 2 +- 12 files changed, 163 insertions(+), 167 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index c18a5bd..1efe85d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -1,13 +1,10 @@ package eu.dnetlib.ariadneplus.elasticsearch; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; - -import javax.annotation.PostConstruct; - import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo; import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; import eu.dnetlib.ariadneplus.reader.ResourceManager; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -19,11 +16,20 @@ import org.elasticsearch.common.xcontent.XContentType; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; + @Service public class BulkUpload { - @Value("${elasticsearch.url:localhost:9200}") - private String elasticsearchUrl; + private static final Log log = LogFactory.getLog(BulkUpload.class); + + @Value("${elasticsearch.hostname}") + private String elasticSearchHostName; + + @Value("${elasticsearch.indexname}") + private String elasticSearchIndexName; private RestHighLevelClient client; @@ -31,7 +37,7 @@ public class BulkUpload { public void init() throws IOException { client = new RestHighLevelClient( RestClient.builder( - new HttpHost("elastic-test.ariadne.d4science.org",9200,"http"))); + new HttpHost(elasticSearchHostName,9200,"http"))); } @@ -46,13 +52,13 @@ public class BulkUpload { testPublisher.setName("TEST"); ace.getPublisher().add(testPublisher); String[] splits = ace.getIdentifier().split("/"); - request.add(new IndexRequest("catalog_test").id(splits[splits.length-1]) + request.add(new IndexRequest(elasticSearchIndexName).id(splits[splits.length-1]) .source(ace.toJson(),XContentType.JSON)); - System.out.println("indexing to ES record "+ace.toJson()); + log.debug("Indexing to ES: "+ace.toJson()); BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); - System.out.println("indexing to ES record completed "+bulkResponse.status()); + log.info("Indexing to ES completed with status: "+bulkResponse.status()); if (bulkResponse.hasFailures()) { - System.out.println("FailureMessage: "+bulkResponse.buildFailureMessage()); + log.error("FailureMessage: "+bulkResponse.buildFailureMessage()); } } catch (ClassNotFoundException e) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java index 8d754d7..15dd15b 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -358,103 +358,4 @@ public class AriadneCatalogEntry { public String toJson(){ return new Gson().toJson(this); } - -// public static AriadneCatalogEntry fromRDFJson(JsonElement json, String identifier, Map map){ -// AriadneCatalogEntry acim = new AriadneCatalogEntry(); -// acim.setIdentifier(identifier.substring(identifier.lastIndexOf("/") + 1)); -// JsonObject content = json.getAsJsonObject(); -// for (Map.Entry stringJsonElementEntry : content.entrySet()) { -// switch (stringJsonElementEntry.getKey()){ -// case "https://www.ariadne-infrastructure.eu/property/accessPolicy": -// acim.setAccessPolicy(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/accessRights": -// acim.setAccessRights(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/contributor": -// JsonArray contributor_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List contributor_list = new ArrayList(); -// for (int i = 0; i < contributor_array.size() ; i++ ){ -// String map_key = contributor_array.get(i).getAsJsonObject().get("value").getAsString(); -// contributor_list.add(AgentInfo.fromRDFJson(map.get(map_key))); -// -// } -// acim.setContributor(contributor_list); -// break; -// case "https://www.ariadne-infrastructure.eu/property/description": -// acim.setDescription(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/isPartOf": -// acim.setPartOf(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/issued": -// acim.setIssued(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/landingPage": -// acim.setLandingPage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/language": -// acim.setLanguage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/modified": -// acim.setModified(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/nativeSubject": -// JsonArray nativeSubject_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List nativeSubject_list = new ArrayList(); -// for (int i = 0; i < nativeSubject_array.size() ; i++ ){ -// String map_key = nativeSubject_array.get(i).getAsJsonObject().get("value").getAsString(); -// nativeSubject_list.add(NativeSubject.fromRDFJson(map.get(map_key))); -// -// } -// acim.setNativeSubject(nativeSubject_list); -// break; -// case "https://www.ariadne-infrastructure.eu/property/originalId": -// acim.setOriginalId(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/resourceType": -// acim.setResourceType(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/spatial": -// JsonArray spatial_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List spatial_list = new ArrayList(); -// for (int i = 0; i < spatial_array.size() ; i++ ){ -// String map_key = spatial_array.get(i).getAsJsonObject().get("value").getAsString(); -// spatial_list.add(Spatial.fromRDFJson(map.get(map_key), map)); -// -// } -// acim.setSpatial(spatial_list); -// break; -// case "https://www.ariadne-infrastructure.eu/property/temporal": -// JsonArray temporal_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List temporal_list = new ArrayList<>(); -// for(int i=0; i < temporal_array.size(); i++){ -// String map_key = temporal_array.get(i).getAsJsonObject().get("value").getAsString(); -// temporal_list.add(AriadneTemporal.fromRDFJson(map.get(map_key))); -// } -// acim.setTemporal(temporal_list); -// break; -// case "https://www.ariadne-infrastructure.eu/property/title": -// acim.setTitle(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/publisher": -// JsonArray publisher_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List publisher_list = new ArrayList(); -// for (int i = 0; i < publisher_array.size() ; i++ ){ -// String map_key = publisher_array.get(i).getAsJsonObject().get("value").getAsString(); -// publisher_list.add(AgentInfo.fromRDFJson(map.get(map_key))); -// -// } -// acim.setPublisher(publisher_list); -// break; -// case "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": -// acim.setArcheologicalResourceType(ArcheologicalResourceType.fromRDFJson(map.get(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()))); -// } -// -// -// } -// -// System.out.println(acim.toJson()); -// return acim; -// } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java index f2d67af..fd8f294 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java @@ -2,8 +2,16 @@ package eu.dnetlib.ariadneplus.graphdb; import java.io.File; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -23,19 +31,32 @@ import org.eclipse.rdf4j.rio.RDFFormat; import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherException; import eu.dnetlib.ariadneplus.rdf.RecordParserHelper; import net.sf.saxon.s9api.SaxonApiException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; /** * @author enrico.ottonello * */ +@Component public class GraphDBClient { private static final Log log = LogFactory.getLog(GraphDBClient.class); public static final String PROVENANCE_NS = "http://www.d-net.research-infrastructures.eu/provenance/"; - public static final int NUM_RECORDS_THRESHOLD = 10; - + public static final int NUM_RECORDS_THRESHOLD = 10; + + @Autowired + private RunSPARQLQueryService runSPQRLQuery; + @Autowired + private ParseRDFJSON parseRDFJSON; + @Autowired + private ResourceManager resourceManager; + @Autowired + private BulkUpload bulkUpload; + private RecordParserHelper recordParserHelper; private String graphDBServerUrl; private String graphDBBaseURI; @@ -43,7 +64,7 @@ public class GraphDBClient { private String writerPwd; private String repository; - protected GraphDBClient(final RecordParserHelper recordParserHelper, + protected void setup(final RecordParserHelper recordParserHelper, final String graphDBServerUrl, final String graphDBBaseURI, final String writerUser, final String writerPwd, final String repository) { this.recordParserHelper = recordParserHelper; this.graphDBServerUrl = graphDBServerUrl; @@ -318,4 +339,31 @@ public class GraphDBClient { throw new AriadnePlusPublisherException(e); } } + + public RunSPARQLQueryService getRunSPQRLQuery() { + return runSPQRLQuery; + } + + public void setRunSPQRLQuery(RunSPARQLQueryService runSPQRLQuery) { + this.runSPQRLQuery = runSPQRLQuery; + } + + public String indexOnES(String datasource, String collectionId) throws AriadnePlusPublisherException { + try { + runSPQRLQuery.setupConnection( getWriterUser(), getWriterPwd(), this.graphDBServerUrl, getRepository()); + runSPQRLQuery.setParser(parseRDFJSON); + runSPQRLQuery.setResourceManager(resourceManager); + runSPQRLQuery.setBulkUpload(bulkUpload); + String recordId = ""; + List recordIds = Arrays.asList(recordId); +// List recordIds = runSPQRLQuery.selectRecordIds(datasource, collectionId); + final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); + runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId); + }catch(Throwable e){ + log.error(e); + throw new AriadnePlusPublisherException(e); + } + return "ok"; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java index 701ec99..78805b6 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java @@ -31,9 +31,12 @@ public class GraphDBClientFactory { @Autowired private RecordParserHelper recordParserHelper; + @Autowired + private GraphDBClient graphDBClient; + public GraphDBClient getGraphDBClient() { - log.debug("Creating GraphDBClient for "+graphDBServerUrl); - return new GraphDBClient(recordParserHelper, graphDBServerUrl, graphDBBaseURI, writerUser, writerPwd, repository); + graphDBClient.setup(recordParserHelper, graphDBServerUrl, graphDBBaseURI, writerUser, writerPwd, repository); + return graphDBClient; } public RecordParserHelper getRecordParserHelper() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java index 91c6454..8d696da 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java @@ -69,4 +69,8 @@ public class AriadnePlusPublisherController { return getAriadnePlusPublisherHelper().feedFromURL(dataUrl, context, getTarget(DEFAULT_TARGET_ENDPOINT)); } + @RequestMapping(value = "/indexOnES", method = RequestMethod.POST) + public String indexOnES(@RequestParam final String datasource, @RequestParam final String collectionId) throws AriadnePlusPublisherException { + return getAriadnePlusPublisherHelper().indexOnES(datasource, collectionId, getTarget(DEFAULT_TARGET_ENDPOINT)); + } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java index 4c5fcf4..eed6ac0 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java @@ -88,6 +88,17 @@ public class AriadnePlusPublisherHelper { return res; } + public String indexOnES(final String datasource, final String collectionId, final AriadnePlusTargets target) throws AriadnePlusPublisherException { + String res; + switch(target){ + case GRAPHDB: + res = indexOnES(datasource, collectionId); + break; + default: throw new AriadnePlusPublisherException("Target "+target+" not supported yet"); + } + return res; + } + private void publishGraphDB(final String record) throws AriadnePlusPublisherException { log.debug("Publishing on graphdb"); GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); @@ -124,4 +135,10 @@ public class AriadnePlusPublisherHelper { GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); return graphDBClient.feedFromURL(dataUrl, context); } + + private String indexOnES(final String datasource, final String collectionId) throws AriadnePlusPublisherException { + log.info("indexOnES "+datasource + " " + collectionId); + GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); + return graphDBClient.indexOnES(datasource, collectionId); + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java index 6eaabc1..cb87c7b 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -14,6 +14,8 @@ import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; import eu.dnetlib.ariadneplus.reader.utils.ClassSpec; import eu.dnetlib.ariadneplus.reader.utils.Mappings; import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -26,6 +28,8 @@ import net.minidev.json.JSONObject; @Service public class ResourceManager { + private static final Log log = LogFactory.getLog(ResourceManager.class); + @Value("${type.path:undefined}") private String type_path; @Value("${general.classpath:undefined}") @@ -53,6 +57,8 @@ public class ResourceManager { public void init(){ Type listType = new TypeToken>(){}.getType(); not_parsable = new Gson().fromJson(exclude_predicates, listType); + propertiesMap = new PropertiesMap(); + propertiesMap.fill(spec); } @@ -87,11 +93,8 @@ public class ResourceManager { } } if (entry == null) { - System.out.println("entry NULL " ); - return null; - + return null; } - System.out.println("class_name: " + class_name); Class c = Class.forName(general_classpath + class_name); Object class_instance = c.newInstance(); ClassSpec class_spec = propertiesMap.get(class_name); @@ -143,6 +146,35 @@ public class ResourceManager { return class_instance; } + public String getType_path() { + return type_path; + } + public void setType_path(String type_path) { + this.type_path = type_path; + } + public String getGeneral_classpath() { + return general_classpath; + } + + public void setGeneral_classpath(String general_classpath) { + this.general_classpath = general_classpath; + } + + public String getExclude_predicates() { + return exclude_predicates; + } + + public void setExclude_predicates(String exclude_predicates) { + this.exclude_predicates = exclude_predicates; + } + + public String getSpec() { + return spec; + } + + public void setSpec(String spec) { + this.spec = spec; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 7186e9a..6aee91b 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -1,10 +1,10 @@ package eu.dnetlib.ariadneplus.reader; import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; -import eu.dnetlib.ariadneplus.reader.ResourceManager; import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.query.*; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; @@ -12,23 +12,17 @@ import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.RDFWriter; import org.eclipse.rdf4j.rio.Rio; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.StringWriter; import java.util.ArrayList; import java.util.Calendar; -import java.util.Collections; import java.util.List; @Service public class RunSPARQLQueryService { - @Value("${sparql.query:undefined}") - private String query; - - @Value("${repository.url:undefined}") - private String repository_url; + private static final Log log = LogFactory.getLog(RunSPARQLQueryService.class); private RepositoryConnection connection; private RemoteRepositoryManager manager; @@ -76,27 +70,26 @@ public class RunSPARQLQueryService { } private String executeQueryGraph(String selectQueryTemplate, String recordId){ - query = selectQueryTemplate.replaceAll("%record", recordId); - System.out.println("execute query "+query); - openConnection(); + log.debug("Retrieving "+recordId+" ..."); + String query = selectQueryTemplate.replaceAll("%record", recordId); + openConnection(); StringWriter recordWriter = null; Model resultsModel = null; String jsonRecord = null; try { - System.out.println("Start connection Time: "+Calendar.getInstance().getTime().toString()); + log.debug("Started at: "+Calendar.getInstance().getTime().toString()); GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query); GraphQueryResult graphQueryResult = graphQuery.evaluate(); resultsModel = QueryResults.asModel(graphQueryResult); graphQueryResult.close(); - System.out.println("End connection Time: "+Calendar.getInstance().getTime().toString()); - System.out.println("count statements: " + resultsModel.size()); + log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); + log.debug("Statements retrieved: " + resultsModel.size()); if (resultsModel.size()==0) { return "noresult"; } recordWriter = new StringWriter(); RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); Rio.write(resultsModel, rdfRecordWriter); - System.out.println("RDF > json record: "+recordWriter.toString()); parser.parse(recordWriter.toString()); resourceManager.manage(parser); bulkUpload.index(resourceManager); @@ -111,38 +104,26 @@ public class RunSPARQLQueryService { return jsonRecord; } - private void dumpModel(Model model) { - System.out.print(" [ dump model ] "); - for (Statement stmt: model) { - System.out.println(stmt.toString()); - } - } - public ParseRDFJSON getParser() { return parser; } - public void setParser(ParseRDFJSON parser) { this.parser = parser; } - public ResourceManager getResourceManager() { return resourceManager; } - public void setResourceManager(ResourceManager resourceManager) { this.resourceManager = resourceManager; } - public BulkUpload getBulkUpload() { return bulkUpload; } - public void setBulkUpload(BulkUpload bulkUpload) { this.bulkUpload = bulkUpload; } @@ -179,33 +160,31 @@ public class RunSPARQLQueryService { RunSPARQLQueryService.graphDBRepository = graphDBRepository; } - public List selectRecordIds(){ - query = "PREFIX rdf: \n" + + public List selectRecordIds(String datasource, String collectionId){ + log.debug("Retrieving recordIds from GraphDB ..."); + String queryTemplate = "PREFIX rdf: \n" + "select * \n" + - "from \n" + + "from \n" + "where { \n" + "\t?recordId rdf:type .\n" + "} \n"; - if (query.equals("undefined")) - return null; -// query = query.replaceAll("%s", ""); + String query = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); openConnection(); String jsonRecord = null; List recordIds = new ArrayList<>(); try { - System.out.println("Start connection Time: "+Calendar.getInstance().getTime().toString()); + log.debug("Started at: "+Calendar.getInstance().getTime().toString()); TupleQuery selectQuery = connection.prepareTupleQuery(QueryLanguage.SPARQL, query); TupleQueryResult selectQueryResult = selectQuery.evaluate(); int counter = 0; while (selectQueryResult.hasNext()) { BindingSet recordSet = selectQueryResult.next(); org.eclipse.rdf4j.model.Value recordIdValue = recordSet.getValue("recordId"); - System.out.println(recordIdValue.stringValue()); recordIds.add(recordIdValue.stringValue()); counter++; } - System.out.println("Total records: "+counter); - System.out.println("End connection Time: "+Calendar.getInstance().getTime().toString()); + log.debug("Total recordIds retrieved: "+counter); + log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); } catch(Exception e){ e.printStackTrace(); } finally{ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java index 41d377d..1ffd590 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java @@ -20,6 +20,7 @@ public class ParseRDFJSON { static JSONObject map ; + @Value("${catalog.entry.path}") private String catalogEntryJsonPath; private String json; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java index 7c7372d..a79b5f2 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java @@ -13,8 +13,8 @@ import java.util.Map; @Component public class PropertiesMap { -// @Value("${class.map.specifications:undefined}") -// private String spec; + @Value("${class.map.specifications:undefined}") + private String spec; private static Map map; @@ -22,10 +22,16 @@ public class PropertiesMap { return map.get(key); } - @PostConstruct public void fill(String spec){ Type mapType = new TypeToken>(){}.getType(); map = new Gson().fromJson(spec,mapType); } + + @PostConstruct + public void fill(){ + Type mapType = new TypeToken>(){}.getType(); + map = new Gson().fromJson(spec,mapType); + + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 3d2e298..f2ca6cb 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -3,15 +3,14 @@ server.servlet.context-path=/ariadneplus-graphdb server.port=8281 -graphdb.serverUrl=http://localhost:7200/ +graphdb.serverUrl=http://graphdb-test.ariadne.d4science.org:7200 graphdb.writer.user=writer -graphdb.writer.pwd=Writer01 +graphdb.writer.pwd=******** graphdb.repository=ariadneplus-ts01 -graphdb.sparqlUrl = http://localhost:7200/sparql - graphdb.baseURI=https://ariadne-infrastructure.eu/ -repository.url=http://graphdb-test.ariadne.d4science.org:7200 +elasticsearch.hostname=elastic-test.ariadne.d4science.org +elasticsearch.indexname=catalog_test catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')] general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index b49f9b7..c979607 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -26,7 +26,7 @@ public class GraphDbReaderAndESIndexTest { private RunSPARQLQueryService runSPQRLQuery; @Test -// @Ignore + @Ignore public void readAndIndexTest() throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); From 42c3f5e88545a89d506901c0a3e8c143fe798e26 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 15 Jun 2020 12:10:59 +0200 Subject: [PATCH 7/8] added indexing of collection data with dedicated query --- .../ariadneplus/graphdb/GraphDBClient.java | 4 + .../reader/RunSPARQLQueryService.java | 26 +++- .../read_collection_data_template.sparql | 124 ++++++++++++++++++ 3 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java index fd8f294..c1b3f4b 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java @@ -360,6 +360,10 @@ public class GraphDBClient { final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId); +// List collectionResourceId = runSPQRLQuery.selectCollectionId(datasource, collectionId); +// final ClassPathResource selectCollectionTemplateRes = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql"); +// String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name()); +// runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, collectionResourceId, datasource, collectionId); }catch(Throwable e){ log.error(e); throw new AriadnePlusPublisherException(e); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 6aee91b..8fac2f4 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -161,7 +161,7 @@ public class RunSPARQLQueryService { } public List selectRecordIds(String datasource, String collectionId){ - log.debug("Retrieving recordIds from GraphDB ..."); + log.debug("Retrieving record Ids from GraphDB ..."); String queryTemplate = "PREFIX rdf: \n" + "select * \n" + "from \n" + @@ -169,9 +169,25 @@ public class RunSPARQLQueryService { "\t?recordId rdf:type .\n" + "} \n"; String query = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); + return executeSelect(query); + } + + public List selectCollectionId(String datasource, String collectionId){ + log.debug("Retrieving collection Id from GraphDB ..."); + String queryTemplate = "PREFIX rdf: \n" + + "select * \n" + + "from \n" + + "where { \n" + + "\t?recordId rdf:type .\n" + + "} \n"; + String query = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); + return executeSelect(query); + } + + private List executeSelect(String query){ openConnection(); String jsonRecord = null; - List recordIds = new ArrayList<>(); + List results = new ArrayList<>(); try { log.debug("Started at: "+Calendar.getInstance().getTime().toString()); TupleQuery selectQuery = connection.prepareTupleQuery(QueryLanguage.SPARQL, query); @@ -180,16 +196,16 @@ public class RunSPARQLQueryService { while (selectQueryResult.hasNext()) { BindingSet recordSet = selectQueryResult.next(); org.eclipse.rdf4j.model.Value recordIdValue = recordSet.getValue("recordId"); - recordIds.add(recordIdValue.stringValue()); + results.add(recordIdValue.stringValue()); counter++; } - log.debug("Total recordIds retrieved: "+counter); + log.debug("Total records retrieved: "+counter); log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); } catch(Exception e){ e.printStackTrace(); } finally{ closeConnection(); } - return recordIds; + return results; } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql new file mode 100644 index 0000000..33449aa --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -0,0 +1,124 @@ +PREFIX aocat: +PREFIX rdfs: +PREFIX skos: +PREFIX aoprop: +PREFIX onto: +PREFIX ariadneplus: +PREFIX time: + +CONSTRUCT { +%record aoprop:identifier %record . +%record aoprop:originalId ?originalId . +%record aoprop:issued ?issued . +%record aoprop:modified ?modified . +%record aoprop:partOf ?partOf . +%record aoprop:creator ?creator . +?creator aoprop:name ?creatorName . +?creator aoprop:email ?creatorEmail . +%record aoprop:contributor ?contributor . +?contributor aoprop:name ?contributorName . +?contributor aoprop:email ?contributorEmail . +%record aoprop:legalResponsible ?legalResponsible . +?legalResponsible aoprop:name ?legalResponsibleName . +?legalResponsible aoprop:email ?legalResponsibleEmail . +%record aoprop:owner ?owner . +?owner aoprop:name ?ownerName . +?owner aoprop:email ?ownerEmail . +%record aoprop:publisher ?publisher . +?publisher aoprop:name ?publisherName . +?publisher aoprop:email ?publisherEmail . +%record aoprop:accessPolicy ?accessPolicy . +%record aoprop:accessRights ?accessRights . +%record aoprop:landingPage ?landingPage . +%record aoprop:spatialRegion ?spatialRegion . +?spatialRegion aoprop:placeName ?spatialPlaceName . +?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . +%record aoprop:spatialRegionPoint ?spatialRegionPoint . +?spatialRegionPoint aoprop:lat ?spatialLocationLat . +?spatialRegionPoint aoprop:lon ?spatialLocationLon . +%record aoprop:spatialRegionBox ?spatialRegionBox . +?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat . +?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . +?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . +?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +%record aoprop:uri ?temporal . +?temporal aoprop:periodName ?temporalPeriodName . +?temporal aoprop:from ?temporalFrom . +?temporal aoprop:until ?temporalUntil . +%record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . +%record aoprop:archeologicalResourceType ?archeologicalResourceType . +?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . +%record aoprop:resourceType ?resourceType . +%record aoprop:nativeSubject ?nativeSubject . +?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . +?nativeSubject aoprop:rdfAbout ?nativeSubject . +%record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +%record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:lang ?aatSubjectsLang . +%record aoprop:title ?title . +%record aoprop:description ?description . +%record aoprop:language ?language . +} +from +from +from +from +from +where { + %record aocat:has_language / skos:prefLabel ?language . + %record aocat:has_original_id ?originalId . + %record aocat:has_creator ?creator . + ?creator aocat:has_name ?creatorName . + %record aocat:has_title ?title . + %record aocat:has_type / skos:prefLabel ?resourceType . + optional { + ?creator aocat:has_email ?creatorEmail . + } + optional { + %record aocat:has_description ?description . + } + optional { + %record aocat:has_access_policy / rdfs:label ?accessPolicy . + } + optional { + %record aocat:has_landing_page / rdfs:label ?landingPage . + } + + { + select * + where { + %record aocat:was_issued ?issued . + %record aocat:was_modified ?modified . + %record aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + %record aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + %record aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + %record aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + %record aocat:has_access_rights ?accessRights . + %record aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } + } + } + + BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang) +} \ No newline at end of file From db9b70feb09252dac5d0b49b2bb33f1a1f932853 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 16 Jun 2020 02:36:16 +0200 Subject: [PATCH 8/8] new node and workflow to index on ES --- .../ariadneplus/elasticsearch/BulkUpload.java | 8 + .../model/ArchaeologicalResourceType.java | 52 ++++++ .../model/ArcheologicalResourceType.java | 52 ------ .../model/AriadneCatalogEntry.java | 10 +- .../ariadneplus/graphdb/GraphDBClient.java | 18 +- .../ariadneplus/reader/ResourceManager.java | 6 +- .../reader/RunSPARQLQueryService.java | 13 +- .../ariadneplus/reader/json/ParseRDFJSON.java | 23 +++ .../src/main/resources/application.properties | 9 +- .../src/main/resources/query.txt | 164 ------------------ .../GraphDbReaderAndESIndexTest.java | 14 +- .../workflows/nodes/IndexOnESJobNode.java | 155 +++++++++++++++++ ...licationContext-ariadneplus-msro-nodes.xml | 2 + .../workflows/repo-hi/index_on_es_wf.xml.st | 54 ++++++ .../workflows/index_on_es_template.xml | 32 ++++ .../workflows/repo_hi_index_on_es.xml | 60 +++++++ 16 files changed, 424 insertions(+), 248 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt create mode 100644 dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java create mode 100644 dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st create mode 100644 dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml create mode 100644 dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 1efe85d..294e29c 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -33,6 +33,14 @@ public class BulkUpload { private RestHighLevelClient client; + public void init(String elasticSearchHostName, String elasticSearchIndexName) throws IOException { + this.elasticSearchIndexName = elasticSearchIndexName; + client = new RestHighLevelClient( + RestClient.builder( + new HttpHost(elasticSearchHostName,9200,"http"))); + + } + @PostConstruct public void init() throws IOException { client = new RestHighLevelClient( diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java new file mode 100644 index 0000000..ae7cce6 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java @@ -0,0 +1,52 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class ArchaeologicalResourceType { + private long id = 100000; + private String name; + + public ArchaeologicalResourceType() { + } + +// public static ArchaeologicalResourceType fromRDFJson(JsonElement json) { +// ArchaeologicalResourceType art = new ArchaeologicalResourceType(); +// for (Map.Entry entry : json.getAsJsonObject().entrySet()){ +// switch (entry.getKey()){ +// case "https://www.ariadne-infrastructure.eu/property/id" : +// art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "https://www.ariadne-infrastructure.eu/property/name": +// String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); +// art.setName(tmp); +// break; +// +// } +// } +// +// return art; +// } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public static ArchaeologicalResourceType fromJson(String json){ + return new Gson().fromJson(json, ArchaeologicalResourceType.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java deleted file mode 100644 index 62fa44d..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java +++ /dev/null @@ -1,52 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.Map; - -public class ArcheologicalResourceType { - private String id; - private String name; - - public ArcheologicalResourceType() { - } - - public static ArcheologicalResourceType fromRDFJson(JsonElement json) { - ArcheologicalResourceType art = new ArcheologicalResourceType(); - for (Map.Entry entry : json.getAsJsonObject().entrySet()){ - switch (entry.getKey()){ - case "https://www.ariadne-infrastructure.eu/property/id" : - art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/name": - String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); - art.setName(tmp); - break; - - } - } - - return art; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public static ArcheologicalResourceType fromJson(String json){ - return new Gson().fromJson(json, ArcheologicalResourceType.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java index 15dd15b..5467e33 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -9,7 +9,7 @@ public class AriadneCatalogEntry { private List aatSubjects; private String accessPolicy; private String accessRights; - private ArcheologicalResourceType archeologicalResourceType; + private ArchaeologicalResourceType archaeologicalResourceType; private String contactPoint; private List contributor; private List creator; @@ -80,12 +80,12 @@ public class AriadneCatalogEntry { this.accessRights = accessRights; } - public ArcheologicalResourceType getArcheologicalResourceType() { - return archeologicalResourceType; + public ArchaeologicalResourceType getArchaeologicalResourceType() { + return archaeologicalResourceType; } - public void setArcheologicalResourceType(ArcheologicalResourceType archeologicalResourceType) { - this.archeologicalResourceType = archeologicalResourceType; + public void setArchaeologicalResourceType(ArchaeologicalResourceType archaeologicalResourceType) { + this.archaeologicalResourceType = archaeologicalResourceType; } public String getContactPoint() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java index c1b3f4b..d685986 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java @@ -354,16 +354,18 @@ public class GraphDBClient { runSPQRLQuery.setParser(parseRDFJSON); runSPQRLQuery.setResourceManager(resourceManager); runSPQRLQuery.setBulkUpload(bulkUpload); - String recordId = ""; - List recordIds = Arrays.asList(recordId); -// List recordIds = runSPQRLQuery.selectRecordIds(datasource, collectionId); +// String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/02E4F4B5-24B7-3AD7-B460-CFA8B1F0BD1F"; +// List recordIds = Arrays.asList(recordId); + List recordIds = runSPQRLQuery.selectRecordIds(datasource, collectionId); final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); - runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId); -// List collectionResourceId = runSPQRLQuery.selectCollectionId(datasource, collectionId); -// final ClassPathResource selectCollectionTemplateRes = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql"); -// String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name()); -// runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, collectionResourceId, datasource, collectionId); + boolean isCollection = false; + runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, isCollection); + List collectionResourceId = runSPQRLQuery.selectCollectionId(datasource, collectionId); + final ClassPathResource selectCollectionTemplateRes = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql"); + String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name()); + isCollection = true; + runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, collectionResourceId, datasource, collectionId, isCollection); }catch(Throwable e){ log.error(e); throw new AriadnePlusPublisherException(e); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java index cb87c7b..73b2196 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -49,8 +49,8 @@ public class ResourceManager { this.type_path = type_path; this.general_classpath = general_classpath; this.exclude_predicates = exclude_predicates; - propertiesMap = new PropertiesMap(); - propertiesMap.fill(spec); + this.spec = spec; + init(); } @PostConstruct @@ -87,7 +87,7 @@ public class ResourceManager { if(entry instanceof LinkedHashMap){ LinkedHashMap tmp = (LinkedHashMap)((JSONArray)((LinkedHashMap)entry).get(type_path)).get(0); class_name = (String)tmp.get("value"); - if (class_name.equals("provided record")) { + if (class_name.equals("provided record") || class_name.equals("Dataset Collection")) { class_name = "AriadneCatalogEntry"; } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 8fac2f4..fd79685 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -59,19 +59,19 @@ public class RunSPARQLQueryService { manager.shutDown(); } - public String executeMultipleQueryGraph(String queryTemplate, List recordIds, String datasource, String collectionId){ + public String executeMultipleQueryGraph(String queryTemplate, List recordIds, String datasource, String collectionId, boolean isCollection){ if (queryTemplate==null) return null; final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); recordIds.forEach(recordId -> { - executeQueryGraph(selectQueryTemplate, recordId); + executeQueryGraph(selectQueryTemplate, recordId, isCollection); }); return "ok"; } - private String executeQueryGraph(String selectQueryTemplate, String recordId){ - log.debug("Retrieving "+recordId+" ..."); - String query = selectQueryTemplate.replaceAll("%record", recordId); + private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){ + log.debug("Retrieving "+recordId+" - isCollection:"+isCollection ); + String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">"); openConnection(); StringWriter recordWriter = null; Model resultsModel = null; @@ -90,6 +90,9 @@ public class RunSPARQLQueryService { recordWriter = new StringWriter(); RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); Rio.write(resultsModel, rdfRecordWriter); + if (isCollection) { + parser.setCollection(true); + } parser.parse(recordWriter.toString()); resourceManager.manage(parser); bulkUpload.index(resourceManager); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java index 1ffd590..4a2549f 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java @@ -3,6 +3,9 @@ package eu.dnetlib.ariadneplus.reader.json; import java.util.Iterator; import java.util.LinkedHashMap; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -18,11 +21,18 @@ import net.minidev.json.parser.ParseException; @Service public class ParseRDFJSON { + private static final Log log = LogFactory.getLog(ParseRDFJSON.class); + static JSONObject map ; @Value("${catalog.entry.path}") private String catalogEntryJsonPath; + @Value("${catalog.entry.collection.path}") + private String catalogEntryCollectionJsonPath; + + private boolean isCollection = false; + private String json; private Iterator it ; @@ -45,8 +55,10 @@ public class ParseRDFJSON { setJson(json); fillMap(); DocumentContext jsonContext = JsonPath.parse(json); + log.debug(getCatalogEntryJsonPath()); JSONArray entries = jsonContext.read(getCatalogEntryJsonPath()); int size = entries.size(); + log.debug("num elements in json: "+size); it = entries.iterator(); } @@ -63,10 +75,21 @@ public class ParseRDFJSON { } public String getCatalogEntryJsonPath() { + if (isCollection) { + return catalogEntryCollectionJsonPath; + } return catalogEntryJsonPath; } public void setCatalogEntryJsonPath(String catalogEntryJsonPath) { this.catalogEntryJsonPath = catalogEntryJsonPath; } + + public boolean isCollection() { + return isCollection; + } + + public void setCollection(boolean collection) { + isCollection = collection; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index f2ca6cb..09399de 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -5,7 +5,7 @@ server.port=8281 graphdb.serverUrl=http://graphdb-test.ariadne.d4science.org:7200 graphdb.writer.user=writer -graphdb.writer.pwd=******** +graphdb.writer.pwd=***** graphdb.repository=ariadneplus-ts01 graphdb.baseURI=https://ariadne-infrastructure.eu/ @@ -13,6 +13,7 @@ elasticsearch.hostname=elastic-test.ariadne.d4science.org elasticsearch.indexname=catalog_test catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')] +catalog.entry.collection.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='Dataset Collection')] general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. type.path=https://www.ariadne-infrastructure.eu/property/resourceType exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] @@ -95,7 +96,7 @@ class.map.specifications={\ "element_type": "java.lang.String"\ }}\ },\ -"ArcheologicalResourceType": {\ +"ArchaeologicalResourceType": {\ "class_type": "unique",\ "mappings": {\ "https://www.ariadne-infrastructure.eu/property/id": {\ @@ -204,8 +205,8 @@ class.map.specifications={\ "substring": "no"\ },\ "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\ -"class_field": "ArcheologicalResourceType",\ -"external_reference": "ArcheologicalResourceType",\ +"class_field": "ArchaeologicalResourceType",\ +"external_reference": "ArchaeologicalResourceType",\ "substring": "no"\ },\ "https://www.ariadne-infrastructure.eu/property/issued": {\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt b/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt deleted file mode 100644 index 0e669bd..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt +++ /dev/null @@ -1,164 +0,0 @@ -PREFIX aocat: -PREFIX rdfs: -PREFIX skos: -PREFIX aoprop: -PREFIX onto: -PREFIX ariadneplus: -PREFIX time: - -CONSTRUCT { -?record aoprop:identifier ?record . -?record aoprop:originalId ?originalId . -?record aoprop:issued ?issued . -?record aoprop:modified ?modified . -?record aoprop:partOf ?partOf . -?record aoprop:creator ?creator . -?creator aoprop:name ?creatorName . -?creator aoprop:email ?creatorEmail . -?record aoprop:contributor ?contributor . -?contributor aoprop:name ?contributorName . -?contributor aoprop:email ?contributorEmail . -?record aoprop:legalResponsible ?legalResponsible . -?legalResponsible aoprop:name ?legalResponsibleName . -?legalResponsible aoprop:email ?legalResponsibleEmail . -?record aoprop:owner ?owner . -?owner aoprop:name ?ownerName . -?owner aoprop:email ?ownerEmail . -?record aoprop:publisher ?publisher . -?publisher aoprop:name ?publisherName . -?publisher aoprop:email ?publisherEmail . -?record aoprop:accessPolicy ?accessPolicy . -?record aoprop:accessRights ?accessRights . -?record aoprop:landingPage ?landingPage . -?record aoprop:spatialRegion ?spatialRegion . -?spatialRegion aoprop:placeName ?spatialPlaceName . -?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . -?record aoprop:spatialRegionPoint ?spatialRegionPoint . -?spatialRegionPoint aoprop:lat ?spatialLocationLat . -?spatialRegionPoint aoprop:lon ?spatialLocationLon . -?record aoprop:spatialRegionBox ?spatialRegionBox . -?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat . -?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . -?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . -?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . -?record aoprop:uri ?temporal . -?temporal aoprop:periodName ?temporalPeriodName . -?temporal aoprop:from ?temporalFrom . -?temporal aoprop:until ?temporalUntil . -?record aoprop:uri ?temporalNative . -?temporalNative aoprop:periodName ?temporalNativePeriodName . -?record aoprop:archeologicalResourceType ?archeologicalResourceType . -?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . -?record aoprop:resourceType ?resourceType . -?record aoprop:nativeSubject ?nativeSubject . -?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . -?nativeSubject aoprop:rdfAbout ?nativeSubject . -?record aoprop:derivedSubject ?derivedSubject . -?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . -?derivedSubject aoprop:source "Getty AAT" . -?record aoprop:aatSubjects ?derivedSubject . -?derivedSubject aoprop:id ?derivedSubject . -?derivedSubject aoprop:label ?derivedSubjectPrefLabel . -?record aoprop:title ?title . -?record aoprop:description ?description . -?record aoprop:language ?language . -} -from -from -from -from -from -where { - ?record aocat:has_language / skos:prefLabel ?language . - ?record aocat:has_original_id ?originalId . - ?record aocat:is_part_of ?partOf . - ?record aocat:has_creator ?creator . - ?creator aocat:has_name ?creatorName . - ?record aocat:has_title ?title . - ?record aocat:has_type / skos:prefLabel ?resourceType . - ?record aocat:has_native_subject ?nativeSubject . - ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . - optional { - ?record aocat:has_derived_subject ?derivedSubject . - ?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel . - } - optional { - ?creator aocat:has_email ?creatorEmail . - } - optional { - ?record aocat:has_description ?description . - } - optional { - ?record aocat:has_access_policy / rdfs:label ?accessPolicy . - } - optional { - ?record aocat:has_landing_page / rdfs:label ?landingPage . - } - optional { - ?record aocat:has_temporal_coverage ?temporalNative . - ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . - } - optional { - ?record aocat:has_temporal_coverage ?temporal . - ?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName . - optional { - ?temporal aocat:from ?temporalFrom . - ?temporal aocat:until ?temporalUntil . - } - } - - { - select * - where { - ?record aocat:is_part_of ?collection . - ?collection aocat:was_issued ?issued . - ?collection aocat:was_modified ?modified . - ?collection aocat:has_contributor ?contributor . - ?contributor aocat:has_name ?contributorName . - ?collection aocat:has_responsible ?legalResponsible . - ?legalResponsible aocat:has_name ?legalResponsibleName . - ?collection aocat:has_owner ?owner . - ?owner aocat:has_name ?ownerName . - ?collection aocat:has_publisher ?publisher . - ?publisher aocat:has_name ?publisherName . - ?collection aocat:has_access_rights ?accessRights . - ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . - ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . - optional { - ?contributor aocat:has_email ?contributorEmail . - } - optional { - ?legalResponsible aocat:has_email ?legalResponsibleEmail . - } - optional { - ?owner aocat:has_email ?ownerEmail . - } - optional { - ?publisher aocat:has_email ?publisherEmail . - } - } - } - { - select * - where { - ?record aocat:has_spatial_coverage ?spatialRegion . - ?spatialRegion aocat:has_place_name ?spatialPlaceName . - optional { - ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem . - } - optional { - ?record aocat:has_spatial_coverage ?spatialRegionPoint . - ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; - aocat:has_longitude ?spatialLocationLon . - } - optional { - ?record aocat:has_spatial_coverage ?spatialRegionBox . - ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; - aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ; - aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ; - aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ; - } - } - } - -} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index c979607..82b6154 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -26,7 +26,7 @@ public class GraphDbReaderAndESIndexTest { private RunSPARQLQueryService runSPQRLQuery; @Test - @Ignore +// @Ignore public void readAndIndexTest() throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); @@ -36,7 +36,7 @@ public class GraphDbReaderAndESIndexTest { runSPQRLQuery.setupConnection( appProps.getProperty("graphdb.writer.user"), appProps.getProperty("graphdb.writer.pwd"), - appProps.getProperty("repository.url"), + appProps.getProperty("graphdb.serverUrl"), appProps.getProperty("graphdb.repository")); ParseRDFJSON parseRDFJSON = new ParseRDFJSON(); parseRDFJSON.setCatalogEntryJsonPath(appProps.getProperty("catalog.entry.path")); @@ -48,12 +48,11 @@ public class GraphDbReaderAndESIndexTest { appProps.getProperty("exclude.predicates"), appProps.getProperty("class.map.specifications") ); - resourceManager.init(); runSPQRLQuery.setResourceManager(resourceManager); BulkUpload bulkUpload = new BulkUpload(); - bulkUpload.init(); + bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname")); runSPQRLQuery.setBulkUpload(bulkUpload); - String recordId = ""; + String recordId = ""; String datasource = "ads"; String collectionId = "271"; List recordIds = Arrays.asList(recordId); @@ -69,14 +68,15 @@ public class GraphDbReaderAndESIndexTest { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); appProps.load(resource.getInputStream()); - + String datasource = "ads"; + String collectionId = "271"; runSPQRLQuery = new RunSPARQLQueryService(); runSPQRLQuery.setupConnection( appProps.getProperty("graphdb.writer.user"), appProps.getProperty("graphdb.writer.pwd"), appProps.getProperty("repository.url"), appProps.getProperty("graphdb.repository")); - runSPQRLQuery.selectRecordIds(); + runSPQRLQuery.selectRecordIds(datasource, collectionId); } @Test diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java new file mode 100644 index 0000000..7704254 --- /dev/null +++ b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java @@ -0,0 +1,155 @@ +package eu.dnetlib.ariadneplus.workflows.nodes; + +import com.google.common.collect.Lists; +import eu.dnetlib.enabling.resultset.client.ResultSetClient; +import eu.dnetlib.msro.workflows.graph.Arc; +import eu.dnetlib.msro.workflows.nodes.AsyncJobNode; +import eu.dnetlib.msro.workflows.procs.Env; +import eu.dnetlib.msro.workflows.procs.Token; +import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider; +import eu.dnetlib.msro.workflows.util.WorkflowsConstants; +import eu.dnetlib.rmi.common.ResultSet; +import eu.dnetlib.rmi.manager.MSROException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.List; + + +public class IndexOnESJobNode extends AsyncJobNode { + + private static final Log log = LogFactory.getLog(IndexOnESJobNode.class); + + private String eprParam; + + @Autowired + private ResultSetClient resultSetClient; + + private String publisherEndpoint; + private String datasourceInterface; + private String datasource; + + //for parallel requests to the publisher endpoint + private int nThreads = 5; + + @Override + protected String execute(final Env env) throws Exception { + + int statusCode = -1; + String indexOnESResult = "noResult"; + log.info("Publisher endpoint: " + getPublisherEndpoint()); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(nThreads); + CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build(); + + log.info("IndexOnES endpoint: " + getIndexOnESEndpoint()); + CloseableHttpResponse responsePOST = null; + try { + HttpPost post = new HttpPost(getIndexOnESEndpoint()); + List params = Lists.newArrayList(); + String[] splits = getDatasourceInterface().split("::"); + String datasource = splits[2]; + String collectionId = splits[3]; + params.add(new BasicNameValuePair("datasource", datasource)); + params.add(new BasicNameValuePair("collectionId", collectionId)); + UrlEncodedFormEntity ent = new UrlEncodedFormEntity(params, "UTF-8"); + post.setEntity(ent); + log.info("Calling IndexOnES endpoint with params: "+getDatasource()+" "+getDatasourceInterface()); + responsePOST = client.execute(post); + statusCode = responsePOST.getStatusLine().getStatusCode(); + switch (statusCode) { + case 200: + log.info("index on ES completed"); + break; + default: + log.error("error indexing on ES " + responsePOST.getStatusLine().getStatusCode() + ": " + responsePOST.getStatusLine().getReasonPhrase()); + break; + } + } catch (ConnectException ce) { + log.error(ce); + throw new MSROException("Unable to connect to Publisher endpoint" + getIndexOnESEndpoint()); + } + catch (IOException e) { + log.error(e); + throw new MSROException("IO Error" + getIndexOnESEndpoint()); + } + finally{ + if(responsePOST != null) responsePOST.close(); + client.close(); + cm.shutdown(); + } + + env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "statusCode", Integer.toString(statusCode)); + env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "enrichResult", indexOnESResult); + + log.info(indexOnESResult); + if (statusCode!=200) { + throw new MSROException("Error from Publisher endpoint [ status code: " + statusCode + " ]"); + } + + return Arc.DEFAULT_ARC; + } + + public String getPublisherEndpoint() { + return publisherEndpoint; + } + + private String getIndexOnESEndpoint() { + return publisherEndpoint.concat("/indexOnES"); + } + + public void setPublisherEndpoint(final String publisherEndpoint) { + this.publisherEndpoint = publisherEndpoint; + } + + public ResultSetClient getResultSetClient() { + return resultSetClient; + } + + public void setResultSetClient(final ResultSetClient resultSetClient) { + this.resultSetClient = resultSetClient; + } + + public String getEprParam() { + return eprParam; + } + + public void setEprParam(String eprParam) { + this.eprParam = eprParam; + } + + public String getDatasourceInterface() { + return datasourceInterface; + } + + + public void setDatasourceInterface(String datasourceInterface) { + this.datasourceInterface = datasourceInterface; + } + + @Override + protected void beforeStart(Token token) { + token.setProgressProvider(new ResultsetProgressProvider(token.getEnv().getAttribute(getEprParam(), ResultSet.class), this.resultSetClient)); + } + + public String getDatasource() { + return datasource; + } + + public void setDatasource(String datasource) { + this.datasource = datasource; + } + +} diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml index cca0863..a3bb235 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml @@ -17,4 +17,6 @@ + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st new file mode 100644 index 0000000..4ced6d1 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st @@ -0,0 +1,54 @@ + + +
+ + + + + +
+ + $name$ + $desc$ + + aggregator + $priority$ + + + + + http://localhost:8080/ariadneplus/publish + + + + Index on Elastic Search all records of a collection from GraphDB + + + + + + + + + + + + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml new file mode 100644 index 0000000..ef24ed1 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml @@ -0,0 +1,32 @@ + +
+ + + + + +
+ + + + + + + http://localhost:8281/ariadneplus-graphdb + + + + Index on Elastic Search all records of a collection from GraphDB + + + + + + + + + + + + + \ No newline at end of file diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml new file mode 100644 index 0000000..668db14 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml @@ -0,0 +1,60 @@ + + +
+ + + + + +
+ + Index On ES + Index on Elastic Search all records of a collection from GraphDB + + IndexOnES + Content Provider + + REPO_HI + 20 + + + + + Verify if DS is pending + + + + + + + + + + + Validate DS + + + + + + + Create Workflow + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + +