D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

fix for from and until values on temporal and native period fields

This commit is contained in:
Enrico Ottonello 2021-06-21 15:20:39 +02:00
parent 54b76c79a0
commit 41896d312b
5 changed files with 57 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -1,15 +1,12 @@
package eu.dnetlib.ariadneplus.elasticsearch;
import eu.dnetlib.ariadneplus.elasticsearch.model.*;
import eu.dnetlib.ariadneplus.elasticsearch.model.AriadnePlusEntry;
import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpHost;
import org.apache.lucene.spatial3d.geom.GeoPoint;
import org.apache.lucene.spatial3d.geom.GeoPolygon;
import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
import org.apache.lucene.spatial3d.geom.PlanetModel;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
@ -17,18 +14,17 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
import org.elasticsearch.common.geo.builders.PolygonBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.geometry.Polygon;
import org.locationtech.jts.geom.Coordinate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.io.OutputStream;
import java.util.*;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Service
@ -70,9 +66,9 @@ public class BulkUpload {
AriadnePlusEntry ace = ((AriadnePlusEntry) next);
if (isCollection) {
ace.setResourceType("collection");
if (ace.getSpatial()==null) {
ace.setSpatial(Arrays.asList(new Spatial()));
}
// if (ace.getSpatial()==null) {
// ace.setSpatial(Arrays.asList(new Spatial()));
// }
}
else {
ace.setResourceType("dataset");
@ -162,9 +158,9 @@ public class BulkUpload {
// ace.getSpatial().removeIf(s -> (s.getPlaceName()!=null&&s.getPlaceName().equals("Name not provided")&&Objects.isNull(s.getLocation())));
// }
}
else {
ace.setSpatial(Arrays.asList(new Spatial()));
}
// else {
// ace.setSpatial(Arrays.asList(new Spatial()));
// }
}
// if (ace.getTemporal()!=null) {

4
dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties
View File

@ -61,7 +61,7 @@ class.map.specifications={\
"NativePeriod": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/from": {\
"https://www.ariadne-infrastructure.eu/property/nativeFrom": {\
"class_field": "From",\
"substring": "no",\
"element_type": "java.lang.String"\
@ -71,7 +71,7 @@ class.map.specifications={\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/until": {\
"https://www.ariadne-infrastructure.eu/property/nativeUntil": {\
"class_field": "Until",\
"substring": "no",\
"element_type": "java.lang.String"\

17
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql
View File

@ -49,8 +49,8 @@ CONSTRUCT {
?temporal aoprop:uri ?temporal .
%record aoprop:temporalNative ?temporalNative .
?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName .
?temporalNative aoprop:from ?temporalNativeFrom .
?temporalNative aoprop:until ?temporalNativeUntil .
?temporalNative aoprop:nativeFrom ?temporalNativeFrom .
?temporalNative aoprop:nativeUntil ?temporalNativeUntil .
%record aoprop:archeologicalResourceType ?archeologicalResourceType .
?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName .
%record aoprop:resourceType ?resourceType .
@ -113,19 +113,22 @@ where {
graph <https://ariadne-infrastructure.eu/api_________::ariadne_plus::%datasource::%collectionId> {
%record aocat:has_temporal_coverage ?temporalNative .
?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName .
}
graph <https://ariadne-infrastructure.eu/ariadneplus::%datasource::periodoplus> {
optional {
?temporalNative aocat:from ?temporalNativeFrom .
?temporalNative aocat:until ?temporalNativeUntil .
}
}
}
graph <https://ariadne-infrastructure.eu/api_________::ariadne_plus::%datasource::periodoplus> {
}
}
optional {
graph <https://ariadne-infrastructure.eu/api_________::ariadne_plus::%datasource::%collectionId> {
%record aocat:has_temporal_coverage ?temporal .
}
graph <https://ariadne-infrastructure.eu/ariadneplus::%datasource::periodoplus> {
?temporal aocat:has_period ?label .
graph <https://ariadne-infrastructure.eu/api_________::ariadne_plus::%datasource::periodo> {
graph <https://ariadne-infrastructure.eu/ariadneplus::%datasource::periodo> {
?label skos:prefLabel ?temporalPeriodName .
}
optional {
@ -134,8 +137,6 @@ where {
}
}
}
}
}
graph <https://ariadne-infrastructure.eu/api_________::ariadne_plus::%datasource::%collectionId> {
{

6
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql
View File

@ -53,8 +53,8 @@ CONSTRUCT {
?temporal aoprop:uri ?temporal .
%record aoprop:temporalNative ?temporalNative .
?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName .
?temporalNative aoprop:from ?temporalNativeFrom .
?temporalNative aoprop:until ?temporalNativeUntil .
?temporalNative aoprop:nativeFrom ?temporalNativeFrom .
?temporalNative aoprop:nativeUntil ?temporalNativeUntil .
%record aoprop:archeologicalResourceType ?archeologicalResourceType .
?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName .
%record aoprop:resourceType ?resourceType .
@ -100,8 +100,6 @@ where {
%record aocat:has_temporal_coverage ?temporalNative .
?temporalNative aocat:has_native_period ?nativeLabel .
?nativeLabel skos:prefLabel ?temporalNativePeriodName .
}
graph <https://ariadne-infrastructure.eu/ariadneplus::%datasource::periodoplus> {
optional {
?temporalNative aocat:from ?temporalNativeFrom .
?temporalNative aocat:until ?temporalNativeUntil .

48
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -32,7 +32,7 @@ public class GraphDbReaderAndESIndexTest {
private RunSPARQLQueryService runSPQRLQuery;
@Test
@Ignore
// @Ignore
public void uploadAMCRFieldworkTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA";
@ -42,7 +42,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadAMCRDocumentTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/AMCR/FC59581D-DC3A-31DA-922A-98DE764F3D76";
@ -52,7 +52,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadAMCRSiteTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/3C7EC936-A7CA-3720-B3DC-413A25754FD4";
@ -62,7 +62,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadAMCRIndividualFindTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/98D717C4-410F-35C6-8072-FABA7686B4A3";
@ -81,7 +81,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadADSRecordWithNativeFromUntilTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF";
@ -91,7 +91,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA";
@ -101,7 +101,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadZbivaRecordSpatialTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479";
@ -111,7 +111,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadHNMCollectionSpatialTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5";
@ -121,10 +121,10 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadHNMCollectionTemporalTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/0F1AF07E-CE63-342C-8E95-11FF1C024BD5";
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10";
String datasource = "hnm";
String collectionId = "hnmad";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
@ -141,8 +141,8 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
public void uploadDansNoSpatialTest() throws Exception {
// @Ignore
public void uploadDansSpatialTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD";
String datasource = "dans";
@ -151,8 +151,8 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
public void uploadDansFieldworkReportTest() throws Exception {
// @Ignore
public void uploadDansNativePeriodTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B";
String datasource = "dans";
@ -161,7 +161,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
public void uploadDansPolygonTest() throws Exception {
public void uploadDansTemporalPolygonTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B";
String datasource = "dans";
@ -170,7 +170,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadSndRockartTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A";
@ -180,7 +180,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadNIAMTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1";
@ -190,7 +190,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadADS398Test() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73";
@ -200,7 +200,7 @@ public class GraphDbReaderAndESIndexTest {
}
@Test
@Ignore
// @Ignore
public void uploadROADTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FC70B370-C489-31C5-B1D4-339CFD28CF2B";
@ -209,6 +209,16 @@ public class GraphDbReaderAndESIndexTest {
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
@Test
// @Ignore
public void uploadADS328NativeFromUntilTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559";
String datasource = "ads";
String collectionId = "328";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
final ClassPathResource resource = new ClassPathResource("application.properties");
Properties appProps = new Properties();