806 lines
34 KiB
Java
806 lines
34 KiB
Java
/**
|
|
*
|
|
*/
|
|
package eu.dnetlib.ariadneplus;
|
|
|
|
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
|
|
import eu.dnetlib.ariadneplus.reader.ResourceManager;
|
|
import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService;
|
|
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
|
|
import org.apache.commons.compress.utils.Lists;
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.eclipse.rdf4j.model.Resource;
|
|
import org.eclipse.rdf4j.repository.Repository;
|
|
import org.eclipse.rdf4j.repository.RepositoryConnection;
|
|
import org.eclipse.rdf4j.repository.RepositoryResult;
|
|
import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager;
|
|
import org.junit.Ignore;
|
|
import org.junit.Test;
|
|
import org.springframework.core.io.ClassPathResource;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.InputStreamReader;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Paths;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.List;
|
|
import java.util.Properties;
|
|
import java.util.stream.Collectors;
|
|
import java.util.stream.Stream;
|
|
|
|
/**
|
|
* @author enrico.ottonello
|
|
*
|
|
*/
|
|
@Ignore
|
|
public class GraphDbReaderAndESIndexTest {
|
|
|
|
private RunSPARQLQueryService runSPQRLQuery;
|
|
|
|
private final static String STAGING_PROPERTIES = "application.properties";
|
|
private final static String PROD_PROPERTIES = "application-prod-DO-NOT-COMMIT.properties";
|
|
|
|
@Test
|
|
public void loadToPublic() throws Exception {
|
|
String uri = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B";
|
|
String datasource = "dans";
|
|
String apiId = "easy";
|
|
readAndIndexProd(true, uri, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void loadListToPublic() throws Exception {
|
|
String datasource = "dans";
|
|
String apiId = "easy";
|
|
String listClasspath = "eu/dnetlib/ariadneplus/reader/resources_uris.txt";
|
|
final ClassPathResource resource = new ClassPathResource(listClasspath);
|
|
try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream()))) {
|
|
String line;
|
|
while ((line = reader.readLine()) != null) {
|
|
readAndIndexProd(true, line, datasource, apiId);
|
|
System.out.println(line);
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test
|
|
public void loadToStaging() throws Exception {
|
|
|
|
//String uri = "https://arche.acdh.oeaw.ac.at/api/255841";
|
|
String uri = "https://ariadne-infrastructure.eu/aocat/Resource/ADS/6CBA2A00-01DE-315B-934C-6ED74C3BC6DC";
|
|
String datasource = "ads";
|
|
String apiId = "archives";
|
|
readAndIndexTest(true, uri, datasource, apiId);
|
|
|
|
}
|
|
|
|
@Test
|
|
public void testRoceehPeriodO() throws Exception {
|
|
|
|
String record ="https://ariadne-infrastructure.eu/aocat/Resource/4F9B4DC6-F236-3CDA-8C3B-B20B5D17DD6E";
|
|
String datasource = "road";
|
|
String apiId = "sites";
|
|
readAndIndexTest(true, record, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void testCENIEHPeriodO() throws Exception {
|
|
|
|
String record ="https://ariadne-infrastructure.eu/aocat/Resource/E8AA2F30-98B7-3668-A0EC-339EA9EF652D";
|
|
String datasource = "cenieh";
|
|
String apiId = "sediments";
|
|
readAndIndexTest(true, record, datasource, apiId);
|
|
}
|
|
|
|
|
|
@Test
|
|
public void testWithNotProvidedARIADNESubject() throws Exception {
|
|
String record ="https://ariadne-infrastructure.eu/aocat/Resource/ADS/28986CB2-37B1-31F8-98B5-E86F9BD98946";
|
|
String datasource = "ads";
|
|
String apiId = "archives";
|
|
readAndIndexTest(true, record, datasource, apiId);
|
|
|
|
}
|
|
@Test
|
|
public void testADS1093() throws Exception {
|
|
String coll ="https://ariadne-infrastructure.eu/aocat/Collection/ADS/5910411B-ED97-364E-8D28-6024558AA14B";
|
|
String record1 ="https://ariadne-infrastructure.eu/aocat/Resource/ADS/D182802E-592C-3999-9DB2-155F25E356E2";
|
|
String record2 ="https://ariadne-infrastructure.eu/aocat/Resource/ADS/15814239-CCF1-317C-AEBA-49EEBB582611";
|
|
String record3 ="https://ariadne-infrastructure.eu/aocat/Resource/ADS/3210FDBD-97DC-3B12-9EFF-3FDAB7AC11CE";
|
|
//different access right and policy
|
|
String record4 = "https://ariadne-infrastructure.eu/aocat/Resource/ADS/B0FC7870-2E80-325C-9BE2-1F05C0E70610";
|
|
String datasource = "ads";
|
|
String apiId = "1093";
|
|
readAndIndexTest(false, coll, datasource, apiId);
|
|
readAndIndexTest(true, record1, datasource, apiId);
|
|
readAndIndexTest(true, record2, datasource, apiId);
|
|
readAndIndexTest(true, record3, datasource, apiId);
|
|
readAndIndexTest(true, record4, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void testADS276() throws Exception {
|
|
String coll ="https://ariadne-infrastructure.eu/aocat/Collection/ADS/B1E1FD6B-42C6-35B6-82E8-06E0082CCC71";
|
|
String record1 ="https://ariadne-infrastructure.eu/aocat/Resource/HERoNI/FB819062-C53C-3164-9A42-AEA6FD127575";
|
|
String record2 ="https://ariadne-infrastructure.eu/aocat/Resource/HERoNI/C460E0A9-F064-3B9D-8905-00CF08E4772D";
|
|
String record3 ="https://ariadne-infrastructure.eu/aocat/Resource/HERoNI/E13FAB63-4870-3C91-9138-21E0E3999F54";
|
|
String datasource = "ads";
|
|
String apiId = "276";
|
|
readAndIndexTest(false, coll, datasource, apiId);
|
|
readAndIndexTest(true, record1, datasource, apiId);
|
|
readAndIndexTest(true, record2, datasource, apiId);
|
|
readAndIndexTest(true, record3, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void testADS1093Record() throws Exception {
|
|
String id ="https://ariadne-infrastructure.eu/aocat/Resource/ADS/D182802E-592C-3999-9DB2-155F25E356E2";
|
|
String datasource = "ads";
|
|
String apiId = "1093";
|
|
readAndIndexTest(true, id, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void testADS276NorthIrelandRecord() throws Exception {
|
|
String id ="https://ariadne-infrastructure.eu/aocat/Resource/HERoNI/FB819062-C53C-3164-9A42-AEA6FD127575";
|
|
String datasource = "ads";
|
|
String apiId = "276";
|
|
readAndIndexTest(true, id, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void testADS276WithOtherIds() throws Exception {
|
|
String id = "https://ariadne-infrastructure.eu/aocat/Resource/HERoNI/A59B2AE2-57BF-337D-BF0D-7DA8B35AECE8";
|
|
String datasource = "ads";
|
|
String apiId = "276";
|
|
readAndIndexTest(true, id, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void testSNDWithOtherId() throws Exception {
|
|
String id = "https://ariadne-infrastructure.eu/aocat/Collection/62D27ED3-20F3-355C-A7AD-3F3F4BC7457C";
|
|
String datasource = "snd";
|
|
String apiId = "zip";
|
|
readAndIndexTest(false, id, datasource, apiId);
|
|
}
|
|
|
|
|
|
@Test
|
|
public void testADS() throws Exception {
|
|
String id = "https://ariadne-infrastructure.eu/aocat/Resource/ADS/90D1C95D-E249-3E74-92D9-B58FDF690CC7";
|
|
String datasource = "ads";
|
|
String apiId = "archives";
|
|
readAndIndexTest(true, id, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadADSArchivesBoundingBoxTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/90D1C95D-E249-3E74-92D9-B58FDF690CC7";
|
|
String datasource = "ads";
|
|
String collectionId = "archives";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
|
|
@Test
|
|
public void testFindSampoCollection() throws Exception {
|
|
String collId = "https://ariadne-infrastructure.eu/aocat/Collection/FHA/86DE51E7-1A20-32E0-96CA-119DEA2220E4";
|
|
String datasource = "findsampo";
|
|
String apiId = "finds";
|
|
readAndIndexTest(false, collId, datasource, apiId);
|
|
}
|
|
@Test
|
|
public void testADSFieldworkReport1093() throws Exception {
|
|
String collId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/5910411B-ED97-364E-8D28-6024558AA14B";
|
|
String datasource = "ads";
|
|
String apiId = "1093";
|
|
readAndIndexTest(false, collId, datasource, apiId);
|
|
}
|
|
|
|
@Test
|
|
public void AMCRForMariaTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/30A1E3C2-2856-3D44-A98D-078BAEE87BBA";
|
|
String datasource = "amcr";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
@Test
|
|
public void uploadAMCRFieldworkTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA";
|
|
String datasource = "amcr";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadAMCRFieldworkEventOnlyPeriodoTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/849F814F-5892-3408-AD5E-904938B4492A";
|
|
String datasource = "amcr";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadAMCRDocumentTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/FC59581D-DC3A-31DA-922A-98DE764F3D76";
|
|
String datasource = "amcr";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadAMCRSiteTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/3C7EC936-A7CA-3720-B3DC-413A25754FD4";
|
|
String datasource = "amcr";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadAMCRIndividualFindTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/98D717C4-410F-35C6-8072-FABA7686B4A3";
|
|
String datasource = "amcr";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadADSRecordWithNativeFromUntilTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF";
|
|
String datasource = "ads";
|
|
String collectionId = "1970";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA";
|
|
String datasource = "ads";
|
|
String collectionId = "304";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadZbivaRecordSpatialTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479";
|
|
String datasource = "zrc_zbiva";
|
|
String collectionId = "sites";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadHNMCollectionSpatialTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5";
|
|
String datasource = "hnm";
|
|
String collectionId = "hnmad";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadHNMCollectionTemporalTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10";
|
|
String datasource = "hnm";
|
|
String collectionId = "hnmad";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
public void uploadDansSpatialTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD";
|
|
String datasource = "dans";
|
|
String collectionId = "easy";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDansNativePeriodTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B";
|
|
String datasource = "dans";
|
|
String collectionId = "easy";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDansTemporalPolygonTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B";
|
|
String datasource = "dans";
|
|
String collectionId = "easy";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadSndRockartTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A";
|
|
String datasource = "snd";
|
|
String collectionId = "rockart";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void SNDCollectionWithWKT() throws Exception {
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/206F1EC8-014F-3201-A1E3-4545C2CD1953";
|
|
String datasource = "snd";
|
|
String collectionId = "zip";
|
|
readAndIndexTest(false, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadNIAMTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1";
|
|
String datasource = "niam";
|
|
String collectionId = "amb";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadADS398Test() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73";
|
|
String datasource = "ads";
|
|
String collectionId = "398";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadROADAfterFullReloadTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E859C58E-0307-3D7C-BCAB-45483DEA917C";
|
|
String datasource = "road";
|
|
String collectionId = "sites";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadADS328NativeFromUntilTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559";
|
|
String datasource = "ads";
|
|
String collectionId = "328";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadFastiMultipleAriadneSubjectTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/FE740BC7-2D9E-3644-A88E-3E8E5A5EA95F";
|
|
String datasource = "fasti";
|
|
String collectionId = "fieldwork";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadSNDArchivePolygonTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/A7A2588A-1FE7-3FDD-AD66-C84EACE1860E";
|
|
String datasource = "snd";
|
|
String collectionId = "zip";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadFastiTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/532D492C-B141-30D0-886A-1E69C2C20474";
|
|
String datasource = "fasti";
|
|
String collectionId = "fieldwork";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDansClosedAccessTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D1C1D6AD-320D-3616-B75D-F2E2C3DA7F3E";
|
|
String datasource = "dans";
|
|
String collectionId = "easy";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDansRestrictedAccessTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/221AAEC8-CC9B-310C-B800-0DFE34850E0B";
|
|
String datasource = "dans";
|
|
String collectionId = "easy";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDansOpenAccessTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/B251A0FE-EFE0-341B-A8B3-E0813CC82856";
|
|
String datasource = "dans";
|
|
String collectionId = "easy";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadConicetCollectionTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IDACOR%20%28CONICET%2FUNC%29/8B238475-2BA7-3036-88C0-968E9A642E64";
|
|
String datasource = "conicet";
|
|
String collectionId = "CollectionInfo";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadConicetRecordTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CONICET/0CD6BF2E-4AB7-39B6-937E-7DD59092DD3F";
|
|
String datasource = "conicet";
|
|
String collectionId = "fieldwork";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadNaraTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/001CA559-D688-31FB-AB48-45CC8DB1E355";
|
|
String datasource = "nara";
|
|
String collectionId = "fieldworkreport";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadINFNTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/INFN/5AE70226-82A9-3B06-9E25-8FB71B64C51C";
|
|
String datasource = "infn";
|
|
String collectionId = "firenze";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadADSGreyLiteratureOver300TriplesTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/35AEC455-6CB5-31BB-B73E-30959FF3C4FE";
|
|
String datasource = "ads";
|
|
String collectionId = "1093";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDimeBoundingBoxTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DIME/0E5FB5C7-6FCA-36AD-BEE5-03C9AFF393CD";
|
|
String datasource = "aarhusdime";
|
|
String collectionId = "oai";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadNormalizerTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DCDF395A-2CC8-3FC2-B9A5-5A924090DF10";
|
|
String datasource = "ads";
|
|
String collectionId = "3";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadNormalizerTest2() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/A0D9DC1F-2C62-3423-BAE6-4248D5072585";
|
|
String datasource = "ads";
|
|
String collectionId = "321";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadNormalizerAllUppercaseTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E644B5AD-3DC2-3FEC-A070-B7FD434B0E90";
|
|
String datasource = "ads";
|
|
String collectionId = "304";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadNormalizerMinusTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/9F96015D-84FA-3BDE-AE3C-90CF3CC7CEDB";
|
|
String datasource = "ads";
|
|
String collectionId = "1788";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadCeniehTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0F55399D-8001-36E6-8D52-0C32981CAD7E";
|
|
String datasource = "cenieh";
|
|
String collectionId = "cir";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadDoliaTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/EEAA72F8-3C33-3211-A56B-296164B5A317";
|
|
String datasource = "dolia";
|
|
String collectionId = "full";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadIAVPFastCatCollectionWasCreatedTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IAVP/E60F992F-C34D-39B5-99AD-406BA27B090B";
|
|
String datasource = "iavp";
|
|
String collectionId = "collection";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void collectionInCollectionTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/THANADOS/3E6614E4-EA97-3E7D-BA72-B0343A63FA39";
|
|
String datasource = "thanados";
|
|
String collectionId = "api";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void collectionInCollectionWithGeoPointTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/THANADOS/6AA9483F-549C-3908-B7C0-1A5BAA521371";
|
|
String datasource = "thanados";
|
|
String collectionId = "api";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
public void recordInCollectionTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/THANADOS/46FD94FF-0C91-369A-A64F-E29B7C9A40D9";
|
|
String datasource = "thanados";
|
|
String collectionId = "test";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void thanadosCollectionTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/THANADOS/29D18DF9-09D8-39E7-B424-01FDE024F191";
|
|
String datasource = "thanados";
|
|
String collectionId = "api";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
|
|
@Test
|
|
public void uploadAdsScotlandAATTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/1039819E-583F-3856-8463-BB58523DB0F9";
|
|
String datasource = "ads";
|
|
String collectionId = "1";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadMibacTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/MiC/662BD48F-7313-3017-8040-14CFFE1A02EB";
|
|
String datasource = "mibac";
|
|
String collectionId = "test";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadBMPASTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/BMPAS/A561885D-2480-3794-82AD-A06B7462DA74";
|
|
String datasource = "bmpas";
|
|
String collectionId = "coins";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadINPTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/INPRAN/99C2A8A2-32A0-344D-9F41-FD8E7B2C21E3";
|
|
String datasource = "inp";
|
|
String collectionId = "sites";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadUBCollectionTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/UB/F334A035-7757-3787-9849-E38FCD6A1FF5";
|
|
String datasource = "ub";
|
|
String collectionId = "inscription";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadRGKTest() throws Exception {
|
|
boolean isRecord = true;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AFE_RGK/C591D12B-9DBF-397D-8E38-F56C674ECF0E";
|
|
String datasource = "afe_rgk";
|
|
String collectionId = "coins";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
public void uploadUKPoolTest() throws Exception {
|
|
boolean isRecord = false;
|
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/OEAW/90A72C82-BD31-3BAB-A512-5084ACD1C235";
|
|
String datasource = "oeaw";
|
|
String collectionId = "CollectionInfo";
|
|
readAndIndexTest(isRecord, recordId, datasource, collectionId);
|
|
}
|
|
|
|
private class CollectionData {
|
|
String URI_ID;
|
|
String id;
|
|
|
|
public CollectionData(String URI_ID, String id) {
|
|
this.URI_ID = URI_ID;
|
|
this.id = id;
|
|
}
|
|
}
|
|
|
|
// used only once to fix overwritten ADS collections
|
|
private void fixPublicADSCollections() throws Exception {
|
|
boolean isRecord = false;
|
|
String datasource = "ads";
|
|
List<CollectionData> collectionToFix = Arrays.asList(
|
|
new CollectionData("AAA81A6D-56F3-341C-BAF0-791C31BC7F73", "398"),
|
|
new CollectionData("B6AF6572-6CF5-3EEE-A6CD-778FD975903A", "1785"),
|
|
new CollectionData("61CC3C06-21FE-3983-93B7-7ADE3535C59A", "1787"),
|
|
new CollectionData("F8FAC6C9-8E3D-355C-83E5-B162FC005EC2", "1788"),
|
|
new CollectionData("C2BC1925-493E-3047-858E-A720403D1F68", "324"),
|
|
new CollectionData("55B8D842-F80A-3841-9A4B-36F714D438BC", "1054"),
|
|
new CollectionData("5DE1B42A-6A52-3164-A1F3-E17BD6A2D082", "272_event"),
|
|
new CollectionData("F350D267-D463-33DE-883B-3561AA523168", "304"),
|
|
new CollectionData("F31C1473-3527-3C56-9801-F833D3C26A70", "3"),
|
|
new CollectionData("DE1A5A67-F32D-318B-808B-4457B1EF329F", "1957"),
|
|
new CollectionData("BB5172FB-55D2-3E6F-B03B-78B9A4B595BA", "1970"),
|
|
new CollectionData("8F4CA5AD-2339-3FED-89BB-F97291DDDC99", "270"),
|
|
new CollectionData("D52279BD-4C39-3682-9BFB-9BC7D0CD9D7F", "397"),
|
|
new CollectionData("57DB9A00-5856-3120-BE2A-2FFBD253CA04", "328"),
|
|
new CollectionData("80BF2F63-5448-3C38-BF89-A38DBC4AFB3E", "292"),
|
|
new CollectionData("58114087-BD7B-3ECC-9B46-A48C8BB36F6C", "858"),
|
|
new CollectionData("E0B88506-0A29-392B-8FDF-E7D7FDA2412B", "321"),
|
|
new CollectionData("C27CDB2E-CC18-3028-96C4-A13C1BC46044", "367"),
|
|
new CollectionData("849B1C0F-4C5F-3D8C-9082-CA60DBB4F557", "271"),
|
|
new CollectionData("95EDB084-6566-39E6-AFC2-82CC3F7B70D2", "420_event"),
|
|
new CollectionData("F68EC4F0-C6DF-3013-B749-AF75A929A3EB", "4"),
|
|
new CollectionData("F18A0D06-2A86-334A-AA1E-046E32A34327", "836"),
|
|
new CollectionData("EFE699F4-F42D-36AC-8CA2-24AD5F0B9307", "1972")
|
|
);
|
|
|
|
collectionToFix.forEach(data -> {
|
|
String prefix = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/";
|
|
String collectionIdURI = prefix.concat(data.URI_ID);
|
|
try {
|
|
readAndIndexTest(isRecord, collectionIdURI, datasource, data.id);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
throw new RuntimeException(e);
|
|
}
|
|
});
|
|
}
|
|
|
|
private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
|
|
readAndIndex( isRecord, recordId, datasource, collectionId, STAGING_PROPERTIES);
|
|
}
|
|
|
|
private void readAndIndexProd(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
|
|
readAndIndex( isRecord, recordId, datasource, collectionId, PROD_PROPERTIES);
|
|
}
|
|
|
|
private void readAndIndex(boolean isRecord, String recordId, String datasource, String collectionId, String propertyFile) throws Exception {
|
|
final ClassPathResource resource = new ClassPathResource(propertyFile);
|
|
Properties appProps = new Properties();
|
|
appProps.load(resource.getInputStream());
|
|
runSPQRLQuery = new RunSPARQLQueryService();
|
|
runSPQRLQuery.setupReadOnlyConnection(
|
|
appProps.getProperty("graphdb.serverUrl"),
|
|
appProps.getProperty("graphdb.repository"));
|
|
ParseRDFJSON parseRDFJSON = new ParseRDFJSON();
|
|
parseRDFJSON.setCatalogEntryJsonPath(appProps.getProperty("catalog.entry.path"));
|
|
parseRDFJSON.setCatalogEntryCollectionJsonPath(appProps.getProperty("catalog.entry.collection.path"));
|
|
runSPQRLQuery.setParser(parseRDFJSON);
|
|
ResourceManager resourceManager = new ResourceManager();
|
|
resourceManager.setup(
|
|
appProps.getProperty("type.path"),
|
|
appProps.getProperty("general.classpath"),
|
|
appProps.getProperty("exclude.predicates"),
|
|
appProps.getProperty("class.map.specifications")
|
|
);
|
|
runSPQRLQuery.setResourceManager(resourceManager);
|
|
BulkUpload bulkUpload = new BulkUpload();
|
|
bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname"));
|
|
runSPQRLQuery.setBulkUpload(bulkUpload);
|
|
final ClassPathResource queryTemplateResource;
|
|
if (isRecord) {
|
|
queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
|
|
}
|
|
else {
|
|
queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql");
|
|
}
|
|
List<String> recordIds = Arrays.asList(recordId);
|
|
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
|
|
String report = "";
|
|
if (isRecord) {
|
|
report = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
|
|
}
|
|
else {
|
|
report = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, true);
|
|
}
|
|
System.out.println(report);
|
|
if (!report.contains("Total errors: 0")) {
|
|
throw new Exception("Error found - check report value.");
|
|
}
|
|
}
|
|
|
|
@Test
|
|
@Ignore
|
|
public void selectRecordsTest() throws Exception {
|
|
final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES);
|
|
Properties appProps = new Properties();
|
|
appProps.load(resource.getInputStream());
|
|
String datasource = "ads";
|
|
String collectionId = "271";
|
|
runSPQRLQuery = new RunSPARQLQueryService();
|
|
runSPQRLQuery.setupReadOnlyConnection(
|
|
appProps.getProperty("graphdb.serverUrl"),
|
|
appProps.getProperty("graphdb.repository"));
|
|
runSPQRLQuery.selectRecordIds(datasource, collectionId);
|
|
}
|
|
|
|
@Test
|
|
@Ignore
|
|
public void loadQueryTest() throws Exception {
|
|
final ClassPathResource resource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
|
|
String queryTemplate = IOUtils.toString(resource.getInputStream(), StandardCharsets.UTF_8.name());
|
|
System.out.println(queryTemplate);
|
|
|
|
}
|
|
|
|
@Test
|
|
@Ignore
|
|
public void httpsRequestTest() throws Exception {
|
|
// RemoteRepositoryManager manager = new RemoteRepositoryManager("http://graphdb-test.ariadne.d4science.org:7200");
|
|
RemoteRepositoryManager manager = new RemoteRepositoryManager("https://graphdb-test.ariadne.d4science.org");
|
|
manager.init();
|
|
manager.setUsernameAndPassword("writer", "******");
|
|
Repository repository = manager.getRepository("ariadneplus-ts01");
|
|
RepositoryConnection connection = repository.getConnection();
|
|
RepositoryResult<Resource> contexts = connection.getContextIDs();
|
|
if (contexts!=null) {
|
|
System.out.println("##### Primo context "+contexts.next().stringValue());
|
|
}
|
|
else {
|
|
System.out.println("No contexts ");
|
|
}
|
|
connection.close();
|
|
repository.shutDown();
|
|
manager.shutDown();
|
|
}
|
|
}
|