AriadnePlus/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java

/**
 * 
 */
package eu.dnetlib.ariadneplus;

import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import org.apache.commons.io.IOUtils;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.RepositoryResult;
import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.core.io.ClassPathResource;

import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;

/**
 * @author enrico.ottonello
 *
 */
@Ignore
public class GraphDbReaderAndESIndexTest {

    private RunSPARQLQueryService runSPQRLQuery;

    @Test
    public void uploadAMCRFieldworkTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA";
        String datasource = "amcr";
        String collectionId = "oai";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadAMCRFieldworkEventOnlyPeriodoTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/849F814F-5892-3408-AD5E-904938B4492A";
        String datasource = "amcr";
        String collectionId = "oai";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadAMCRDocumentTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/FC59581D-DC3A-31DA-922A-98DE764F3D76";
        String datasource = "amcr";
        String collectionId = "oai";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadAMCRSiteTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/3C7EC936-A7CA-3720-B3DC-413A25754FD4";
        String datasource = "amcr";
        String collectionId = "oai";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadAMCRIndividualFindTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AIS%20CR/98D717C4-410F-35C6-8072-FABA7686B4A3";
        String datasource = "amcr";
        String collectionId = "oai";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadADSRecordWithNativeFromUntilTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF";
        String datasource = "ads";
        String collectionId = "1970";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA";
        String datasource = "ads";
        String collectionId = "304";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadZbivaRecordSpatialTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479";
        String datasource = "zrc_zbiva";
        String collectionId = "sites";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadHNMCollectionSpatialTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5";
        String datasource = "hnm";
        String collectionId = "hnmad";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadHNMCollectionTemporalTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10";
        String datasource = "hnm";
        String collectionId = "hnmad";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadADSArchivesBoundingBoxTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/90D1C95D-E249-3E74-92D9-B58FDF690CC7";
        String datasource = "ads";
        String collectionId = "archives";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDansSpatialTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD";
        String datasource = "dans";
        String collectionId = "easy";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDansNativePeriodTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B";
        String datasource = "dans";
        String collectionId = "easy";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDansTemporalPolygonTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B";
        String datasource = "dans";
        String collectionId = "easy";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadSndRockartTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A";
        String datasource = "snd";
        String collectionId = "rockart";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadNIAMTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1";
        String datasource = "niam";
        String collectionId = "amb";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadADS398Test() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73";
        String datasource = "ads";
        String collectionId = "398";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadROADAfterFullReloadTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E859C58E-0307-3D7C-BCAB-45483DEA917C";
        String datasource = "road";
        String collectionId = "sites";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadADS328NativeFromUntilTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559";
        String datasource = "ads";
        String collectionId = "328";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadFastiMultipleAriadneSubjectTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/FE740BC7-2D9E-3644-A88E-3E8E5A5EA95F";
        String datasource = "fasti";
        String collectionId = "fieldwork";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadSNDArchivePolygonTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/A7A2588A-1FE7-3FDD-AD66-C84EACE1860E";
        String datasource = "snd";
        String collectionId = "zip";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadFastiTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/532D492C-B141-30D0-886A-1E69C2C20474";
        String datasource = "fasti";
        String collectionId = "fieldwork";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDansClosedAccessTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D1C1D6AD-320D-3616-B75D-F2E2C3DA7F3E";
        String datasource = "dans";
        String collectionId = "easy";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDansRestrictedAccessTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/221AAEC8-CC9B-310C-B800-0DFE34850E0B";
        String datasource = "dans";
        String collectionId = "easy";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDansOpenAccessTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/B251A0FE-EFE0-341B-A8B3-E0813CC82856";
        String datasource = "dans";
        String collectionId = "easy";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadConicetCollectionTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IDACOR%20%28CONICET%2FUNC%29/8B238475-2BA7-3036-88C0-968E9A642E64";
        String datasource = "conicet";
        String collectionId = "CollectionInfo";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadConicetRecordTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CONICET/0CD6BF2E-4AB7-39B6-937E-7DD59092DD3F";
        String datasource = "conicet";
        String collectionId = "fieldwork";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadNaraTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/001CA559-D688-31FB-AB48-45CC8DB1E355";
        String datasource = "nara";
        String collectionId = "fieldworkreport";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadINFNTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/INFN/5AE70226-82A9-3B06-9E25-8FB71B64C51C";
        String datasource = "infn";
        String collectionId = "firenze";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadADSGreyLiteratureOver300TriplesTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/35AEC455-6CB5-31BB-B73E-30959FF3C4FE";
        String datasource = "ads";
        String collectionId = "1093";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDimeBoundingBoxTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DIME/0E5FB5C7-6FCA-36AD-BEE5-03C9AFF393CD";
        String datasource = "aarhusdime";
        String collectionId = "oai";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadNormalizerTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DCDF395A-2CC8-3FC2-B9A5-5A924090DF10";
        String datasource = "ads";
        String collectionId = "3";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadNormalizerTest2() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/A0D9DC1F-2C62-3423-BAE6-4248D5072585";
        String datasource = "ads";
        String collectionId = "321";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadNormalizerAllUppercaseTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/E644B5AD-3DC2-3FEC-A070-B7FD434B0E90";
        String datasource = "ads";
        String collectionId = "304";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadNormalizerMinusTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/9F96015D-84FA-3BDE-AE3C-90CF3CC7CEDB";
        String datasource = "ads";
        String collectionId = "1788";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadCeniehTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0F55399D-8001-36E6-8D52-0C32981CAD7E";
        String datasource = "cenieh";
        String collectionId = "cir";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadDoliaTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/F0E19AC1-AE02-3BE5-AF78-D9EEC1670AC1";
        String datasource = "dolia";
        String collectionId = "fieldworkreport";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
        recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/28630596-6EFF-38D7-91C7-8A6134CBD91B";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void uploadIAVPFastCatCollectionWasCreatedTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/IAVP/E60F992F-C34D-39B5-99AD-406BA27B090B";
        String datasource = "iavp";
        String collectionId = "collection";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void collectionInCollectionTest() throws Exception {
        boolean isRecord = false;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/THANADOS/3E6614E4-EA97-3E7D-BA72-B0343A63FA39";
        String datasource = "thanados";
        String collectionId = "test";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    @Test
    public void recordInCollectionTest() throws Exception {
        boolean isRecord = true;
        String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/THANADOS/46FD94FF-0C91-369A-A64F-E29B7C9A40D9";
        String datasource = "thanados";
        String collectionId = "test";
        readAndIndexTest(isRecord, recordId, datasource, collectionId);
    }

    private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
        final ClassPathResource resource = new ClassPathResource("application.properties");
        Properties appProps = new Properties();
        appProps.load(resource.getInputStream());

        runSPQRLQuery = new RunSPARQLQueryService();
        runSPQRLQuery.setupReadOnlyConnection(
                appProps.getProperty("graphdb.serverUrl"),
                appProps.getProperty("graphdb.repository"));
        ParseRDFJSON parseRDFJSON = new ParseRDFJSON();
        parseRDFJSON.setCatalogEntryJsonPath(appProps.getProperty("catalog.entry.path"));
        parseRDFJSON.setCatalogEntryCollectionJsonPath(appProps.getProperty("catalog.entry.collection.path"));
        runSPQRLQuery.setParser(parseRDFJSON);
        ResourceManager resourceManager = new ResourceManager();
        resourceManager.setup(
                appProps.getProperty("type.path"),
                appProps.getProperty("general.classpath"),
                appProps.getProperty("exclude.predicates"),
                appProps.getProperty("class.map.specifications")
        );
    	runSPQRLQuery.setResourceManager(resourceManager);
        BulkUpload bulkUpload = new BulkUpload();
        bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname"));
    	runSPQRLQuery.setBulkUpload(bulkUpload);
        final ClassPathResource queryTemplateResource;
        if (isRecord) {
            queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
        }
        else {
            queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql");
        }
        List<String> recordIds = Arrays.asList(recordId);
        String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
        String report = "";
        if (isRecord) {
            report = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
        }
        else {
            report =  runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, true);
        }
        System.out.println(report);
        if (!report.contains("Total errors: 0")) {
            throw new Exception("Error found - check report value.");
        }
    }

    @Test
    @Ignore
    public void selectRecordsTest() throws Exception {
        final ClassPathResource resource = new ClassPathResource("application.properties");
        Properties appProps = new Properties();
        appProps.load(resource.getInputStream());
        String datasource = "ads";
        String collectionId = "271";
        runSPQRLQuery = new RunSPARQLQueryService();
        runSPQRLQuery.setupReadOnlyConnection(
                appProps.getProperty("graphdb.serverUrl"),
                appProps.getProperty("graphdb.repository"));
        runSPQRLQuery.selectRecordIds(datasource, collectionId);
    }

    @Test
    @Ignore
    public void loadQueryTest() throws Exception {
        final ClassPathResource resource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
        String queryTemplate = IOUtils.toString(resource.getInputStream(), StandardCharsets.UTF_8.name());
        System.out.println(queryTemplate);

    }

    @Test
    @Ignore
    public void httpsRequestTest() throws Exception {
//        RemoteRepositoryManager manager = new RemoteRepositoryManager("http://graphdb-test.ariadne.d4science.org:7200");
        RemoteRepositoryManager manager = new RemoteRepositoryManager("https://graphdb-test.ariadne.d4science.org");
        manager.init();
        manager.setUsernameAndPassword("writer", "******");
        Repository repository = manager.getRepository("ariadneplus-ts01");
        RepositoryConnection connection = repository.getConnection();
        RepositoryResult<Resource> contexts = connection.getContextIDs();
        if (contexts!=null) {
            System.out.println("##### Primo context "+contexts.next().stringValue());
        }
        else {
            System.out.println("No contexts ");
        }
        connection.close();
        repository.shutDown();
        manager.shutDown();
    }
}