From 9bdcd7ce9005ae560b3895c069301f9da2c5ed17 Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Thu, 7 Sep 2023 14:53:30 +0200 Subject: [PATCH] Updated mail template for wf notifications of ariadne --- .../FindMissingRecordsInIndexTest.java | 79 +++++++++++++++++++ .../GraphDbReaderAndESIndexTest.java | 12 +-- 2 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/FindMissingRecordsInIndexTest.java diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/FindMissingRecordsInIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/FindMissingRecordsInIndexTest.java new file mode 100644 index 0000000..34f06f5 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/FindMissingRecordsInIndexTest.java @@ -0,0 +1,79 @@ +package eu.dnetlib.ariadneplus; + +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import eu.dnetlib.ariadneplus.reader.utils.ESUtils; +import org.apache.http.HttpHost; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.springframework.core.io.ClassPathResource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Properties; + +@Ignore +public class FindMissingRecordsInIndexTest { + + private RunSPARQLQueryService runSPQRLQuery; + + private final static String STAGING_PROPERTIES = "application.properties"; + private final static String PROD_PROPERTIES = "application-prod-DO-NOT-COMMIT.properties"; + + private RestHighLevelClient client; + private Properties appProps; + @Before + public void setUp() throws IOException { + final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES); + appProps = new Properties(); + appProps.load(resource.getInputStream()); + runSPQRLQuery = new RunSPARQLQueryService(); + runSPQRLQuery.setupReadOnlyConnection( + appProps.getProperty("graphdb.serverUrl"), + appProps.getProperty("graphdb.repository")); + client = new RestHighLevelClient( + RestClient.builder( + new HttpHost(appProps.getProperty("elasticsearch.hostname"), 9200, "http"))); + } + + @After + public void tearDown() throws IOException { + client.close(); + } + @Test + public void findMissingRecordsTest() throws NoSuchAlgorithmException, IOException { + String datasource = "ads"; + String collectionId = "archives"; + List uris = runSPQRLQuery.selectRecordIds(datasource, collectionId); + System.out.println("Got list of ids, they are "+uris.size()); + for(String uri : uris){ + //compute the md5 and query the index to check if there is a record with that uri. If not, print it + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] encodedhash = digest.digest( + uri.getBytes(StandardCharsets.UTF_8)); + String idES = ESUtils.bytesToHex(encodedhash); + // High level API + GetRequest getRequest = new GetRequest( + appProps.getProperty("elasticsearch.indexname"), + idES); + getRequest.fetchSourceContext(new FetchSourceContext(false)); + getRequest.storedFields("_none_"); + + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); + if(!exists) System.out.println(uri); + } + System.out.println("Done"); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index ea3c5e8..c4327b7 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -47,9 +47,9 @@ public class GraphDbReaderAndESIndexTest { public void loadToStaging() throws Exception { //String uri = "https://arche.acdh.oeaw.ac.at/api/255841"; - String uri = "https://ariadne-infrastructure.eu/aocat/Resource/CENIEH/013A55AB-432D-3B51-B3A6-B6AFE343054B"; - String datasource = "cenieh"; - String apiId = "cir"; + String uri = "https://ariadne-infrastructure.eu/aocat/Resource/ADS/6CBA2A00-01DE-315B-934C-6ED74C3BC6DC"; + String datasource = "ads"; + String apiId = "archives"; readAndIndexTest(true, uri, datasource, apiId); } @@ -684,11 +684,11 @@ public class GraphDbReaderAndESIndexTest { } private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { - readAndIndex( isRecord, recordId, datasource, collectionId, "application.properties"); + readAndIndex( isRecord, recordId, datasource, collectionId, STAGING_PROPERTIES); } private void readAndIndexProd(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { - readAndIndex( isRecord, recordId, datasource, collectionId, "application-prod-DO-NOT-COMMIT.properties"); + readAndIndex( isRecord, recordId, datasource, collectionId, PROD_PROPERTIES); } private void readAndIndex(boolean isRecord, String recordId, String datasource, String collectionId, String propertyFile) throws Exception { @@ -739,7 +739,7 @@ public class GraphDbReaderAndESIndexTest { @Test @Ignore public void selectRecordsTest() throws Exception { - final ClassPathResource resource = new ClassPathResource("application.properties"); + final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES); Properties appProps = new Properties(); appProps.load(resource.getInputStream()); String datasource = "ads";