D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Updated mail template for wf notifications of ariadne

This commit is contained in:
Alessia Bardi 2023-09-07 14:53:30 +02:00
parent 1419401c9a
commit 9bdcd7ce90
2 changed files with 85 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/FindMissingRecordsInIndexTest.java Normal file
View File

@ -0,0 +1,79 @@
package eu.dnetlib.ariadneplus;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import eu.dnetlib.ariadneplus.reader.utils.ESUtils;
import org.apache.http.HttpHost;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.core.io.ClassPathResource;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Properties;
@Ignore
public class FindMissingRecordsInIndexTest {
private RunSPARQLQueryService runSPQRLQuery;
private final static String STAGING_PROPERTIES = "application.properties";
private final static String PROD_PROPERTIES = "application-prod-DO-NOT-COMMIT.properties";
private RestHighLevelClient client;
private Properties appProps;
@Before
public void setUp() throws IOException {
final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES);
appProps = new Properties();
appProps.load(resource.getInputStream());
runSPQRLQuery = new RunSPARQLQueryService();
runSPQRLQuery.setupReadOnlyConnection(
appProps.getProperty("graphdb.serverUrl"),
appProps.getProperty("graphdb.repository"));
client = new RestHighLevelClient(
RestClient.builder(
new HttpHost(appProps.getProperty("elasticsearch.hostname"), 9200, "http")));
}
@After
public void tearDown() throws IOException {
client.close();
}
@Test
public void findMissingRecordsTest() throws NoSuchAlgorithmException, IOException {
String datasource = "ads";
String collectionId = "archives";
List<String> uris = runSPQRLQuery.selectRecordIds(datasource, collectionId);
System.out.println("Got list of ids, they are "+uris.size());
for(String uri : uris){
//compute the md5 and query the index to check if there is a record with that uri. If not, print it
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] encodedhash = digest.digest(
uri.getBytes(StandardCharsets.UTF_8));
String idES = ESUtils.bytesToHex(encodedhash);
// High level API
GetRequest getRequest = new GetRequest(
appProps.getProperty("elasticsearch.indexname"),
idES);
getRequest.fetchSourceContext(new FetchSourceContext(false));
getRequest.storedFields("_none_");
boolean exists = client.exists(getRequest, RequestOptions.DEFAULT);
if(!exists) System.out.println(uri);
}
System.out.println("Done");
}
}

12
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -47,9 +47,9 @@ public class GraphDbReaderAndESIndexTest {
public void loadToStaging() throws Exception {
//String uri = "https://arche.acdh.oeaw.ac.at/api/255841";
String uri = "https://ariadne-infrastructure.eu/aocat/Resource/CENIEH/013A55AB-432D-3B51-B3A6-B6AFE343054B";
String datasource = "cenieh";
String apiId = "cir";
String uri = "https://ariadne-infrastructure.eu/aocat/Resource/ADS/6CBA2A00-01DE-315B-934C-6ED74C3BC6DC";
String datasource = "ads";
String apiId = "archives";
readAndIndexTest(true, uri, datasource, apiId);
}
@ -684,11 +684,11 @@ public class GraphDbReaderAndESIndexTest {
}
private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
readAndIndex( isRecord, recordId, datasource, collectionId, "application.properties");
readAndIndex( isRecord, recordId, datasource, collectionId, STAGING_PROPERTIES);
}
private void readAndIndexProd(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
readAndIndex( isRecord, recordId, datasource, collectionId, "application-prod-DO-NOT-COMMIT.properties");
readAndIndex( isRecord, recordId, datasource, collectionId, PROD_PROPERTIES);
}
private void readAndIndex(boolean isRecord, String recordId, String datasource, String collectionId, String propertyFile) throws Exception {
@ -739,7 +739,7 @@ public class GraphDbReaderAndESIndexTest {
@Test
@Ignore
public void selectRecordsTest() throws Exception {
final ClassPathResource resource = new ClassPathResource("application.properties");
final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES);
Properties appProps = new Properties();
appProps.load(resource.getInputStream());
String datasource = "ads";