remove -x from the name to disable noisy logging on tests

dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/FindMissingRecordsInIndexTest.java

@@ -0,0 +1,79 @@
+package eu.dnetlib.ariadneplus;
+
+import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
+import eu.dnetlib.ariadneplus.reader.ResourceManager;
+import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService;
+import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
+import eu.dnetlib.ariadneplus.reader.utils.ESUtils;
+import org.apache.http.HttpHost;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Properties;
+
+@Ignore
+public class FindMissingRecordsInIndexTest {
+
+    private RunSPARQLQueryService runSPQRLQuery;
+
+    private final static String STAGING_PROPERTIES = "application.properties";
+    private final static String PROD_PROPERTIES = "application-prod-DO-NOT-COMMIT.properties";
+
+    private RestHighLevelClient client;
+    private  Properties appProps;
+    @Before
+    public void setUp() throws IOException {
+        final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES);
+        appProps = new Properties();
+        appProps.load(resource.getInputStream());
+        runSPQRLQuery = new RunSPARQLQueryService();
+        runSPQRLQuery.setupReadOnlyConnection(
+                appProps.getProperty("graphdb.serverUrl"),
+                appProps.getProperty("graphdb.repository"));
+         client = new RestHighLevelClient(
+                RestClient.builder(
+                        new HttpHost(appProps.getProperty("elasticsearch.hostname"), 9200, "http")));
+    }
+
+    @After
+    public void tearDown() throws IOException {
+        client.close();
+    }
+    @Test
+    public void findMissingRecordsTest() throws NoSuchAlgorithmException, IOException {
+        String datasource = "ads";
+        String collectionId = "archives";
+        List<String> uris = runSPQRLQuery.selectRecordIds(datasource, collectionId);
+        System.out.println("Got list of ids, they are "+uris.size());
+        for(String uri : uris){
+            //compute the md5 and query the index to check if there is a record with that uri. If not, print it
+            MessageDigest digest = MessageDigest.getInstance("SHA-256");
+            byte[] encodedhash = digest.digest(
+                    uri.getBytes(StandardCharsets.UTF_8));
+            String idES = ESUtils.bytesToHex(encodedhash);
+            // High level API
+            GetRequest getRequest = new GetRequest(
+                    appProps.getProperty("elasticsearch.indexname"),
+                    idES);
+            getRequest.fetchSourceContext(new FetchSourceContext(false));
+            getRequest.storedFields("_none_");
+
+            boolean exists = client.exists(getRequest, RequestOptions.DEFAULT);
+            if(!exists) System.out.println(uri);
+        }
+        System.out.println("Done");
+    }
+}

dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java

@@ -47,9 +47,9 @@ public class GraphDbReaderAndESIndexTest {
     public void loadToStaging() throws Exception {
 
         //String uri = "https://arche.acdh.oeaw.ac.at/api/255841";
-        String uri = "https://ariadne-infrastructure.eu/aocat/Resource/CENIEH/013A55AB-432D-3B51-B3A6-B6AFE343054B";
-        String datasource = "cenieh";
-        String apiId = "cir";
+        String uri = "https://ariadne-infrastructure.eu/aocat/Resource/ADS/6CBA2A00-01DE-315B-934C-6ED74C3BC6DC";
+        String datasource = "ads";
+        String apiId = "archives";
         readAndIndexTest(true, uri, datasource, apiId);
 
     }
@@ -684,11 +684,11 @@ public class GraphDbReaderAndESIndexTest {
     }
 
     private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
-        readAndIndex( isRecord,  recordId,  datasource,  collectionId, "application.properties");
+        readAndIndex( isRecord,  recordId,  datasource,  collectionId, STAGING_PROPERTIES);
     }
 
     private void readAndIndexProd(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
-        readAndIndex( isRecord,  recordId,  datasource,  collectionId, "application-prod-DO-NOT-COMMIT.properties");
+        readAndIndex( isRecord,  recordId,  datasource,  collectionId, PROD_PROPERTIES);
     }
 
     private void readAndIndex(boolean isRecord, String recordId, String datasource, String collectionId, String propertyFile) throws Exception {
@@ -739,7 +739,7 @@ public class GraphDbReaderAndESIndexTest {
     @Test
     @Ignore
     public void selectRecordsTest() throws Exception {
-        final ClassPathResource resource = new ClassPathResource("application.properties");
+        final ClassPathResource resource = new ClassPathResource(STAGING_PROPERTIES);
         Properties appProps = new Properties();
         appProps.load(resource.getInputStream());
         String datasource = "ads";

dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUploadTest.java

@@ -5,7 +5,13 @@ import org.junit.Before;
 import org.junit.Test;
 
 public class BulkUploadTest {
-
+ /*
+ About BoundingBox and WKT:
+ X = LONGITUDE; Y = LATITUDE
+bbox2wkt(minx = NA, miny = NA, maxx = NA, maxy = NA, bbox = NULL)
+bbox:   MIN LONG, MIN LAT,MAX LONG, MAX LAT
+wkt <- "POLYGON((MINLONG MINLAT,MINLONG MAXLAT,MAXLON MAXLAT,MAXLON MINLAT,MINLON MINLAT))"
+  */
     private BulkUpload bu;
 
     @Before
@@ -55,5 +61,4 @@ public class BulkUploadTest {
         System.out.println("Long: "+centroid.getLon()+", Lat: "+centroid.getLat());
     }
 
-
 }

dnet-ariadneplus-graphdb-publisher/test/resources/logback-test-x.xml

@@ -0,0 +1,2 @@
+<!-- only one line, shut up logback ! -->
+<configuration />