cleaning and test

src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java

@@ -3,6 +3,7 @@ package eu.dnetlib.app.directindex.solr;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -17,6 +18,7 @@ import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.SolrParams;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
@@ -36,7 +38,21 @@ public class SolrIndexClient {
 
 	public void addRecords(final Stream<SolrRecord> records) throws DirectIndexApiException {
 		try {
-			solrClient.add(records.map(this::prepareSolrDocument).filter(o -> o != null).iterator());
+			final Iterator<SolrInputDocument> iter = records.map(this::prepareSolrDocument)
+					.map(r -> {
+						if (log.isDebugEnabled()) {
+							try {
+								log.debug("Indexing record: " + new ObjectMapper().writeValueAsString(r));
+							} catch (final JsonProcessingException e) {
+								log.debug("The record seems invalid: " + r);
+							}
+						}
+						return r;
+					})
+					.filter(o -> o != null)
+					.iterator();
+
+			solrClient.add(iter);
 			solrClient.commit();
 		} catch (final Throwable e) {
 			throw new DirectIndexApiException("Error creating solr document", e);
@@ -59,7 +75,7 @@ public class SolrIndexClient {
 		}
 	}
 
-	private SolrInputDocument prepareSolrDocument(final SolrRecord record) {
+	protected SolrInputDocument prepareSolrDocument(final SolrRecord record) {
 		try {
 			return SolrInputDocumentMapper.map(record, XMLSolrSerializer.generateXML(record));
 		} catch (final Throwable e) {

src/main/resources/xqueries/findIndexDsInfo.xquery

@@ -1,11 +0,0 @@
-distinct-values(for $s in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public'] ]
-let $format := $s/PROPERTY[@key = "mdformat"]/@value/string()
-return(
-for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType')
-where
-    $x//METADATA_FORMAT =$format and
-            $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and
-            $x//METADATA_FORMAT_LAYOUT = 'index'
-return
-    concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format , ' @@@ ', $format , '-index-openaire')
-))

src/main/resources/xqueries/findSolrIndexUrl.xquery

@@ -1 +0,0 @@
-distinct-values(collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string())

src/test/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapperTest.java

@@ -1,12 +1,14 @@
 package eu.dnetlib.app.directindex.mapping;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.common.SolrInputDocument;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -24,6 +26,7 @@ import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
 import eu.dnetlib.app.directindex.input.DatasourceEntry;
 import eu.dnetlib.app.directindex.input.ResultEntry;
 import eu.dnetlib.dhp.schema.solr.SolrRecord;
+import eu.dnetlib.dhp.solr.mapping.SolrInputDocumentMapper;
 
 @ExtendWith(MockitoExtension.class)
 class SolrRecordMapperTest {
@@ -77,7 +80,7 @@ class SolrRecordMapperTest {
 	@Test
 	public void testToSolrRecord() throws Exception {
 
-		final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("sample-result-01.json"), ResultEntry.class);
+		final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class);
 
 		final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result);
 
@@ -128,7 +131,7 @@ class SolrRecordMapperTest {
 
 	@Test
 	public void testToXmlSolrRecord() throws Exception {
-		final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("sample-result-01.json"), ResultEntry.class);
+		final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class);
 
 		final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result);
 
@@ -148,4 +151,19 @@ class SolrRecordMapperTest {
 
 	}
 
+	@Test
+	void testSolrInputDocumentMapper() throws Exception {
+		final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class);
+
+		final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result);
+
+		// System.out.println(solrRecord.getLinks().get(0).getFunding().getLevel1());
+
+		final String xml = XMLSolrSerializer.generateXML(solrRecord);
+
+		final SolrInputDocument solrInputDocumentMapper = SolrInputDocumentMapper.map(solrRecord, xml);
+
+		assertNotNull(solrInputDocumentMapper);
+	}
+
 }

src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java

@@ -2,8 +2,9 @@ package eu.dnetlib.app.directindex.solr;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -21,20 +22,11 @@ class SolrIndexClientTest {
 
 	@BeforeEach
 	public void initEach() throws DirectIndexApiException {
+
 		solrIndexClient = new SolrIndexClient(
 				new SolrIndexClientFactory()
 						.getLoadBalancedClient("http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr", "DMF-index-openaire"));
-	}
 
-	@Test
-	void testParameters() {
-		final String id = "test________::1234";
-
-		final String query1 = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);
-		final String query2 = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id), ClientUtils.escapeQueryChars(id));
-
-		System.out.println(query1);
-		System.out.println(query2);
 	}
 
 	@Test
@@ -46,6 +38,9 @@ class SolrIndexClientTest {
 		assertNotNull(rec);
 		assertEquals(objIdentifier, rec.getHeader().getId());
 
+		assertEquals(objIdentifier, rec.getHeader().getId());
+		assertTrue(StringUtils.isNoneBlank(rec.getResult().getMaintitle()));
+
 		System.err.println("Record:");
 		System.err.println(new ObjectMapper().writeValueAsString(rec));
 		System.err.println("END");