diff --git a/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java b/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java index 4496fa1..3d77bf5 100644 --- a/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java +++ b/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java @@ -3,6 +3,7 @@ package eu.dnetlib.app.directindex.solr; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.stream.Stream; @@ -17,6 +18,7 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.app.directindex.errors.DirectIndexApiException; @@ -36,7 +38,21 @@ public class SolrIndexClient { public void addRecords(final Stream records) throws DirectIndexApiException { try { - solrClient.add(records.map(this::prepareSolrDocument).filter(o -> o != null).iterator()); + final Iterator iter = records.map(this::prepareSolrDocument) + .map(r -> { + if (log.isDebugEnabled()) { + try { + log.debug("Indexing record: " + new ObjectMapper().writeValueAsString(r)); + } catch (final JsonProcessingException e) { + log.debug("The record seems invalid: " + r); + } + } + return r; + }) + .filter(o -> o != null) + .iterator(); + + solrClient.add(iter); solrClient.commit(); } catch (final Throwable e) { throw new DirectIndexApiException("Error creating solr document", e); @@ -59,7 +75,7 @@ public class SolrIndexClient { } } - private SolrInputDocument prepareSolrDocument(final SolrRecord record) { + protected SolrInputDocument prepareSolrDocument(final SolrRecord record) { try { return SolrInputDocumentMapper.map(record, XMLSolrSerializer.generateXML(record)); } catch (final Throwable e) { diff --git a/src/main/resources/xqueries/findIndexDsInfo.xquery b/src/main/resources/xqueries/findIndexDsInfo.xquery deleted file mode 100644 index 4b75513..0000000 --- a/src/main/resources/xqueries/findIndexDsInfo.xquery +++ /dev/null @@ -1,11 +0,0 @@ -distinct-values(for $s in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public'] ] -let $format := $s/PROPERTY[@key = "mdformat"]/@value/string() -return( -for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType') -where - $x//METADATA_FORMAT =$format and - $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and - $x//METADATA_FORMAT_LAYOUT = 'index' -return - concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format , ' @@@ ', $format , '-index-openaire') -)) diff --git a/src/main/resources/xqueries/findSolrIndexUrl.xquery b/src/main/resources/xqueries/findSolrIndexUrl.xquery deleted file mode 100644 index ba3ceab..0000000 --- a/src/main/resources/xqueries/findSolrIndexUrl.xquery +++ /dev/null @@ -1 +0,0 @@ -distinct-values(collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string()) diff --git a/src/test/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapperTest.java b/src/test/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapperTest.java index 9ef2a99..5207627 100644 --- a/src/test/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapperTest.java +++ b/src/test/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapperTest.java @@ -1,12 +1,14 @@ package eu.dnetlib.app.directindex.mapping; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import org.apache.commons.lang3.StringUtils; +import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -24,6 +26,7 @@ import eu.dnetlib.app.directindex.errors.DirectIndexApiException; import eu.dnetlib.app.directindex.input.DatasourceEntry; import eu.dnetlib.app.directindex.input.ResultEntry; import eu.dnetlib.dhp.schema.solr.SolrRecord; +import eu.dnetlib.dhp.solr.mapping.SolrInputDocumentMapper; @ExtendWith(MockitoExtension.class) class SolrRecordMapperTest { @@ -77,7 +80,7 @@ class SolrRecordMapperTest { @Test public void testToSolrRecord() throws Exception { - final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("sample-result-01.json"), ResultEntry.class); + final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class); final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result); @@ -128,7 +131,7 @@ class SolrRecordMapperTest { @Test public void testToXmlSolrRecord() throws Exception { - final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("sample-result-01.json"), ResultEntry.class); + final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class); final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result); @@ -148,4 +151,19 @@ class SolrRecordMapperTest { } + @Test + void testSolrInputDocumentMapper() throws Exception { + final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class); + + final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result); + + // System.out.println(solrRecord.getLinks().get(0).getFunding().getLevel1()); + + final String xml = XMLSolrSerializer.generateXML(solrRecord); + + final SolrInputDocument solrInputDocumentMapper = SolrInputDocumentMapper.map(solrRecord, xml); + + assertNotNull(solrInputDocumentMapper); + } + } diff --git a/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java b/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java index 36cce67..fd28e1c 100644 --- a/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java +++ b/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java @@ -2,8 +2,9 @@ package eu.dnetlib.app.directindex.solr; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -21,20 +22,11 @@ class SolrIndexClientTest { @BeforeEach public void initEach() throws DirectIndexApiException { + solrIndexClient = new SolrIndexClient( new SolrIndexClientFactory() .getLoadBalancedClient("http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr", "DMF-index-openaire")); - } - @Test - void testParameters() { - final String id = "test________::1234"; - - final String query1 = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id); - final String query2 = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id), ClientUtils.escapeQueryChars(id)); - - System.out.println(query1); - System.out.println(query2); } @Test @@ -46,6 +38,9 @@ class SolrIndexClientTest { assertNotNull(rec); assertEquals(objIdentifier, rec.getHeader().getId()); + assertEquals(objIdentifier, rec.getHeader().getId()); + assertTrue(StringUtils.isNoneBlank(rec.getResult().getMaintitle())); + System.err.println("Record:"); System.err.println(new ObjectMapper().writeValueAsString(rec)); System.err.println("END"); diff --git a/src/test/resources/eu/dnetlib/app/directindex/mapping/sample-result-01.json b/src/test/resources/samples/sample-result-01.json similarity index 100% rename from src/test/resources/eu/dnetlib/app/directindex/mapping/sample-result-01.json rename to src/test/resources/samples/sample-result-01.json