D-Net
/
dnet-directindex-application
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleaning and test

This commit is contained in:
Michele Artini 2024-12-09 12:18:47 +01:00
parent f83681edd0
commit c518c18a12
6 changed files with 44 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.app.directindex.solr;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.stream.Stream;
@ -17,6 +18,7 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
@ -36,7 +38,21 @@ public class SolrIndexClient {
public void addRecords(final Stream<SolrRecord> records) throws DirectIndexApiException {
try {
solrClient.add(records.map(this::prepareSolrDocument).filter(o -> o != null).iterator());
final Iterator<SolrInputDocument> iter = records.map(this::prepareSolrDocument)
.map(r -> {
if (log.isDebugEnabled()) {
try {
log.debug("Indexing record: " + new ObjectMapper().writeValueAsString(r));
} catch (final JsonProcessingException e) {
log.debug("The record seems invalid: " + r);
}
}
return r;
})
.filter(o -> o != null)
.iterator();
solrClient.add(iter);
solrClient.commit();
} catch (final Throwable e) {
throw new DirectIndexApiException("Error creating solr document", e);
@ -59,7 +75,7 @@ public class SolrIndexClient {
}
}
private SolrInputDocument prepareSolrDocument(final SolrRecord record) {
protected SolrInputDocument prepareSolrDocument(final SolrRecord record) {
try {
return SolrInputDocumentMapper.map(record, XMLSolrSerializer.generateXML(record));
} catch (final Throwable e) {

11
src/main/resources/xqueries/findIndexDsInfo.xquery
View File

@ -1,11 +0,0 @@
distinct-values(for $s in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public'] ]
let $format := $s/PROPERTY[@key = "mdformat"]/@value/string()
return(
for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType')
where
$x//METADATA_FORMAT =$format and
$x//METADATA_FORMAT_INTERPRETATION = 'openaire' and
$x//METADATA_FORMAT_LAYOUT = 'index'
return
concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format , ' @@@ ', $format , '-index-openaire')
))

1
src/main/resources/xqueries/findSolrIndexUrl.xquery
View File

@ -1 +0,0 @@
distinct-values(collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string())

22
src/test/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapperTest.java
View File

@ -1,12 +1,14 @@
package eu.dnetlib.app.directindex.mapping;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.common.SolrInputDocument;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -24,6 +26,7 @@ import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.app.directindex.input.DatasourceEntry;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
import eu.dnetlib.dhp.solr.mapping.SolrInputDocumentMapper;
@ExtendWith(MockitoExtension.class)
class SolrRecordMapperTest {
@ -77,7 +80,7 @@ class SolrRecordMapperTest {
@Test
public void testToSolrRecord() throws Exception {
final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("sample-result-01.json"), ResultEntry.class);
final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class);
final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result);
@ -128,7 +131,7 @@ class SolrRecordMapperTest {
@Test
public void testToXmlSolrRecord() throws Exception {
final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("sample-result-01.json"), ResultEntry.class);
final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class);
final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result);
@ -148,4 +151,19 @@ class SolrRecordMapperTest {
}
@Test
void testSolrInputDocumentMapper() throws Exception {
final ResultEntry result = new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/sample-result-01.json"), ResultEntry.class);
final SolrRecord solrRecord = solrRecordMapper.toSolrRecord(result);
// System.out.println(solrRecord.getLinks().get(0).getFunding().getLevel1());
final String xml = XMLSolrSerializer.generateXML(solrRecord);
final SolrInputDocument solrInputDocumentMapper = SolrInputDocumentMapper.map(solrRecord, xml);
assertNotNull(solrInputDocumentMapper);
}
}

17
src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java
View File

@ -2,8 +2,9 @@ package eu.dnetlib.app.directindex.solr;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@ -21,20 +22,11 @@ class SolrIndexClientTest {
@BeforeEach
public void initEach() throws DirectIndexApiException {
solrIndexClient = new SolrIndexClient(
new SolrIndexClientFactory()
.getLoadBalancedClient("http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr", "DMF-index-openaire"));
}
@Test
void testParameters() {
final String id = "test________::1234";
final String query1 = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);
final String query2 = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id), ClientUtils.escapeQueryChars(id));
System.out.println(query1);
System.out.println(query2);
}
@Test
@ -46,6 +38,9 @@ class SolrIndexClientTest {
assertNotNull(rec);
assertEquals(objIdentifier, rec.getHeader().getId());
assertEquals(objIdentifier, rec.getHeader().getId());
assertTrue(StringUtils.isNoneBlank(rec.getResult().getMaintitle()));
System.err.println("Record:");
System.err.println(new ObjectMapper().writeValueAsString(rec));
System.err.println("END");

0
src/test/resources/eu/dnetlib/app/directindex/mapping/sample-result-01.json → src/test/resources/samples/sample-result-01.json
View File