multiple implmementation od solr clients

This commit is contained in:
Michele Artini 2024-12-06 15:14:47 +01:00
parent 8a59cd01e5
commit bc67714e2f
4 changed files with 85 additions and 20 deletions

View File

@ -49,14 +49,6 @@ public class XMLSolrSerializer {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static void populateResultFields(final SolrRecord sr, final Element fields) { private static void populateResultFields(final SolrRecord sr, final Element fields) {
if (sr.getCollectedfrom() != null) {
sr.getCollectedfrom().forEach(p -> {
final Element collectedFrom = fields.addElement("collectedfrom");
collectedFrom.addAttribute("id", p.getDsId());
collectedFrom.addAttribute("name", p.getDsName());
});
}
if (sr.getHeader().getOriginalId() != null) { if (sr.getHeader().getOriginalId() != null) {
sr.getHeader().getOriginalId().forEach(id -> fields.addElement("originalId").addText(id)); sr.getHeader().getOriginalId().forEach(id -> fields.addElement("originalId").addText(id));
} }
@ -112,6 +104,14 @@ public class XMLSolrSerializer {
}); });
} }
if (sr.getCollectedfrom() != null) {
sr.getCollectedfrom().forEach(p -> {
final Element collectedFrom = fields.addElement("collectedfrom");
collectedFrom.addAttribute("id", p.getDsId());
collectedFrom.addAttribute("name", p.getDsName());
});
}
final Element datainfo = fields.addElement("datainfo"); final Element datainfo = fields.addElement("datainfo");
datainfo.addElement("inferred").addText("false"); datainfo.addElement("inferred").addText("false");
datainfo.addElement("deletedbyinference").addText("false"); datainfo.addElement("deletedbyinference").addText("false");

View File

@ -1,11 +1,14 @@
package eu.dnetlib.app.directindex.solr; package eu.dnetlib.app.directindex.solr;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient; import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient.Endpoint; import org.apache.solr.client.solrj.impl.LBSolrClient.Endpoint;
@ -19,31 +22,89 @@ public class SolrIndexClientFactory {
public static final String CHROOT_SEPARATOR = "/"; public static final String CHROOT_SEPARATOR = "/";
@Value("${dnet.directindex.solr.url}") @Value("${dnet.directindex.solr.urls}")
public String solrUrls; public String solrUrls;
@Value("${dnet.directindex.solr.collection}") @Value("${dnet.directindex.solr.collection}")
public String solrCollection; public String solrCollection;
@Value("${dnet.directindex.solr.client}")
public String solrClientType;
@Value("${dnet.directindex.solr.zk_chroot}")
public String solrZkChroot;
public SolrIndexClient getClient() { public SolrIndexClient getClient() {
return getClient(solrUrls, solrCollection);
final SolrClient client;
switch (solrClientType.toUpperCase()) {
case "LBHTTP2":
client = getLoadBalancedClient(solrUrls, solrCollection);
break;
case "CLOUD":
client = getCloudClient(solrUrls, solrCollection);
break;
case "ZK_CLOUD":
client = getZKCloudClient(solrUrls, solrCollection, Optional.of(solrZkChroot));
break;
default:
client = getSimpleClient(solrUrls, solrCollection);
break;
}
return new SolrIndexClient(client);
} }
protected SolrIndexClient getClient(final String urls, final String coll) { protected SolrClient getLoadBalancedClient(final String urls, final String coll) {
log.info(String.format("Initializing solr client, urls: %s, collection: %s", urls, coll)); log.info(String.format("Initializing solr client (Load Balanced), urls: %s, collection: %s", urls, coll));
final Endpoint[] endpoints = Arrays.stream(urls.split(",")) final Endpoint[] endpoints = Arrays.stream(urls.split(","))
.map(String::trim) .map(String::trim)
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.map(url -> url += "/solr/" + coll) .map(url -> url += "/" + coll)
.map(Endpoint::new) .map(Endpoint::new)
.toArray(Endpoint[]::new); .toArray(Endpoint[]::new);
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build(); final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
final SolrClient client = new LBHttp2SolrClient.Builder(http2SolrClient, endpoints).build(); return new LBHttp2SolrClient.Builder(http2SolrClient, endpoints).build();
return new SolrIndexClient(client);
} }
private SolrClient getCloudClient(final String urls, final String coll) {
log.info(String.format("Initializing solr client (cloud), urls: %s, collection: %s", urls, coll));
final List<String> urlList = Arrays.stream(urls.split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.toList();
return new CloudSolrClient.Builder(urlList)
.withParallelUpdates(true)
.withDefaultCollection(coll)
.build();
}
protected SolrClient getZKCloudClient(final String urls, final String coll, final Optional<String> chroot) {
log.info(String.format("Initializing solr client (cloud), urls: %s, collection: %s", urls, coll));
final List<String> urlList = Arrays.stream(urls.split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.toList();
return new CloudSolrClient.Builder(urlList, chroot)
.withParallelUpdates(true)
.withDefaultCollection(coll)
.build();
}
protected SolrClient getSimpleClient(final String url, final String coll) {
log.info(String.format("Initializing solr client (SIMPLE), url: %s, collection: %s", url, coll));
final Endpoint endpoint = new Endpoint(url.trim() + "/" + coll);
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
return new LBHttp2SolrClient.Builder(http2SolrClient, endpoint).build();
}
} }

View File

@ -8,8 +8,13 @@ dnet.directindex.scheduling.enabled = false
dnet.directindex.legacy.enabled = true dnet.directindex.legacy.enabled = true
dnet.directindex.sword.enabled = false dnet.directindex.sword.enabled = false
dnet.directindex.solr.urls = http://localhost:8981,http://localhost:8982,http://localhost:8983 dnet.directindex.solr.urls = http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr
dnet.directindex.solr.collection = DMF-index-openaire dnet.directindex.solr.collection = DMF-index-openaire
dnet.directindex.solr.client = LBHTTP2
#dnet.directindex.solr.client = CLOUD
#dnet.directindex.solr.client = ZK_CLOUD
#dnet.directindex.solr.client = SIMPLE
dnet.directindex.solr.zk_chroot = /solr
dnet.directindex.community.url = https://services.openaire.eu/openaire/community dnet.directindex.community.url = https://services.openaire.eu/openaire/community
dnet.directindex.dsm.url = https://services.openaire.eu/openaire/ds dnet.directindex.dsm.url = https://services.openaire.eu/openaire/ds

View File

@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.client.solrj.util.ClientUtils;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@ -13,7 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException; import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.dhp.schema.solr.SolrRecord; import eu.dnetlib.dhp.schema.solr.SolrRecord;
@Disabled // @Disabled
class SolrIndexClientTest { class SolrIndexClientTest {
// Class under test // Class under test
@ -21,8 +20,8 @@ class SolrIndexClientTest {
@BeforeEach @BeforeEach
public void initEach() throws DirectIndexApiException { public void initEach() throws DirectIndexApiException {
solrIndexClient = new SolrIndexClientFactory().getClient("http://localhost:8981,http://localhost:8982,http://localhost:8983", "DMF-index-openaire"); solrIndexClient = new SolrIndexClient(
new SolrIndexClientFactory().getLoadBalancedClient("http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr", "DMF-index-openaire"));
} }
@Test @Test