multiple implmementation od solr clients

This commit is contained in:
Michele Artini 2024-12-06 15:14:47 +01:00
parent 8a59cd01e5
commit bc67714e2f
4 changed files with 85 additions and 20 deletions

View File

@ -49,14 +49,6 @@ public class XMLSolrSerializer {
@SuppressWarnings("deprecation")
private static void populateResultFields(final SolrRecord sr, final Element fields) {
if (sr.getCollectedfrom() != null) {
sr.getCollectedfrom().forEach(p -> {
final Element collectedFrom = fields.addElement("collectedfrom");
collectedFrom.addAttribute("id", p.getDsId());
collectedFrom.addAttribute("name", p.getDsName());
});
}
if (sr.getHeader().getOriginalId() != null) {
sr.getHeader().getOriginalId().forEach(id -> fields.addElement("originalId").addText(id));
}
@ -112,6 +104,14 @@ public class XMLSolrSerializer {
});
}
if (sr.getCollectedfrom() != null) {
sr.getCollectedfrom().forEach(p -> {
final Element collectedFrom = fields.addElement("collectedfrom");
collectedFrom.addAttribute("id", p.getDsId());
collectedFrom.addAttribute("name", p.getDsName());
});
}
final Element datainfo = fields.addElement("datainfo");
datainfo.addElement("inferred").addText("false");
datainfo.addElement("deletedbyinference").addText("false");

View File

@ -1,11 +1,14 @@
package eu.dnetlib.app.directindex.solr;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient.Endpoint;
@ -19,31 +22,89 @@ public class SolrIndexClientFactory {
public static final String CHROOT_SEPARATOR = "/";
@Value("${dnet.directindex.solr.url}")
@Value("${dnet.directindex.solr.urls}")
public String solrUrls;
@Value("${dnet.directindex.solr.collection}")
public String solrCollection;
@Value("${dnet.directindex.solr.client}")
public String solrClientType;
@Value("${dnet.directindex.solr.zk_chroot}")
public String solrZkChroot;
public SolrIndexClient getClient() {
return getClient(solrUrls, solrCollection);
final SolrClient client;
switch (solrClientType.toUpperCase()) {
case "LBHTTP2":
client = getLoadBalancedClient(solrUrls, solrCollection);
break;
case "CLOUD":
client = getCloudClient(solrUrls, solrCollection);
break;
case "ZK_CLOUD":
client = getZKCloudClient(solrUrls, solrCollection, Optional.of(solrZkChroot));
break;
default:
client = getSimpleClient(solrUrls, solrCollection);
break;
}
return new SolrIndexClient(client);
}
protected SolrIndexClient getClient(final String urls, final String coll) {
log.info(String.format("Initializing solr client, urls: %s, collection: %s", urls, coll));
protected SolrClient getLoadBalancedClient(final String urls, final String coll) {
log.info(String.format("Initializing solr client (Load Balanced), urls: %s, collection: %s", urls, coll));
final Endpoint[] endpoints = Arrays.stream(urls.split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.map(url -> url += "/solr/" + coll)
.map(url -> url += "/" + coll)
.map(Endpoint::new)
.toArray(Endpoint[]::new);
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
final SolrClient client = new LBHttp2SolrClient.Builder(http2SolrClient, endpoints).build();
return new LBHttp2SolrClient.Builder(http2SolrClient, endpoints).build();
return new SolrIndexClient(client);
}
private SolrClient getCloudClient(final String urls, final String coll) {
log.info(String.format("Initializing solr client (cloud), urls: %s, collection: %s", urls, coll));
final List<String> urlList = Arrays.stream(urls.split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.toList();
return new CloudSolrClient.Builder(urlList)
.withParallelUpdates(true)
.withDefaultCollection(coll)
.build();
}
protected SolrClient getZKCloudClient(final String urls, final String coll, final Optional<String> chroot) {
log.info(String.format("Initializing solr client (cloud), urls: %s, collection: %s", urls, coll));
final List<String> urlList = Arrays.stream(urls.split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.toList();
return new CloudSolrClient.Builder(urlList, chroot)
.withParallelUpdates(true)
.withDefaultCollection(coll)
.build();
}
protected SolrClient getSimpleClient(final String url, final String coll) {
log.info(String.format("Initializing solr client (SIMPLE), url: %s, collection: %s", url, coll));
final Endpoint endpoint = new Endpoint(url.trim() + "/" + coll);
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
return new LBHttp2SolrClient.Builder(http2SolrClient, endpoint).build();
}
}

View File

@ -8,8 +8,13 @@ dnet.directindex.scheduling.enabled = false
dnet.directindex.legacy.enabled = true
dnet.directindex.sword.enabled = false
dnet.directindex.solr.urls = http://localhost:8981,http://localhost:8982,http://localhost:8983
dnet.directindex.solr.urls = http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr
dnet.directindex.solr.collection = DMF-index-openaire
dnet.directindex.solr.client = LBHTTP2
#dnet.directindex.solr.client = CLOUD
#dnet.directindex.solr.client = ZK_CLOUD
#dnet.directindex.solr.client = SIMPLE
dnet.directindex.solr.zk_chroot = /solr
dnet.directindex.community.url = https://services.openaire.eu/openaire/community
dnet.directindex.dsm.url = https://services.openaire.eu/openaire/ds

View File

@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -13,7 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
@Disabled
// @Disabled
class SolrIndexClientTest {
// Class under test
@ -21,8 +20,8 @@ class SolrIndexClientTest {
@BeforeEach
public void initEach() throws DirectIndexApiException {
solrIndexClient = new SolrIndexClientFactory().getClient("http://localhost:8981,http://localhost:8982,http://localhost:8983", "DMF-index-openaire");
solrIndexClient = new SolrIndexClient(
new SolrIndexClientFactory().getLoadBalancedClient("http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr", "DMF-index-openaire"));
}
@Test