multiple implmementation od solr clients
This commit is contained in:
parent
8a59cd01e5
commit
bc67714e2f
|
@ -49,14 +49,6 @@ public class XMLSolrSerializer {
|
||||||
@SuppressWarnings("deprecation")
|
@SuppressWarnings("deprecation")
|
||||||
private static void populateResultFields(final SolrRecord sr, final Element fields) {
|
private static void populateResultFields(final SolrRecord sr, final Element fields) {
|
||||||
|
|
||||||
if (sr.getCollectedfrom() != null) {
|
|
||||||
sr.getCollectedfrom().forEach(p -> {
|
|
||||||
final Element collectedFrom = fields.addElement("collectedfrom");
|
|
||||||
collectedFrom.addAttribute("id", p.getDsId());
|
|
||||||
collectedFrom.addAttribute("name", p.getDsName());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sr.getHeader().getOriginalId() != null) {
|
if (sr.getHeader().getOriginalId() != null) {
|
||||||
sr.getHeader().getOriginalId().forEach(id -> fields.addElement("originalId").addText(id));
|
sr.getHeader().getOriginalId().forEach(id -> fields.addElement("originalId").addText(id));
|
||||||
}
|
}
|
||||||
|
@ -112,6 +104,14 @@ public class XMLSolrSerializer {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (sr.getCollectedfrom() != null) {
|
||||||
|
sr.getCollectedfrom().forEach(p -> {
|
||||||
|
final Element collectedFrom = fields.addElement("collectedfrom");
|
||||||
|
collectedFrom.addAttribute("id", p.getDsId());
|
||||||
|
collectedFrom.addAttribute("name", p.getDsName());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
final Element datainfo = fields.addElement("datainfo");
|
final Element datainfo = fields.addElement("datainfo");
|
||||||
datainfo.addElement("inferred").addText("false");
|
datainfo.addElement("inferred").addText("false");
|
||||||
datainfo.addElement("deletedbyinference").addText("false");
|
datainfo.addElement("deletedbyinference").addText("false");
|
||||||
|
|
|
@ -1,11 +1,14 @@
|
||||||
package eu.dnetlib.app.directindex.solr;
|
package eu.dnetlib.app.directindex.solr;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.solr.client.solrj.SolrClient;
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
import org.apache.solr.client.solrj.impl.Http2SolrClient;
|
import org.apache.solr.client.solrj.impl.Http2SolrClient;
|
||||||
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
|
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
|
||||||
import org.apache.solr.client.solrj.impl.LBSolrClient.Endpoint;
|
import org.apache.solr.client.solrj.impl.LBSolrClient.Endpoint;
|
||||||
|
@ -19,31 +22,89 @@ public class SolrIndexClientFactory {
|
||||||
|
|
||||||
public static final String CHROOT_SEPARATOR = "/";
|
public static final String CHROOT_SEPARATOR = "/";
|
||||||
|
|
||||||
@Value("${dnet.directindex.solr.url}")
|
@Value("${dnet.directindex.solr.urls}")
|
||||||
public String solrUrls;
|
public String solrUrls;
|
||||||
|
|
||||||
@Value("${dnet.directindex.solr.collection}")
|
@Value("${dnet.directindex.solr.collection}")
|
||||||
public String solrCollection;
|
public String solrCollection;
|
||||||
|
|
||||||
|
@Value("${dnet.directindex.solr.client}")
|
||||||
|
public String solrClientType;
|
||||||
|
|
||||||
|
@Value("${dnet.directindex.solr.zk_chroot}")
|
||||||
|
public String solrZkChroot;
|
||||||
|
|
||||||
public SolrIndexClient getClient() {
|
public SolrIndexClient getClient() {
|
||||||
return getClient(solrUrls, solrCollection);
|
|
||||||
|
final SolrClient client;
|
||||||
|
|
||||||
|
switch (solrClientType.toUpperCase()) {
|
||||||
|
case "LBHTTP2":
|
||||||
|
client = getLoadBalancedClient(solrUrls, solrCollection);
|
||||||
|
break;
|
||||||
|
case "CLOUD":
|
||||||
|
client = getCloudClient(solrUrls, solrCollection);
|
||||||
|
break;
|
||||||
|
case "ZK_CLOUD":
|
||||||
|
client = getZKCloudClient(solrUrls, solrCollection, Optional.of(solrZkChroot));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
client = getSimpleClient(solrUrls, solrCollection);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new SolrIndexClient(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected SolrIndexClient getClient(final String urls, final String coll) {
|
protected SolrClient getLoadBalancedClient(final String urls, final String coll) {
|
||||||
log.info(String.format("Initializing solr client, urls: %s, collection: %s", urls, coll));
|
log.info(String.format("Initializing solr client (Load Balanced), urls: %s, collection: %s", urls, coll));
|
||||||
|
|
||||||
final Endpoint[] endpoints = Arrays.stream(urls.split(","))
|
final Endpoint[] endpoints = Arrays.stream(urls.split(","))
|
||||||
.map(String::trim)
|
.map(String::trim)
|
||||||
.filter(StringUtils::isNotBlank)
|
.filter(StringUtils::isNotBlank)
|
||||||
.map(url -> url += "/solr/" + coll)
|
.map(url -> url += "/" + coll)
|
||||||
.map(Endpoint::new)
|
.map(Endpoint::new)
|
||||||
.toArray(Endpoint[]::new);
|
.toArray(Endpoint[]::new);
|
||||||
|
|
||||||
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
|
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
|
||||||
|
|
||||||
final SolrClient client = new LBHttp2SolrClient.Builder(http2SolrClient, endpoints).build();
|
return new LBHttp2SolrClient.Builder(http2SolrClient, endpoints).build();
|
||||||
|
|
||||||
return new SolrIndexClient(client);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private SolrClient getCloudClient(final String urls, final String coll) {
|
||||||
|
log.info(String.format("Initializing solr client (cloud), urls: %s, collection: %s", urls, coll));
|
||||||
|
|
||||||
|
final List<String> urlList = Arrays.stream(urls.split(","))
|
||||||
|
.map(String::trim)
|
||||||
|
.filter(StringUtils::isNotBlank)
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
return new CloudSolrClient.Builder(urlList)
|
||||||
|
.withParallelUpdates(true)
|
||||||
|
.withDefaultCollection(coll)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected SolrClient getZKCloudClient(final String urls, final String coll, final Optional<String> chroot) {
|
||||||
|
log.info(String.format("Initializing solr client (cloud), urls: %s, collection: %s", urls, coll));
|
||||||
|
|
||||||
|
final List<String> urlList = Arrays.stream(urls.split(","))
|
||||||
|
.map(String::trim)
|
||||||
|
.filter(StringUtils::isNotBlank)
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
return new CloudSolrClient.Builder(urlList, chroot)
|
||||||
|
.withParallelUpdates(true)
|
||||||
|
.withDefaultCollection(coll)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
protected SolrClient getSimpleClient(final String url, final String coll) {
|
||||||
|
log.info(String.format("Initializing solr client (SIMPLE), url: %s, collection: %s", url, coll));
|
||||||
|
final Endpoint endpoint = new Endpoint(url.trim() + "/" + coll);
|
||||||
|
final Http2SolrClient http2SolrClient = new Http2SolrClient.Builder().build();
|
||||||
|
return new LBHttp2SolrClient.Builder(http2SolrClient, endpoint).build();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,8 +8,13 @@ dnet.directindex.scheduling.enabled = false
|
||||||
dnet.directindex.legacy.enabled = true
|
dnet.directindex.legacy.enabled = true
|
||||||
dnet.directindex.sword.enabled = false
|
dnet.directindex.sword.enabled = false
|
||||||
|
|
||||||
dnet.directindex.solr.urls = http://localhost:8981,http://localhost:8982,http://localhost:8983
|
dnet.directindex.solr.urls = http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr
|
||||||
dnet.directindex.solr.collection = DMF-index-openaire
|
dnet.directindex.solr.collection = DMF-index-openaire
|
||||||
|
dnet.directindex.solr.client = LBHTTP2
|
||||||
|
#dnet.directindex.solr.client = CLOUD
|
||||||
|
#dnet.directindex.solr.client = ZK_CLOUD
|
||||||
|
#dnet.directindex.solr.client = SIMPLE
|
||||||
|
dnet.directindex.solr.zk_chroot = /solr
|
||||||
|
|
||||||
dnet.directindex.community.url = https://services.openaire.eu/openaire/community
|
dnet.directindex.community.url = https://services.openaire.eu/openaire/community
|
||||||
dnet.directindex.dsm.url = https://services.openaire.eu/openaire/ds
|
dnet.directindex.dsm.url = https://services.openaire.eu/openaire/ds
|
||||||
|
|
|
@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
|
||||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Disabled;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
@ -13,7 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
|
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
|
||||||
import eu.dnetlib.dhp.schema.solr.SolrRecord;
|
import eu.dnetlib.dhp.schema.solr.SolrRecord;
|
||||||
|
|
||||||
@Disabled
|
// @Disabled
|
||||||
class SolrIndexClientTest {
|
class SolrIndexClientTest {
|
||||||
|
|
||||||
// Class under test
|
// Class under test
|
||||||
|
@ -21,8 +20,8 @@ class SolrIndexClientTest {
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
public void initEach() throws DirectIndexApiException {
|
public void initEach() throws DirectIndexApiException {
|
||||||
solrIndexClient = new SolrIndexClientFactory().getClient("http://localhost:8981,http://localhost:8982,http://localhost:8983", "DMF-index-openaire");
|
solrIndexClient = new SolrIndexClient(
|
||||||
|
new SolrIndexClientFactory().getLoadBalancedClient("http://localhost:8981/solr,http://localhost:8982/solr,http://localhost:8983/solr", "DMF-index-openaire"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
Loading…
Reference in New Issue