diff --git a/apps/scholexplorer-api/pom.xml b/apps/scholexplorer-api/pom.xml index 04b31ca5..58eb06ea 100644 --- a/apps/scholexplorer-api/pom.xml +++ b/apps/scholexplorer-api/pom.xml @@ -30,6 +30,11 @@ dhp-schemas + + org.apache.commons + commons-pool2 + + diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java index 2ce535e4..98de0853 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java @@ -42,11 +42,6 @@ public class MainApplication extends AbstractDnetApp { @Value("${dhp.swagger.api.basePath}") private String swaggerPath; - final String descriptionAPI = "

\"ScholeXplorer\"

" + - "The Scholix Swagger API allows clients to run REST queries over the Scholexplorer index in order to fetch links matching given criteria. In the current version, clients can search for:" + - ""; public static void main(final String[] args) { SpringApplication.run(MainApplication.class, args); @@ -57,20 +52,17 @@ public class MainApplication extends AbstractDnetApp { @Bean public TaggedCounter myCounter(MeterRegistry meterRegistry) { - return new TaggedCounter("scholixLinkCounter", "links",meterRegistry); + return new TaggedCounter(ScholixAPIConstants.SCHOLIX_MANAGER_COUNTER_NAME, ScholixAPIConstants.SCHOLIX_MANAGER_TAG_NAME,meterRegistry); } @Bean public TimedAspect timedAspect(MeterRegistry meterRegistry) { - - MeterFilter mf = new MeterFilter() { @Override public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { - if (id.getName().startsWith("scholix")) { + if (id.getName().startsWith(ScholixAPIConstants.SCHOLIX_COUNTER_PREFIX)) { return DistributionStatisticConfig.builder() - .percentiles(0.20, 0.50, 0.75,0.95) .percentilesHistogram(false) .serviceLevelObjectives( histogramValues) .build() @@ -79,31 +71,24 @@ public class MainApplication extends AbstractDnetApp { return config; } }; - - meterRegistry.config().meterFilter(mf); - - - return new TimedAspect(meterRegistry); } - - @Override protected void configSwagger(final Docket docket) { docket .host(swaggetHost) .pathMapping(swaggerPath) - .groupName("Scholexplorer V1") + .groupName(ScholixAPIConstants.API_V1_NAME) .select() .apis(RequestHandlerSelectors.any()) .paths(p -> p.startsWith("/v1")) .build() .apiInfo(new ApiInfoBuilder() - .title("Scholexplorer API V1.0") - .description(descriptionAPI) + .title(ScholixAPIConstants.API_V1_NAME) + .description(ScholixAPIConstants.API_DESCRIPTION) .version("1.0") .contact(ApiInfo.DEFAULT_CONTACT) .license("Apache 2.0") @@ -113,23 +98,20 @@ public class MainApplication extends AbstractDnetApp { - @Bean (name = "SpringDOcketv2") + @Bean (name = "SpringDocketv2") public Docket v2Docket() { - - final Docket docket = new Docket(DocumentationType.SWAGGER_2); - docket .host(swaggetHost) .pathMapping(swaggerPath) - .groupName("Scholexplorer V2") + .groupName(ScholixAPIConstants.API_V2_NAME) .select() .apis(RequestHandlerSelectors.any()) .paths(p -> p.startsWith("/v2")) .build() .apiInfo(new ApiInfoBuilder() - .title("Scholexplorer API V2.0") - .description(descriptionAPI) + .title(ScholixAPIConstants.API_V2_NAME) + .description(ScholixAPIConstants.API_DESCRIPTION) .version("2.0") .contact(ApiInfo.DEFAULT_CONTACT) .license("Apache 2.0") diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java index 078e489b..92b9a27f 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java @@ -1,5 +1,6 @@ package eu.dnetlib.scholix.api; +import eu.dnetlib.scholix.api.index.ElasticSearchPool; import eu.dnetlib.scholix.api.index.ElasticSearchProperties; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; @@ -12,20 +13,29 @@ import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfig import java.time.Duration; @Configuration -public class RestClientConfig extends AbstractElasticsearchConfiguration { +public class RestClientConfig { @Autowired private ElasticSearchProperties elasticSearchProperties; - @Override - @Bean - public RestHighLevelClient elasticsearchClient() { - final ClientConfiguration clientConfiguration = ClientConfiguration.builder() - .connectedTo(elasticSearchProperties.getClusterNodes().split(",")) - .withConnectTimeout(elasticSearchProperties.getConnectionTimeout()) - .withSocketTimeout(elasticSearchProperties.getSocketTimeout()) - .build(); - return RestClients.create(clientConfiguration).rest(); + @Bean + public ElasticSearchPool connectionPool() { + + ElasticSearchPool pool = new ElasticSearchPool(elasticSearchProperties); + return pool; } + + +// @Override +// @Bean +// public RestHighLevelClient elasticsearchClient() { +// +// final ClientConfiguration clientConfiguration = ClientConfiguration.builder() +// .connectedTo(elasticSearchProperties.getClusterNodes().split(",")) +// .withConnectTimeout(elasticSearchProperties.getConnectionTimeout()) +// .withSocketTimeout(elasticSearchProperties.getSocketTimeout()) +// .build(); +// return RestClients.create(clientConfiguration).rest(); +// } } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIConstants.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIConstants.java new file mode 100644 index 00000000..6f0b8a2d --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIConstants.java @@ -0,0 +1,23 @@ +package eu.dnetlib.scholix.api; + +public class ScholixAPIConstants { + + public static final String API_V1_NAME = "Scholexplorer API V1.0"; + public static final String API_V2_NAME = "Scholexplorer API V2.0"; + + public static String API_DESCRIPTION ="

\"ScholeXplorer\"

" + + "The Scholix Swagger API allows clients to run REST queries over the Scholexplorer index in order to fetch links matching given criteria. In the current version, clients can search for:" + + ""; + + + public static String SCHOLIX_MANAGER_COUNTER_NAME= "scholixLinkCounter"; + public static final String SCHOLIX_MANAGER_TAG_NAME = "links"; + + public static String SCHOLIX_COUNTER_PREFIX = "scholix"; + + + + +} diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java index 116a1e77..b54a65cb 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java @@ -4,6 +4,7 @@ package eu.dnetlib.scholix.api.controller; import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.dhp.schema.sx.scholix.ScholixIdentifier; import eu.dnetlib.scholix.api.ScholixAPIVersion; +import eu.dnetlib.scholix.api.ScholixException; import eu.dnetlib.scholix.api.index.ScholixIndexManager; import eu.dnetlib.scholix.api.model.v2.PageResultType; import eu.dnetlib.scholix.api.model.v2.ScholixType; @@ -13,6 +14,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -77,19 +79,20 @@ public class ScholixControllerV2 extends AbstractDnetController { @Parameter(in = ParameterIn.QUERY, description = "select page of result") Integer page) throws Exception { + if (StringUtils.isEmpty(sourcePid) && StringUtils.isEmpty(targetPid) && StringUtils.isEmpty(sourcePublisher)&& StringUtils.isEmpty(targetPublisher)&& StringUtils.isEmpty(linkProvider)) + throw new ScholixException("The method requires one of the following parameters: sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider"); - - final int currentPage = page!= null? page : 0; - - System.out.println(currentPage); - Pair> scholixResult = manager.linksFromPid(ScholixAPIVersion.V2, linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, harvestedAfter, currentPage); - - final PageResultType pageResult = new PageResultType(); - pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10); - pageResult.setTotalLinks(scholixResult.getLeft().intValue()); - pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList())); - - return pageResult; + try { + final int currentPage = page != null ? page : 0; + Pair> scholixResult = manager.linksFromPid(ScholixAPIVersion.V2, linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, harvestedAfter, currentPage); + final PageResultType pageResult = new PageResultType(); + pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10); + pageResult.setTotalLinks(scholixResult.getLeft().intValue()); + pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList())); + return pageResult; + } catch (Throwable e) { + throw new ScholixException("Error on requesting url ", e); + } } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchClientFactory.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchClientFactory.java new file mode 100644 index 00000000..85893680 --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchClientFactory.java @@ -0,0 +1,66 @@ +package eu.dnetlib.scholix.api.index; + +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.client.RestClients; + +public class ElasticSearchClientFactory implements PooledObjectFactory { + + private ElasticSearchProperties elasticSearchProperties; + + + public ElasticSearchClientFactory(final ElasticSearchProperties elasticSearchProperties){ + this.elasticSearchProperties = elasticSearchProperties; + + } + + public PooledObject makeObject() throws Exception { + + + + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(elasticSearchProperties.getClusterNodes().split(",")) + .withConnectTimeout(elasticSearchProperties.getConnectionTimeout()) + .withSocketTimeout(elasticSearchProperties.getSocketTimeout()) + .build(); + RestHighLevelClient cc = RestClients.create(clientConfiguration).rest(); + + + return new DefaultPooledObject(cc); + } + + public void destroyObject(PooledObject pooledObject) throws Exception { + RestHighLevelClient client = pooledObject.getObject(); + if(client!=null&&client.ping(RequestOptions.DEFAULT)){ + try { + client.close(); + }catch (Exception e){ + //ignore + } + } + } + + public boolean validateObject(PooledObject pooledObject) { + RestHighLevelClient client = pooledObject.getObject(); + try { + return client.ping(RequestOptions.DEFAULT); + }catch(Exception e){ + return false; + } + } + + public void activateObject(PooledObject pooledObject) throws Exception { + RestHighLevelClient client = pooledObject.getObject(); + boolean response = client.ping(RequestOptions.DEFAULT); + } + + public void passivateObject(PooledObject pooledObject) throws Exception { + //nothing + } + + +} \ No newline at end of file diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchPool.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchPool.java new file mode 100644 index 00000000..6bf79fd3 --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchPool.java @@ -0,0 +1,19 @@ +package eu.dnetlib.scholix.api.index; + +import org.elasticsearch.client.RestHighLevelClient; + +public class ElasticSearchPool extends Pool { + + private final ElasticSearchProperties elasticSearchProperties; + + public ElasticSearchPool(ElasticSearchProperties elasticSearchProperties){ + super(elasticSearchProperties, new ElasticSearchClientFactory(elasticSearchProperties)); + this.elasticSearchProperties = elasticSearchProperties; + } + + public ElasticSearchProperties getElasticSearchProperties() { + return elasticSearchProperties; + } +} + + diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java index 38e7016e..15f53154 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java @@ -1,5 +1,6 @@ package eu.dnetlib.scholix.api.index; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -10,7 +11,7 @@ import javax.validation.constraints.NotNull; */ @Component("elasticSearchProperties") @ConfigurationProperties(prefix = "scholix.elastic") -public class ElasticSearchProperties { +public class ElasticSearchProperties extends GenericObjectPoolConfig { @NotNull private String clusterNodes; diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/Pool.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/Pool.java new file mode 100644 index 00000000..53647ac7 --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/Pool.java @@ -0,0 +1,138 @@ +package eu.dnetlib.scholix.api.index; + +import eu.dnetlib.scholix.api.ScholixException; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; + +public class Pool implements Cloneable { + + protected GenericObjectPool internalPool ; + + public Pool(){ + super(); + } + + public Pool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory factory){ + initPool(poolConfig, factory); + } + + public void initPool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory factory) { + + if (this.internalPool != null) { + try { + closeInternalPool(); + } catch (Exception e) { + } + } + + this.internalPool = new GenericObjectPool(factory, poolConfig); + } + + protected void closeInternalPool() throws ScholixException { + try { + internalPool.close(); + } catch (Exception e) { + throw new ScholixException("Could not destroy the pool", e); + } + } + + public T getResource() throws ScholixException { + try { + return internalPool.borrowObject(); + } catch (Exception e) { + throw new ScholixException("Could not get a resource from the pool", e); + } + } + + + public void returnResource(final T resource) throws ScholixException { + if (resource != null) { + returnResourceObject(resource); + } + } + + private void returnResourceObject(final T resource) throws ScholixException { + if (resource == null) { + return; + } + try { + internalPool.returnObject(resource); + } catch (Exception e) { + throw new ScholixException("Could not return the resource to the pool", e); + } + } + + public void returnBrokenResource(final T resource) throws ScholixException { + if (resource != null) { + returnBrokenResourceObject(resource); + } + } + + private void returnBrokenResourceObject(T resource) throws ScholixException { + try { + internalPool.invalidateObject(resource); + } catch (Exception e) { + throw new ScholixException("Could not return the resource to the pool", e); + } + } + + public void destroy() throws ScholixException { + closeInternalPool(); + } + + + public int getNumActive() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getNumActive(); + } + + public int getNumIdle() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getNumIdle(); + } + + public int getNumWaiters() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getNumWaiters(); + } + + public long getMeanBorrowWaitTimeMillis() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getMeanBorrowWaitTimeMillis(); + } + + public long getMaxBorrowWaitTimeMillis() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getMaxBorrowWaitTimeMillis(); + } + + private boolean poolInactive() { + return this.internalPool == null || this.internalPool.isClosed(); + } + + public void addObjects(int count) throws Exception { + try { + for (int i = 0; i < count; i++) { + this.internalPool.addObject(); + } + } catch (Exception e) { + throw new Exception("Error trying to add idle objects", e); + } + } +} \ No newline at end of file diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java index daefcab3..c958aa53 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java @@ -11,13 +11,14 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; -import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; @@ -45,7 +46,7 @@ public class ScholixIndexManager { * The Elasticsearch template. */ @Autowired - ElasticsearchOperations elasticsearchTemplate; + ElasticSearchPool connectionPool; /** * The My counter. @@ -205,12 +206,13 @@ public class ScholixIndexManager { .withPageable(PageRequest.of(page,10)) .build(); - long tt = elasticsearchTemplate.count(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); - System.out.println(tt); + RestHighLevelClient client = connectionPool.getResource(); + ElasticsearchRestTemplate template = new ElasticsearchRestTemplate(client); + long tt = template.count(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); - SearchHits scholixRes = elasticsearchTemplate.search(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + SearchHits scholixRes = template.search(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); - System.out.println("SIZE OF HITS ->"+scholixRes.getSearchHits().size()); + connectionPool.returnResource(client); return new ImmutablePair<>(tt,scholixRes.stream().map(SearchHit::getContent).collect(Collectors.toList())); } diff --git a/pom.xml b/pom.xml index 34ea20d2..d3e31c1e 100644 --- a/pom.xml +++ b/pom.xml @@ -142,6 +142,13 @@ test + + org.apache.commons + commons-pool2 + 2.11.1 + + +