D-Net
/
dnet-applications
23
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implemented first test of connection to index elasticSearch

This commit is contained in:
Sandro La Bruzzo 2022-02-04 15:52:49 +01:00
parent 04ea22b244
commit f505f379af
8 changed files with 166 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/scholexplorer-api/pom.xml
View File

@ -15,12 +15,20 @@
<dependencies>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-schemas</artifactId>
</dependency>
</dependencies>

48
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java
View File

@ -1,12 +1,19 @@
package eu.dnetlib.scholix.api;
import eu.dnetlib.common.app.AbstractDnetApp;
import io.micrometer.core.aop.TimedAspect;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
@ -15,11 +22,13 @@ import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
import java.time.Duration;
@SpringBootApplication
@EnableSwagger2
@EnableCaching
@EnableScheduling
//@EntityScan("eu.dnetlib.scholix.api.controller")
@ComponentScan(basePackages = "eu.dnetlib")
public class MainApplication extends AbstractDnetApp {
@Value("${dhp.swagger.api.host}")
@ -38,6 +47,41 @@ public class MainApplication extends AbstractDnetApp {
SpringApplication.run(MainApplication.class, args);
}
@Bean
public TimedAspect timedAspect(MeterRegistry meterRegistry) {
double scale = 1000000;
MeterFilter mf = new MeterFilter() {
@Override
public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) {
if (id.getName().startsWith("scholix")) {
return DistributionStatisticConfig.builder()
.percentiles(0.20, 0.50, 0.75,0.95)
.percentilesHistogram(false)
.serviceLevelObjectives(5.0*scale, 10.0*scale, 250*scale, 500*scale, 750*scale ,1000.0*scale,2500.0*scale,5000.0*scale,7500.0*scale,10000.0*scale )
// .minimumExpectedValue(0.008)
//// .maximumExpectedValue(1000000.0)
.build()
.merge(config);
}
return config;
}
};
meterRegistry.config().meterFilter(mf);
return new TimedAspect(meterRegistry);
}
@Override
protected void configSwagger(final Docket docket) {
docket

32
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java Normal file
View File

@ -0,0 +1,32 @@
package eu.dnetlib.scholix.api;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
import java.time.Duration;
@Configuration
public class RestClientConfig extends AbstractElasticsearchConfiguration {
@Override
@Bean
public RestHighLevelClient elasticsearchClient() {
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo("localhost:9200")
.withSocketTimeout(Duration.ofSeconds(60))
.build();
return RestClients.create(clientConfiguration).rest();
}
}

4
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/DatasourceV1.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.scholix.api.model.v1.LinkPublisher;
import io.micrometer.core.annotation.Timed;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import org.springframework.web.bind.annotation.GetMapping;
@ -19,10 +20,11 @@ import java.util.List;
})
public class DatasourceV1 extends AbstractDnetController {
@Timed(value = "scholix.v1.datasources", description = "Time taken to return all datasources on Version 1.0 of Scholix")
@Operation(
summary = "Get all Datasources",
description = "returns a list of all datasources")
@GetMapping("/v1/listDatasources")
@GetMapping("/listDatasources")
public List<LinkPublisher> getDatasources() {
return Arrays.asList(
new LinkPublisher().name("pippo").totalRelationships(30),

52
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java
View File

@ -2,18 +2,34 @@ package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.sx.scholix.ScholixIdentifier;
import eu.dnetlib.scholix.api.model.v2.PageResultType;
import io.micrometer.core.annotation.Timed;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.Query;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.validation.constraints.NotNull;
import java.util.Arrays;
import java.util.Collections;
@RestController
@RequestMapping("/v2")
@ -22,6 +38,13 @@ import java.util.Arrays;
})
public class ScholixControllerV2 extends AbstractDnetController {
@Autowired
ElasticsearchOperations elasticsearchTemplate;
@Timed(value = "scholix.v2.links", description = "Time taken to return links on Version 2.0 of Scholix")
@ApiOperation("Get Scholix Links")
@GetMapping("/Links")
public PageResultType links(
@ -46,9 +69,28 @@ public class ScholixControllerV2 extends AbstractDnetController {
@Parameter(in = ParameterIn.QUERY,
description = "Filter scholix Links having collected after this date") String harvestedAfter,
@Parameter(in = ParameterIn.QUERY,
description = "select page of result") Integer page) {
description = "select page of result") Integer page) throws Exception {
return new PageResultType();
NestedQueryBuilder nb = new NestedQueryBuilder("source.identifier", new TermQueryBuilder("source.identifier.schema", "pdb"), ScoreMode.None);
NativeSearchQuery doiquery = new NativeSearchQueryBuilder()
.withQuery(nb)
.withPageable(PageRequest.of(1,10))
.build();
long tt = elasticsearchTemplate.count(doiquery, Scholix.class, IndexCoordinates.of("dli_scholix"));
System.out.println(tt);
SearchHits<Scholix> result = elasticsearchTemplate.search(doiquery, Scholix.class, IndexCoordinates.of("dli_scholix"));
final PageResultType pageResult = new PageResultType();
pageResult.setTotalPages((int) tt );
return pageResult;
}

13
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/metrics/RequestCounter.java Normal file
View File

@ -0,0 +1,13 @@
package eu.dnetlib.scholix.api.metrics;
import eu.dnetlib.common.metrics.MetricInfo;
import org.springframework.stereotype.Component;
@Component("scholexplorer_test")
public class RequestCounter implements MetricInfo {
@Override
public double obtainValue() {
return 0L;
}
}

45
apps/scholexplorer-api/src/main/resources/application.properties
View File

@ -9,40 +9,15 @@ maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.
#
#spring.thymeleaf.cache=false
#
#management.endpoints.web.exposure.include = prometheus,health
#management.endpoints.web.base-path = /
#management.endpoints.web.path-mapping.prometheus = metrics
#management.endpoints.web.path-mapping.health = health
#
#spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager
#spring.datasource.username=
#spring.datasource.password=
#
#spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
#
## Hibernate ddl auto (create, create-drop, validate, update)
#spring.jpa.hibernate.ddl-auto = validate
#spring.jpa.properties.hibernate.hbm2dll.extra_physical_table_types = MATERIALIZED VIEW
#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
#spring.jpa.open-in-view=true
#
## logs
#logging.level.io.swagger.models.parameters.AbstractSerializableParameter = error
#
## Hadoop
#dhp.mdstore-manager.hadoop.cluster = GARR
#dhp.mdstore-manager.hdfs.base-path = /data/dnet.dev/mdstore
#dhp.mdstore-manager.hadoop.user = dnet.dev
#
##dhp.mdstore-manager.hadoop.zeppelin.base-url = https://iis-cdh5-test-gw.ocean.icm.edu.pl/zeppelin
##dhp.mdstore-manager.hadoop.zeppelin.login =
##dhp.mdstore-manager.hadoop.zeppelin.password =
#dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org
#dhp.mdstore-manager.hadoop.zeppelin.login =
#dhp.mdstore-manager.hadoop.zeppelin.password =
#
#dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
#
#dhp.mdstore-manager.inspector.records.max = 1000
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
management.metrics.distribution.percentiles-histogram.http.server.requests=false
management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999

8
pom.xml
View File

@ -232,6 +232,12 @@
<artifactId>springfox-boot-starter</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<version>4.3.1</version>
</dependency>
<!-- Hadoop -->
<dependency>
@ -390,7 +396,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
<java.version>1.8</java.version>
<dhp-schemas-version>2.3.6</dhp-schemas-version>
<dhp-schemas-version>2.10.24</dhp-schemas-version>
<apache.solr.version>7.1.0</apache.solr.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version>
<springfox-version>2.8.0</springfox-version>