diff --git a/apps/scholexplorer-api/pom.xml b/apps/scholexplorer-api/pom.xml index 7ce034c1..04b31ca5 100644 --- a/apps/scholexplorer-api/pom.xml +++ b/apps/scholexplorer-api/pom.xml @@ -15,12 +15,20 @@ + + org.springframework.data + spring-data-elasticsearch + org.springframework.boot spring-boot-starter-test test + + eu.dnetlib.dhp + dhp-schemas + diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java index b7761633..500691c6 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java @@ -1,12 +1,19 @@ package eu.dnetlib.scholix.api; import eu.dnetlib.common.app.AbstractDnetApp; +import io.micrometer.core.aop.TimedAspect; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableScheduling; import springfox.documentation.builders.ApiInfoBuilder; import springfox.documentation.builders.RequestHandlerSelectors; @@ -15,11 +22,13 @@ import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; import springfox.documentation.swagger2.annotations.EnableSwagger2; +import java.time.Duration; + @SpringBootApplication @EnableSwagger2 @EnableCaching @EnableScheduling -//@EntityScan("eu.dnetlib.scholix.api.controller") +@ComponentScan(basePackages = "eu.dnetlib") public class MainApplication extends AbstractDnetApp { @Value("${dhp.swagger.api.host}") @@ -38,6 +47,41 @@ public class MainApplication extends AbstractDnetApp { SpringApplication.run(MainApplication.class, args); } + + @Bean + public TimedAspect timedAspect(MeterRegistry meterRegistry) { + + double scale = 1000000; + MeterFilter mf = new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + if (id.getName().startsWith("scholix")) { + + return DistributionStatisticConfig.builder() + .percentiles(0.20, 0.50, 0.75,0.95) + .percentilesHistogram(false) + .serviceLevelObjectives(5.0*scale, 10.0*scale, 250*scale, 500*scale, 750*scale ,1000.0*scale,2500.0*scale,5000.0*scale,7500.0*scale,10000.0*scale ) + +// .minimumExpectedValue(0.008) +//// .maximumExpectedValue(1000000.0) + .build() + .merge(config); + } + return config; + } + }; + + + meterRegistry.config().meterFilter(mf); + + + + return new TimedAspect(meterRegistry); + } + + + + @Override protected void configSwagger(final Docket docket) { docket diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java new file mode 100644 index 00000000..2abcd0f3 --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java @@ -0,0 +1,32 @@ +package eu.dnetlib.scholix.api; + +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.client.RestClients; +import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration; + +import java.time.Duration; + +@Configuration +public class RestClientConfig extends AbstractElasticsearchConfiguration { + + @Override + @Bean + public RestHighLevelClient elasticsearchClient() { + + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo("localhost:9200") + .withSocketTimeout(Duration.ofSeconds(60)) + .build(); + + return RestClients.create(clientConfiguration).rest(); + + } + + + + + +} diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/DatasourceV1.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/DatasourceV1.java index 007a6caf..684ec92f 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/DatasourceV1.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/DatasourceV1.java @@ -2,6 +2,7 @@ package eu.dnetlib.scholix.api.controller; import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.scholix.api.model.v1.LinkPublisher; +import io.micrometer.core.annotation.Timed; import io.swagger.annotations.Api; import io.swagger.v3.oas.annotations.Operation; import org.springframework.web.bind.annotation.GetMapping; @@ -19,10 +20,11 @@ import java.util.List; }) public class DatasourceV1 extends AbstractDnetController { + @Timed(value = "scholix.v1.datasources", description = "Time taken to return all datasources on Version 1.0 of Scholix") @Operation( summary = "Get all Datasources", description = "returns a list of all datasources") - @GetMapping("/v1/listDatasources") + @GetMapping("/listDatasources") public List getDatasources() { return Arrays.asList( new LinkPublisher().name("pippo").totalRelationships(30), diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java index ee5fef8a..e6fd4f84 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java @@ -2,18 +2,34 @@ package eu.dnetlib.scholix.api.controller; import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixIdentifier; import eu.dnetlib.scholix.api.model.v2.PageResultType; +import io.micrometer.core.annotation.Timed; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; +import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; +import org.springframework.data.elasticsearch.core.query.Query; +import org.elasticsearch.index.query.QueryBuilders; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.validation.constraints.NotNull; -import java.util.Arrays; +import java.util.Collections; @RestController @RequestMapping("/v2") @@ -22,6 +38,13 @@ import java.util.Arrays; }) public class ScholixControllerV2 extends AbstractDnetController { + @Autowired + ElasticsearchOperations elasticsearchTemplate; + + + + + @Timed(value = "scholix.v2.links", description = "Time taken to return links on Version 2.0 of Scholix") @ApiOperation("Get Scholix Links") @GetMapping("/Links") public PageResultType links( @@ -46,9 +69,28 @@ public class ScholixControllerV2 extends AbstractDnetController { @Parameter(in = ParameterIn.QUERY, description = "Filter scholix Links having collected after this date") String harvestedAfter, @Parameter(in = ParameterIn.QUERY, - description = "select page of result") Integer page) { + description = "select page of result") Integer page) throws Exception { - return new PageResultType(); + + NestedQueryBuilder nb = new NestedQueryBuilder("source.identifier", new TermQueryBuilder("source.identifier.schema", "pdb"), ScoreMode.None); + + + + NativeSearchQuery doiquery = new NativeSearchQueryBuilder() + .withQuery(nb) + .withPageable(PageRequest.of(1,10)) + .build(); + + long tt = elasticsearchTemplate.count(doiquery, Scholix.class, IndexCoordinates.of("dli_scholix")); + System.out.println(tt); + + SearchHits result = elasticsearchTemplate.search(doiquery, Scholix.class, IndexCoordinates.of("dli_scholix")); + + + final PageResultType pageResult = new PageResultType(); + pageResult.setTotalPages((int) tt ); + + return pageResult; } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/metrics/RequestCounter.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/metrics/RequestCounter.java new file mode 100644 index 00000000..56c9ed2c --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/metrics/RequestCounter.java @@ -0,0 +1,13 @@ +package eu.dnetlib.scholix.api.metrics; + +import eu.dnetlib.common.metrics.MetricInfo; +import org.springframework.stereotype.Component; + + +@Component("scholexplorer_test") +public class RequestCounter implements MetricInfo { + @Override + public double obtainValue() { + return 0L; + } +} diff --git a/apps/scholexplorer-api/src/main/resources/application.properties b/apps/scholexplorer-api/src/main/resources/application.properties index 38401105..6bcb8ef7 100644 --- a/apps/scholexplorer-api/src/main/resources/application.properties +++ b/apps/scholexplorer-api/src/main/resources/application.properties @@ -9,40 +9,15 @@ maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom. # #spring.thymeleaf.cache=false # -#management.endpoints.web.exposure.include = prometheus,health -#management.endpoints.web.base-path = / -#management.endpoints.web.path-mapping.prometheus = metrics -#management.endpoints.web.path-mapping.health = health -# -#spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager -#spring.datasource.username= -#spring.datasource.password= -# -#spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect -# -## Hibernate ddl auto (create, create-drop, validate, update) -#spring.jpa.hibernate.ddl-auto = validate -#spring.jpa.properties.hibernate.hbm2dll.extra_physical_table_types = MATERIALIZED VIEW -#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true -#spring.jpa.open-in-view=true -# -## logs -#logging.level.io.swagger.models.parameters.AbstractSerializableParameter = error -# -## Hadoop -#dhp.mdstore-manager.hadoop.cluster = GARR -#dhp.mdstore-manager.hdfs.base-path = /data/dnet.dev/mdstore -#dhp.mdstore-manager.hadoop.user = dnet.dev -# -##dhp.mdstore-manager.hadoop.zeppelin.base-url = https://iis-cdh5-test-gw.ocean.icm.edu.pl/zeppelin -##dhp.mdstore-manager.hadoop.zeppelin.login = -##dhp.mdstore-manager.hadoop.zeppelin.password = -#dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org -#dhp.mdstore-manager.hadoop.zeppelin.login = -#dhp.mdstore-manager.hadoop.zeppelin.password = -# -#dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager -# -#dhp.mdstore-manager.inspector.records.max = 1000 +management.endpoints.web.exposure.include = prometheus,health +management.endpoints.web.base-path = / +management.endpoints.web.path-mapping.prometheus = metrics +management.endpoints.web.path-mapping.health = health + +management.metrics.distribution.percentiles-histogram.http.server.requests=false +management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms +management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999 + + diff --git a/pom.xml b/pom.xml index 98f32305..8f02c9d7 100644 --- a/pom.xml +++ b/pom.xml @@ -232,6 +232,12 @@ springfox-boot-starter 3.0.0 + + + org.springframework.data + spring-data-elasticsearch + 4.3.1 + @@ -390,7 +396,7 @@ UTF-8 3.6.0 1.8 - 2.3.6 + 2.10.24 7.1.0 3.4.2 2.8.0