scholexplorer #4
|
|
@ -5,6 +5,7 @@
|
||||||
*.iml
|
*.iml
|
||||||
*.ipr
|
*.ipr
|
||||||
*.iws
|
*.iws
|
||||||
|
*.java-version
|
||||||
*~
|
*~
|
||||||
/**/*.sh
|
/**/*.sh
|
||||||
/**/my_application.properties
|
/**/my_application.properties
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>apps</artifactId>
|
<artifactId>apps</artifactId>
|
||||||
<version>3.2.5-SNAPSHOT</version>
|
<version>3.2.5-SNAPSHOT</version>
|
||||||
<relativePath>../</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>apps</artifactId>
|
<artifactId>apps</artifactId>
|
||||||
<version>3.2.5-SNAPSHOT</version>
|
<version>3.2.5-SNAPSHOT</version>
|
||||||
<relativePath>../</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>apps</artifactId>
|
<artifactId>apps</artifactId>
|
||||||
<version>3.2.5-SNAPSHOT</version>
|
<version>3.2.5-SNAPSHOT</version>
|
||||||
<relativePath>../</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>dnet-applications</artifactId>
|
<artifactId>dnet-applications</artifactId>
|
||||||
<version>3.2.5-SNAPSHOT</version>
|
<version>3.2.5-SNAPSHOT</version>
|
||||||
<relativePath>../</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
@ -16,6 +16,7 @@
|
||||||
<module>dhp-broker-public-application</module>
|
<module>dhp-broker-public-application</module>
|
||||||
<module>dhp-mdstore-manager</module>
|
<module>dhp-mdstore-manager</module>
|
||||||
<module>dnet-orgs-database-application</module>
|
<module>dnet-orgs-database-application</module>
|
||||||
|
<module>scholexplorer-api</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<parent>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>apps</artifactId>
|
||||||
|
<version>3.2.5-SNAPSHOT</version>
|
||||||
|
<relativePath>../pom.xml</relativePath>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
<artifactId>scholexplorer-api</artifactId>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.data</groupId>
|
||||||
|
<artifactId>spring-data-elasticsearch</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-test</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-schemas</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
<artifactId>commons-pool2</artifactId>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-help-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,124 @@
|
||||||
|
package eu.dnetlib.scholix.api;
|
||||||
|
|
||||||
|
import eu.dnetlib.common.app.AbstractDnetApp;
|
||||||
|
import io.micrometer.core.aop.TimedAspect;
|
||||||
|
import io.micrometer.core.instrument.*;
|
||||||
|
import io.micrometer.core.instrument.config.MeterFilter;
|
||||||
|
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.boot.SpringApplication;
|
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
import org.springframework.cache.annotation.EnableCaching;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.ComponentScan;
|
||||||
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||||
|
import springfox.documentation.builders.ApiInfoBuilder;
|
||||||
|
import springfox.documentation.builders.RequestHandlerSelectors;
|
||||||
|
import springfox.documentation.service.ApiInfo;
|
||||||
|
import springfox.documentation.spi.DocumentationType;
|
||||||
|
import springfox.documentation.spring.web.plugins.Docket;
|
||||||
|
import springfox.documentation.swagger2.annotations.EnableSwagger2;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@SpringBootApplication
|
||||||
|
@EnableSwagger2
|
||||||
|
@EnableCaching
|
||||||
|
@EnableScheduling
|
||||||
|
@ComponentScan(basePackages = "eu.dnetlib")
|
||||||
|
public class MainApplication extends AbstractDnetApp {
|
||||||
|
|
||||||
|
private double scale = 1000000000;
|
||||||
|
|
||||||
|
private double[] histogramValues =new double[]{.005 * scale, .01 * scale, .25 * scale, .5 * scale, .75 * scale, scale, 2.5 * scale, 5.0 * scale, 7.5 * scale, 10.0 * scale};
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Value("${dhp.swagger.api.host}")
|
||||||
|
private String swaggetHost;
|
||||||
|
|
||||||
|
@Value("${dhp.swagger.api.basePath}")
|
||||||
|
private String swaggerPath;
|
||||||
|
|
||||||
|
|
||||||
|
public static void main(final String[] args) {
|
||||||
|
SpringApplication.run(MainApplication.class, args);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public TaggedCounter myCounter(MeterRegistry meterRegistry) {
|
||||||
|
|
||||||
|
return new TaggedCounter(ScholixAPIConstants.SCHOLIX_MANAGER_COUNTER_NAME, ScholixAPIConstants.SCHOLIX_MANAGER_TAG_NAME,meterRegistry);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public TimedAspect timedAspect(MeterRegistry meterRegistry) {
|
||||||
|
MeterFilter mf = new MeterFilter() {
|
||||||
|
@Override
|
||||||
|
public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) {
|
||||||
|
if (id.getName().startsWith(ScholixAPIConstants.SCHOLIX_COUNTER_PREFIX)) {
|
||||||
|
|
||||||
|
return DistributionStatisticConfig.builder()
|
||||||
|
.percentilesHistogram(false)
|
||||||
|
.serviceLevelObjectives( histogramValues)
|
||||||
|
.build()
|
||||||
|
.merge(config);
|
||||||
|
}
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
meterRegistry.config().meterFilter(mf);
|
||||||
|
return new TimedAspect(meterRegistry);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void configSwagger(final Docket docket) {
|
||||||
|
docket
|
||||||
|
.host(swaggetHost)
|
||||||
|
.pathMapping(swaggerPath)
|
||||||
|
.groupName(ScholixAPIConstants.API_V1_NAME)
|
||||||
|
.select()
|
||||||
|
.apis(RequestHandlerSelectors.any())
|
||||||
|
.paths(p -> p.startsWith("/v1"))
|
||||||
|
.build()
|
||||||
|
.apiInfo(new ApiInfoBuilder()
|
||||||
|
.title(ScholixAPIConstants.API_V1_NAME)
|
||||||
|
.description(ScholixAPIConstants.API_DESCRIPTION)
|
||||||
|
.version("1.0")
|
||||||
|
.contact(ApiInfo.DEFAULT_CONTACT)
|
||||||
|
.license("Apache 2.0")
|
||||||
|
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Bean (name = "SpringDocketv2")
|
||||||
|
public Docket v2Docket() {
|
||||||
|
final Docket docket = new Docket(DocumentationType.SWAGGER_2);
|
||||||
|
docket
|
||||||
|
.host(swaggetHost)
|
||||||
|
.pathMapping(swaggerPath)
|
||||||
|
.groupName(ScholixAPIConstants.API_V2_NAME)
|
||||||
|
.select()
|
||||||
|
.apis(RequestHandlerSelectors.any())
|
||||||
|
.paths(p -> p.startsWith("/v2"))
|
||||||
|
.build()
|
||||||
|
.apiInfo(new ApiInfoBuilder()
|
||||||
|
.title(ScholixAPIConstants.API_V2_NAME)
|
||||||
|
.description(ScholixAPIConstants.API_DESCRIPTION)
|
||||||
|
.version("2.0")
|
||||||
|
.contact(ApiInfo.DEFAULT_CONTACT)
|
||||||
|
.license("Apache 2.0")
|
||||||
|
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
|
||||||
|
.build());
|
||||||
|
|
||||||
|
return docket;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
package eu.dnetlib.scholix.api;
|
||||||
|
|
||||||
|
import eu.dnetlib.scholix.api.index.ElasticSearchPool;
|
||||||
|
import eu.dnetlib.scholix.api.index.ElasticSearchProperties;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
public class RestClientConfig {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private ElasticSearchProperties elasticSearchProperties;
|
||||||
|
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public ElasticSearchPool connectionPool() {
|
||||||
|
|
||||||
|
elasticSearchProperties.setMaxIdle(5);
|
||||||
|
elasticSearchProperties.setMaxTotal(10);
|
||||||
|
ElasticSearchPool pool = new ElasticSearchPool(elasticSearchProperties);
|
||||||
|
return pool;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
package eu.dnetlib.scholix.api;
|
||||||
|
|
||||||
|
public class ScholixAPIConstants {
|
||||||
|
|
||||||
|
public static final String API_V1_NAME = "Scholexplorer API V1.0";
|
||||||
|
public static final String API_V2_NAME = "Scholexplorer API V2.0";
|
||||||
|
|
||||||
|
public static String API_DESCRIPTION =" <p style=\"text-align:center;\"><img src=\"/logo.png\" alt=\"ScholeXplorer\"> </p>" +
|
||||||
|
"The Scholix Swagger API allows clients to run REST queries over the Scholexplorer index in order to fetch links matching given criteria. In the current version, clients can search for:" +
|
||||||
|
"<ul><li>Links whose source object has a given PID or PID type</li>" +
|
||||||
|
"<li>Links whose source object has been published by a given data source (\"data source as publisher\")</li>" +
|
||||||
|
"<li>Links that were collected from a given data source (\"data source as provider\").</li></ul>";
|
||||||
|
|
||||||
|
|
||||||
|
public static String SCHOLIX_MANAGER_COUNTER_NAME= "scholixLinkCounter";
|
||||||
|
public static final String SCHOLIX_MANAGER_TAG_NAME = "links";
|
||||||
|
|
||||||
|
public static String SCHOLIX_COUNTER_PREFIX = "scholix";
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package eu.dnetlib.scholix.api;
|
||||||
|
|
||||||
|
public enum ScholixAPIVersion {
|
||||||
|
|
||||||
|
V1,
|
||||||
|
V2
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
package eu.dnetlib.scholix.api;
|
||||||
|
|
||||||
|
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||||
|
|
||||||
|
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
|
||||||
|
public class ScholixException extends Exception{
|
||||||
|
|
||||||
|
private static final long serialVersionUID = -3414428892721711308L;
|
||||||
|
|
||||||
|
|
||||||
|
public ScholixException() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public ScholixException(String message, Throwable cause, boolean enableSuppression,
|
||||||
|
boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ScholixException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ScholixException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ScholixException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
package eu.dnetlib.scholix.api;
|
||||||
|
|
||||||
|
import io.micrometer.core.instrument.Counter;
|
||||||
|
|
||||||
|
import io.micrometer.core.instrument.MeterRegistry;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
|
||||||
|
public class TaggedCounter {
|
||||||
|
|
||||||
|
private String name;
|
||||||
|
private String tagName;
|
||||||
|
private MeterRegistry registry;
|
||||||
|
private Map<String, Counter> counters = new HashMap<>();
|
||||||
|
|
||||||
|
|
||||||
|
public TaggedCounter(String name, String tagName, MeterRegistry registry) {
|
||||||
|
this.name = name;
|
||||||
|
this.tagName = tagName;
|
||||||
|
this.registry = registry;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void increment(String tagValue){
|
||||||
|
Counter counter = counters.get(tagValue);
|
||||||
|
if(counter == null) {
|
||||||
|
counter = Counter.builder(name).tags(tagName, tagValue).register(registry);
|
||||||
|
counters.put(tagValue, counter);
|
||||||
|
}
|
||||||
|
counter.increment();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
package eu.dnetlib.scholix.api.controller;
|
||||||
|
|
||||||
|
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.api.model.v1.LinkPublisher;
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
|
||||||
|
|
||||||
|
import io.micrometer.core.annotation.Timed;
|
||||||
|
import io.swagger.annotations.Api;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/v1")
|
||||||
|
@Api(tags = {
|
||||||
|
"Datasources"
|
||||||
|
})
|
||||||
|
public class DatasourceV1 extends AbstractDnetController {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private ScholixIndexManager manager;
|
||||||
|
|
||||||
|
@Timed(value = "scholix.v1.datasources", description = "Time taken to return all datasources on Version 1.0 of Scholix")
|
||||||
|
@Operation(
|
||||||
|
summary = "Get all Datasources",
|
||||||
|
description = "returns a list of all datasources")
|
||||||
|
@GetMapping("/listDatasources")
|
||||||
|
public List<LinkPublisher> getDatasources() throws ScholixException {
|
||||||
|
|
||||||
|
|
||||||
|
final List<Pair<String, Long>> result = manager.totalLinksByProvider(null);
|
||||||
|
|
||||||
|
if (result == null)
|
||||||
|
return new ArrayList<>();
|
||||||
|
|
||||||
|
return result.stream().map(p -> new LinkPublisher().name(p.getKey()).totalRelationships(p.getValue().intValue())).collect(Collectors.toList());
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
package eu.dnetlib.scholix.api.controller;
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType;
|
||||||
|
import io.swagger.annotations.Api;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.enums.ParameterIn;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/v2")
|
||||||
|
@Api(tags = {
|
||||||
|
"LinkProvider : Operation related to the Link Provider"
|
||||||
|
})
|
||||||
|
public class LinkProviderV2 {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
ScholixIndexManager manager;
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Get all Link Providers",
|
||||||
|
description = "Return a list of link provider and relative number of relations")
|
||||||
|
@GetMapping("/LinkProvider")
|
||||||
|
public List<LinkProviderType> getLinkProviders(
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "Filter the link provider name") @RequestParam(required = false) String name
|
||||||
|
) throws ScholixException {
|
||||||
|
|
||||||
|
List<Pair<String, Long>> result = manager.totalLinksByProvider(name);
|
||||||
|
|
||||||
|
if (result==null)
|
||||||
|
return new ArrayList<>();
|
||||||
|
|
||||||
|
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
package eu.dnetlib.scholix.api.controller;
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType;
|
||||||
|
import io.swagger.annotations.Api;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.enums.ParameterIn;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/v2/LinkPublisher")
|
||||||
|
@Api(tags = {
|
||||||
|
"LinkPublisher : Operation related to the Link Publisher"
|
||||||
|
})
|
||||||
|
public class LinkPublisherV2 {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
ScholixIndexManager manager;
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Get All Publishers that provide source object",
|
||||||
|
description = "Return a List of all Publishers that provide source objects in Scholix " +
|
||||||
|
"links and the total number of links where the source object comes from this publisher")
|
||||||
|
@GetMapping("/inSource")
|
||||||
|
public List<LinkProviderType> getInSource(
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) String name
|
||||||
|
) throws ScholixException {
|
||||||
|
List<Pair<String, Long>> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.source,name);
|
||||||
|
|
||||||
|
if (result==null)
|
||||||
|
return new ArrayList<>();
|
||||||
|
|
||||||
|
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Get All Publishers that provide target object",
|
||||||
|
description = "Return a List of all Publishers that provide source objects in Scholix " +
|
||||||
|
"links and the total number of links where the target object comes from this publisher")
|
||||||
|
@GetMapping("/inTarget")
|
||||||
|
public List<LinkProviderType> getInTarget(
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) String name
|
||||||
|
) throws ScholixException {
|
||||||
|
List<Pair<String, Long>> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.target,name);
|
||||||
|
|
||||||
|
if (result==null)
|
||||||
|
return new ArrayList<>();
|
||||||
|
|
||||||
|
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
package eu.dnetlib.scholix.api.controller;
|
||||||
|
|
||||||
|
|
||||||
|
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.api.model.v1.ScholixV1;
|
||||||
|
import io.micrometer.core.annotation.Timed;
|
||||||
|
import io.swagger.annotations.Api;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.enums.ParameterIn;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import javax.validation.constraints.NotNull;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/v1")
|
||||||
|
@Api(tags = {"Scholix"})
|
||||||
|
public class ScholixControllerV1 extends AbstractDnetController {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
ScholixIndexManager manager;
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Get all Scholix relation collected from a publisher",
|
||||||
|
description = "return a list of scholix object published from a specific publisher"
|
||||||
|
)
|
||||||
|
@GetMapping("/linksFromPublisher")
|
||||||
|
@Timed(value = "scholix.v1.linksFromPublisher", description = "Time taken to return links on Version 1.0 of Scholix collected from a publisher")
|
||||||
|
public List<ScholixV1> linksFromPublisher(
|
||||||
|
@Parameter(
|
||||||
|
in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships collected from a publisher",
|
||||||
|
schema = @Schema(), required = true) String publisher,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "The page number") @RequestParam(required = false) Integer page
|
||||||
|
) throws ScholixException {
|
||||||
|
|
||||||
|
final int currentPage = page != null ? page : 0;
|
||||||
|
|
||||||
|
Pair<Long, List<Scholix>> scholixResult =manager.linksFromPid(null,null,null,publisher,
|
||||||
|
null,null,null,null,null,null, currentPage
|
||||||
|
);
|
||||||
|
List<Scholix> scholixData = scholixResult.getValue();
|
||||||
|
if (scholixData== null)
|
||||||
|
return null;
|
||||||
|
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Get all Scholix relation collected from a datasource",
|
||||||
|
description = "return a list of scholix object collected from a specific datasource"
|
||||||
|
)
|
||||||
|
@GetMapping("/linksFromDatasource")
|
||||||
|
@Timed(value = "scholix.v1.linksFromDatasource", description = "Time taken to return links on Version 1.0 of Scholix collected from a LinkProvider")
|
||||||
|
public List<ScholixV1> linksFromDatasource(
|
||||||
|
@Parameter(
|
||||||
|
in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships collected from a LinkProvider",
|
||||||
|
schema = @Schema()) @NotNull String datasource,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "The page number") @RequestParam(required = false) Integer page
|
||||||
|
) throws ScholixException {
|
||||||
|
|
||||||
|
final int currentPage = page != null ? page : 0;
|
||||||
|
Pair<Long, List<Scholix>> scholixResult =manager.linksFromPid(datasource,null,null,null,
|
||||||
|
null,null,null,null,null,null, currentPage
|
||||||
|
);
|
||||||
|
List<Scholix> scholixData = scholixResult.getValue();
|
||||||
|
if (scholixData== null)
|
||||||
|
return null;
|
||||||
|
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Retrieve all scholix links from a persistent identifier",
|
||||||
|
description = "The linksFromPid endpoint returns a list of scholix object related from a specific persistent identifier"
|
||||||
|
)
|
||||||
|
@GetMapping("/linksFromPid")
|
||||||
|
@Timed(value = "scholix.v1.linksFromPid", description = "Time taken to return links on Version 1.0 of Scholix related from a specific persistent identifier")
|
||||||
|
public List<ScholixV1> linksFromPid(
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "persistent Identifier") @NotNull String pid,
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "Persistent Identifier Type") @RequestParam(required = false) String pidType,
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "typology target filter should be publication, dataset or unknown") @RequestParam(required = false) String typologyTarget,
|
||||||
|
@Parameter(in = ParameterIn.QUERY, description = "a datasource provenance filter of the target relation") @RequestParam(required = false) String datasourceTarget,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "The page number") @RequestParam(required = false) Integer page
|
||||||
|
) throws ScholixException {
|
||||||
|
|
||||||
|
final int currentPage = page != null ? page : 0;
|
||||||
|
Pair<Long, List<Scholix>> scholixResult =manager.linksFromPid(datasourceTarget,null,null,null,
|
||||||
|
typologyTarget,pid,pidType,null,null,null, currentPage
|
||||||
|
);
|
||||||
|
List<Scholix> scholixData = scholixResult.getValue();
|
||||||
|
if (scholixData== null)
|
||||||
|
return null;
|
||||||
|
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,77 @@
|
||||||
|
package eu.dnetlib.scholix.api.controller;
|
||||||
|
|
||||||
|
|
||||||
|
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.api.model.v2.PageResultType;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.api.model.v2.ScholixType;
|
||||||
|
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
|
||||||
|
import io.micrometer.core.annotation.Timed;
|
||||||
|
import io.swagger.annotations.Api;
|
||||||
|
import io.swagger.annotations.ApiOperation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.enums.ParameterIn;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/v2")
|
||||||
|
@Api(tags = {
|
||||||
|
"Links : Operation related to the Scholix Links"
|
||||||
|
})
|
||||||
|
public class ScholixControllerV2 extends AbstractDnetController {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private ScholixIndexManager manager;
|
||||||
|
|
||||||
|
@Timed(value = "scholix.v2.links", description = "Time taken to return links on Version 2.0 of Scholix")
|
||||||
|
@ApiOperation("Get Scholix Links")
|
||||||
|
@GetMapping("/Links")
|
||||||
|
public PageResultType links(
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships collected from a LinkProvider") String linkProvider,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a target pid") String targetPid,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a target pid type") String targetPidType,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a target published in a Publisher named targetPublisher") String targetPublisher,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a target type (literature, dataset, unknown)") String targetType,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a source pid") String sourcePid,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a source pid type") String sourcePidType,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a source published in a Publisher named sourcePublisher") String sourcePublisher,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter Scholix relationships having a source type (literature, dataset, unknown)") String sourceType,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "Filter scholix Links having collected after this date") String harvestedAfter,
|
||||||
|
@Parameter(in = ParameterIn.QUERY,
|
||||||
|
description = "select page of result") Integer page) throws Exception {
|
||||||
|
|
||||||
|
if (StringUtils.isEmpty(sourcePid) && StringUtils.isEmpty(targetPid) && StringUtils.isEmpty(sourcePublisher)&& StringUtils.isEmpty(targetPublisher)&& StringUtils.isEmpty(linkProvider))
|
||||||
|
throw new ScholixException("The method requires one of the following parameters: sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider");
|
||||||
|
|
||||||
|
try {
|
||||||
|
final int currentPage = page != null ? page : 0;
|
||||||
|
Pair<Long, List<Scholix>> scholixResult = manager.linksFromPid( linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, harvestedAfter, currentPage);
|
||||||
|
final PageResultType pageResult = new PageResultType();
|
||||||
|
pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10);
|
||||||
|
pageResult.setTotalLinks(scholixResult.getLeft().intValue());
|
||||||
|
pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList()));
|
||||||
|
return pageResult;
|
||||||
|
} catch (Throwable e) {
|
||||||
|
throw new ScholixException("Error on requesting url ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
package eu.dnetlib.scholix.api.index;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.apache.commons.pool2.PooledObject;
|
||||||
|
import org.apache.commons.pool2.PooledObjectFactory;
|
||||||
|
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||||
|
import org.elasticsearch.client.RequestOptions;
|
||||||
|
import org.elasticsearch.client.RestHighLevelClient;
|
||||||
|
import org.springframework.data.elasticsearch.client.ClientConfiguration;
|
||||||
|
import org.springframework.data.elasticsearch.client.RestClients;
|
||||||
|
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The type Elastic search client factory.
|
||||||
|
*/
|
||||||
|
public class ElasticSearchClientFactory implements PooledObjectFactory<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> {
|
||||||
|
|
||||||
|
private final ElasticSearchProperties elasticSearchProperties;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new Elastic search client factory.
|
||||||
|
*
|
||||||
|
* @param elasticSearchProperties the elastic search properties
|
||||||
|
*/
|
||||||
|
public ElasticSearchClientFactory(final ElasticSearchProperties elasticSearchProperties){
|
||||||
|
this.elasticSearchProperties = elasticSearchProperties;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> makeObject() throws Exception {
|
||||||
|
|
||||||
|
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
|
||||||
|
.connectedTo(elasticSearchProperties.getClusterNodes().split(","))
|
||||||
|
.withConnectTimeout(elasticSearchProperties.getConnectionTimeout())
|
||||||
|
.withSocketTimeout(elasticSearchProperties.getSocketTimeout())
|
||||||
|
.build();
|
||||||
|
RestHighLevelClient cc = RestClients.create(clientConfiguration).rest();
|
||||||
|
|
||||||
|
return new DefaultPooledObject<>(new ImmutablePair<>(cc, new ElasticsearchRestTemplate(cc)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void destroyObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
|
||||||
|
RestHighLevelClient client = pooledObject.getObject().getLeft();
|
||||||
|
if(client!=null&&client.ping(RequestOptions.DEFAULT)){
|
||||||
|
try {
|
||||||
|
client.close();
|
||||||
|
}catch (Exception e){
|
||||||
|
//ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean validateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) {
|
||||||
|
RestHighLevelClient client = pooledObject.getObject().getLeft();
|
||||||
|
try {
|
||||||
|
return client.ping(RequestOptions.DEFAULT);
|
||||||
|
}catch(Exception e){
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void activateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
|
||||||
|
RestHighLevelClient client = pooledObject.getObject().getLeft();
|
||||||
|
boolean response = client.ping(RequestOptions.DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void passivateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
|
||||||
|
//nothing
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
package eu.dnetlib.scholix.api.index;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.elasticsearch.client.RestHighLevelClient;
|
||||||
|
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The type Elastic search pool.
|
||||||
|
*/
|
||||||
|
public class ElasticSearchPool extends Pool<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> {
|
||||||
|
|
||||||
|
private final ElasticSearchProperties elasticSearchProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new Elastic search pool.
|
||||||
|
*
|
||||||
|
* @param elasticSearchProperties the elastic search properties
|
||||||
|
*/
|
||||||
|
public ElasticSearchPool(ElasticSearchProperties elasticSearchProperties){
|
||||||
|
super(elasticSearchProperties, new ElasticSearchClientFactory(elasticSearchProperties));
|
||||||
|
this.elasticSearchProperties = elasticSearchProperties;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets elastic search properties.
|
||||||
|
*
|
||||||
|
* @return the elastic search properties
|
||||||
|
*/
|
||||||
|
public ElasticSearchProperties getElasticSearchProperties() {
|
||||||
|
return elasticSearchProperties;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
package eu.dnetlib.scholix.api.index;
|
||||||
|
|
||||||
|
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import javax.validation.constraints.NotNull;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The type Elastic search properties.
|
||||||
|
*/
|
||||||
|
@Component("elasticSearchProperties")
|
||||||
|
@ConfigurationProperties(prefix = "scholix.elastic")
|
||||||
|
public class ElasticSearchProperties extends GenericObjectPoolConfig {
|
||||||
|
|
||||||
|
@NotNull
|
||||||
|
private String clusterNodes;
|
||||||
|
@NotNull
|
||||||
|
private String indexName;
|
||||||
|
@NotNull
|
||||||
|
private long connectionTimeout;
|
||||||
|
@NotNull
|
||||||
|
private long socketTimeout;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets cluster nodes.
|
||||||
|
*
|
||||||
|
* @return the cluster nodes
|
||||||
|
*/
|
||||||
|
public String getClusterNodes() {
|
||||||
|
return clusterNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets cluster nodes.
|
||||||
|
*
|
||||||
|
* @param clusterNodes the cluster nodes
|
||||||
|
* @return the cluster nodes
|
||||||
|
*/
|
||||||
|
public ElasticSearchProperties setClusterNodes(String clusterNodes) {
|
||||||
|
this.clusterNodes = clusterNodes;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets index name.
|
||||||
|
*
|
||||||
|
* @return the index name
|
||||||
|
*/
|
||||||
|
public String getIndexName() {
|
||||||
|
return indexName;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets index name.
|
||||||
|
*
|
||||||
|
* @param indexName the index name
|
||||||
|
* @return the index name
|
||||||
|
*/
|
||||||
|
public ElasticSearchProperties setIndexName(String indexName) {
|
||||||
|
this.indexName = indexName;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets connection timeout.
|
||||||
|
*
|
||||||
|
* @return the connection timeout
|
||||||
|
*/
|
||||||
|
public long getConnectionTimeout() {
|
||||||
|
return connectionTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets connection timeout.
|
||||||
|
*
|
||||||
|
* @param connectionTimeout the connection timeout
|
||||||
|
* @return the connection timeout
|
||||||
|
*/
|
||||||
|
public ElasticSearchProperties setConnectionTimeout(long connectionTimeout) {
|
||||||
|
this.connectionTimeout = connectionTimeout;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets socket timeout.
|
||||||
|
*
|
||||||
|
* @return the socket timeout
|
||||||
|
*/
|
||||||
|
public long getSocketTimeout() {
|
||||||
|
return socketTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets socket timeout.
|
||||||
|
*
|
||||||
|
* @param socketTimeout the socket timeout
|
||||||
|
* @return the socket timeout
|
||||||
|
*/
|
||||||
|
public ElasticSearchProperties setSocketTimeout(long socketTimeout) {
|
||||||
|
this.socketTimeout = socketTimeout;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,231 @@
|
||||||
|
package eu.dnetlib.scholix.api.index;
|
||||||
|
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import org.apache.commons.pool2.PooledObjectFactory;
|
||||||
|
import org.apache.commons.pool2.impl.GenericObjectPool;
|
||||||
|
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* When using the Java High Level REST Client provided by the Elasticsearch official website, it is found that there is no in the client API.
|
||||||
|
* Connecting to connect the pool, create a new connection every time, this is impact in high concurrency situation, so it is ready to be on the client
|
||||||
|
* API increases the concept of pool.
|
||||||
|
*
|
||||||
|
* Fortunately, we don't need to turn your weight to write the implementation of the connection pool, because Apache provides us with the general framework of the connection pool.
|
||||||
|
* Commons-pool2, and we only need to implement some logic according to the frame design. Used in the REDIS client API
|
||||||
|
* Jedispool is based on Commons-pool2 implementation.
|
||||||
|
*
|
||||||
|
* Let's take a look at how to achieve it.
|
||||||
|
*
|
||||||
|
* First we have to create a pool class, this pool introduces GenericObjectPool in Commons-pool2 through dependent manner. In this class
|
||||||
|
* In, we define how to borrow objects and returns objects from the pool.
|
||||||
|
*
|
||||||
|
* @param <T> the type parameter
|
||||||
|
*/
|
||||||
|
public class Pool<T> implements Cloneable {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Internal pool.
|
||||||
|
*/
|
||||||
|
protected GenericObjectPool<T> internalPool ;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new Pool.
|
||||||
|
*/
|
||||||
|
public Pool(){
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new Pool.
|
||||||
|
*
|
||||||
|
* @param poolConfig the pool config
|
||||||
|
* @param factory the factory
|
||||||
|
*/
|
||||||
|
public Pool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory<T> factory){
|
||||||
|
initPool(poolConfig, factory);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Init pool.
|
||||||
|
*
|
||||||
|
* @param poolConfig the pool config
|
||||||
|
* @param factory the factory
|
||||||
|
*/
|
||||||
|
public void initPool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory<T> factory) {
|
||||||
|
|
||||||
|
if (this.internalPool != null) {
|
||||||
|
try {
|
||||||
|
closeInternalPool();
|
||||||
|
} catch (Exception e) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.internalPool = new GenericObjectPool<T>(factory, poolConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Close internal pool.
|
||||||
|
*
|
||||||
|
* @throws ScholixException the scholix exception
|
||||||
|
*/
|
||||||
|
protected void closeInternalPool() throws ScholixException {
|
||||||
|
try {
|
||||||
|
internalPool.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ScholixException("Could not destroy the pool", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets resource.
|
||||||
|
*
|
||||||
|
* @return the resource
|
||||||
|
* @throws ScholixException the scholix exception
|
||||||
|
*/
|
||||||
|
public T getResource() throws ScholixException {
|
||||||
|
try {
|
||||||
|
return internalPool.borrowObject();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ScholixException("Could not get a resource from the pool", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return resource.
|
||||||
|
*
|
||||||
|
* @param resource the resource
|
||||||
|
* @throws ScholixException the scholix exception
|
||||||
|
*/
|
||||||
|
public void returnResource(final T resource) throws ScholixException {
|
||||||
|
if (resource != null) {
|
||||||
|
returnResourceObject(resource);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void returnResourceObject(final T resource) throws ScholixException {
|
||||||
|
if (resource == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
internalPool.returnObject(resource);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ScholixException("Could not return the resource to the pool", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return broken resource.
|
||||||
|
*
|
||||||
|
* @param resource the resource
|
||||||
|
* @throws ScholixException the scholix exception
|
||||||
|
*/
|
||||||
|
public void returnBrokenResource(final T resource) throws ScholixException {
|
||||||
|
if (resource != null) {
|
||||||
|
returnBrokenResourceObject(resource);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void returnBrokenResourceObject(T resource) throws ScholixException {
|
||||||
|
try {
|
||||||
|
internalPool.invalidateObject(resource);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ScholixException("Could not return the resource to the pool", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Destroy.
|
||||||
|
*
|
||||||
|
* @throws ScholixException the scholix exception
|
||||||
|
*/
|
||||||
|
public void destroy() throws ScholixException {
|
||||||
|
closeInternalPool();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets num active.
|
||||||
|
*
|
||||||
|
* @return the num active
|
||||||
|
*/
|
||||||
|
public int getNumActive() {
|
||||||
|
if (poolInactive()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.internalPool.getNumActive();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets num idle.
|
||||||
|
*
|
||||||
|
* @return the num idle
|
||||||
|
*/
|
||||||
|
public int getNumIdle() {
|
||||||
|
if (poolInactive()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.internalPool.getNumIdle();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets num waiters.
|
||||||
|
*
|
||||||
|
* @return the num waiters
|
||||||
|
*/
|
||||||
|
public int getNumWaiters() {
|
||||||
|
if (poolInactive()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.internalPool.getNumWaiters();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets mean borrow wait time millis.
|
||||||
|
*
|
||||||
|
* @return the mean borrow wait time millis
|
||||||
|
*/
|
||||||
|
public long getMeanBorrowWaitTimeMillis() {
|
||||||
|
if (poolInactive()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.internalPool.getMeanBorrowWaitTimeMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets max borrow wait time millis.
|
||||||
|
*
|
||||||
|
* @return the max borrow wait time millis
|
||||||
|
*/
|
||||||
|
public long getMaxBorrowWaitTimeMillis() {
|
||||||
|
if (poolInactive()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.internalPool.getMaxBorrowWaitTimeMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean poolInactive() {
|
||||||
|
return this.internalPool == null || this.internalPool.isClosed();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add objects.
|
||||||
|
*
|
||||||
|
* @param count the count
|
||||||
|
* @throws Exception the exception
|
||||||
|
*/
|
||||||
|
public void addObjects(int count) throws Exception {
|
||||||
|
try {
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
this.internalPool.addObject();
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new Exception("Error trying to add idle objects", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,315 @@
|
||||||
|
package eu.dnetlib.scholix.api.index;
|
||||||
|
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
|
||||||
|
import eu.dnetlib.scholix.api.ScholixException;
|
||||||
|
import eu.dnetlib.scholix.api.TaggedCounter;
|
||||||
|
import io.micrometer.core.annotation.Timed;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.apache.lucene.search.join.ScoreMode;
|
||||||
|
import org.elasticsearch.action.search.SearchType;
|
||||||
|
import org.elasticsearch.client.RestHighLevelClient;
|
||||||
|
import org.elasticsearch.index.query.*;
|
||||||
|
import org.elasticsearch.search.aggregations.Aggregation;
|
||||||
|
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
||||||
|
import org.elasticsearch.search.aggregations.Aggregations;
|
||||||
|
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
|
||||||
|
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.data.domain.PageRequest;
|
||||||
|
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
|
||||||
|
import org.springframework.data.elasticsearch.core.SearchHit;
|
||||||
|
import org.springframework.data.elasticsearch.core.SearchHits;
|
||||||
|
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
|
||||||
|
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
|
||||||
|
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The type Scholix index manager.
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class ScholixIndexManager {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Elastic search properties.
|
||||||
|
*/
|
||||||
|
@Autowired
|
||||||
|
ElasticSearchProperties elasticSearchProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Elasticsearch template.
|
||||||
|
*/
|
||||||
|
@Autowired
|
||||||
|
ElasticSearchPool connectionPool;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The My counter.
|
||||||
|
*/
|
||||||
|
@Autowired
|
||||||
|
TaggedCounter myCounter;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The enum Pid type prefix.
|
||||||
|
*/
|
||||||
|
public enum RelationPrefix {
|
||||||
|
/**
|
||||||
|
* Source pid type prefix.
|
||||||
|
*/
|
||||||
|
source,
|
||||||
|
/**
|
||||||
|
* Target pid type prefix.
|
||||||
|
*/
|
||||||
|
target
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private QueryBuilder createObjectTypeQuery(final RelationPrefix prefix, final String objectType ) throws ScholixException{
|
||||||
|
if (prefix == null){
|
||||||
|
throw new ScholixException("prefix cannot be null");
|
||||||
|
}
|
||||||
|
return new NestedQueryBuilder(String.format("%s", prefix), new TermQueryBuilder(String.format("%s.objectType",prefix), objectType), ScoreMode.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private QueryBuilder createPidTypeQuery(final RelationPrefix prefix, final String pidTypeValue ) throws ScholixException{
|
||||||
|
if (prefix == null){
|
||||||
|
throw new ScholixException("prefix cannot be null");
|
||||||
|
}
|
||||||
|
return new NestedQueryBuilder(String.format("%s.identifier", prefix), new TermQueryBuilder(String.format("%s.identifier.schema",prefix), pidTypeValue), ScoreMode.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private QueryBuilder createLinkProviderQuery(final String providerName ) throws ScholixException{
|
||||||
|
if (providerName == null){
|
||||||
|
throw new ScholixException("prefix cannot be null");
|
||||||
|
}
|
||||||
|
return new NestedQueryBuilder("linkprovider", new TermQueryBuilder("linkprovider.name",providerName), ScoreMode.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
private QueryBuilder createLinkPublisherQuery(final RelationPrefix prefix, final String publisher ) throws ScholixException{
|
||||||
|
if (prefix == null){
|
||||||
|
throw new ScholixException("prefix cannot be null");
|
||||||
|
}
|
||||||
|
return new NestedQueryBuilder(String.format("%s.publisher", prefix), new TermQueryBuilder(String.format("%s.publisher.name",prefix), publisher), ScoreMode.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
private QueryBuilder createPidValueQuery(final RelationPrefix prefix, final String pidValue ) throws ScholixException{
|
||||||
|
if (prefix == null){
|
||||||
|
throw new ScholixException("prefix cannot be null");
|
||||||
|
}
|
||||||
|
return new NestedQueryBuilder(String.format("%s.identifier", prefix), new TermQueryBuilder(String.format("%s.identifier.identifier",prefix), pidValue), ScoreMode.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private QueryBuilder createFinalQuery(final List<QueryBuilder> queries) throws ScholixException{
|
||||||
|
|
||||||
|
if (queries == null || queries.isEmpty())
|
||||||
|
throw new ScholixException("the list of queries must be not empty");
|
||||||
|
|
||||||
|
|
||||||
|
if (queries.size() ==1) {
|
||||||
|
return queries.get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
else {
|
||||||
|
final BoolQueryBuilder b = new BoolQueryBuilder();
|
||||||
|
b.must().addAll(queries);
|
||||||
|
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private void incrementPidCounter(RelationPrefix prefix, String value) {
|
||||||
|
switch (value.toLowerCase()){
|
||||||
|
case "doi": {
|
||||||
|
myCounter.increment(String.format("%s_doi", prefix));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "pmc": {
|
||||||
|
myCounter.increment(String.format("%s_pmc", prefix));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
myCounter.increment(String.format("%s_other", prefix));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public List<Pair<String, Long>> totalLinksByProvider(final String filterName) throws ScholixException {
|
||||||
|
|
||||||
|
|
||||||
|
final QueryBuilder query = StringUtils.isNoneBlank(filterName)?createLinkProviderQuery(filterName):QueryBuilders.matchAllQuery();
|
||||||
|
|
||||||
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
||||||
|
.withQuery(query)
|
||||||
|
.withSearchType(SearchType.DEFAULT)
|
||||||
|
.withPageable(PageRequest.of(0,10))
|
||||||
|
.addAggregation(AggregationBuilders.nested("nested", "linkprovider")
|
||||||
|
.subAggregation(AggregationBuilders.terms("by_map").field("linkprovider.name").size(100).minDocCount(1)))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
|
||||||
|
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = connectionPool.getResource();
|
||||||
|
ElasticsearchRestTemplate client = resource.getValue();
|
||||||
|
final SearchHits<Scholix> hits = client.search(searchQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
|
||||||
|
|
||||||
|
final Aggregations aggregations = hits.getAggregations();
|
||||||
|
connectionPool.returnResource(resource);
|
||||||
|
|
||||||
|
if(aggregations == null)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
final Aggregation aggByMap = ((ParsedNested) aggregations.asMap().get("nested")).getAggregations().asMap().get("by_map");
|
||||||
|
|
||||||
|
|
||||||
|
return ((ParsedStringTerms) aggByMap).getBuckets()
|
||||||
|
.stream()
|
||||||
|
.map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public List<Pair<String, Long>> totalLinksPublisher(final RelationPrefix prefix, final String filterName) throws ScholixException {
|
||||||
|
|
||||||
|
|
||||||
|
final QueryBuilder query = StringUtils.isNoneBlank(filterName)?createLinkPublisherQuery(prefix,filterName):QueryBuilders.matchAllQuery();
|
||||||
|
|
||||||
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
||||||
|
.withQuery(query)
|
||||||
|
.withSearchType(SearchType.DEFAULT)
|
||||||
|
.withPageable(PageRequest.of(0,10))
|
||||||
|
.addAggregation(AggregationBuilders.nested("nested", String.format("%s.publisher", prefix ))
|
||||||
|
.subAggregation(AggregationBuilders.terms("by_map").field(String.format("%s.publisher.name", prefix )).size(100).minDocCount(1)))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
|
||||||
|
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = connectionPool.getResource();
|
||||||
|
ElasticsearchRestTemplate client = resource.getValue();
|
||||||
|
final SearchHits<Scholix> hits = client.search(searchQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
|
||||||
|
|
||||||
|
final Aggregations aggregations = hits.getAggregations();
|
||||||
|
connectionPool.returnResource(resource);
|
||||||
|
|
||||||
|
if(aggregations == null)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
final Aggregation aggByMap = ((ParsedNested) aggregations.asMap().get("nested")).getAggregations().asMap().get("by_map");
|
||||||
|
|
||||||
|
|
||||||
|
return ((ParsedStringTerms) aggByMap).getBuckets()
|
||||||
|
.stream()
|
||||||
|
.map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Links from pid pair.
|
||||||
|
*
|
||||||
|
* @param linkProvider the link provider
|
||||||
|
* @param targetPid the target pid
|
||||||
|
* @param targetPidType the target pid type
|
||||||
|
* @param targetPublisher the target publisher
|
||||||
|
* @param targetType the target type
|
||||||
|
* @param sourcePid the source pid
|
||||||
|
* @param sourcePidType the source pid type
|
||||||
|
* @param sourcePublisher the source publisher
|
||||||
|
* @param sourceType the source type
|
||||||
|
* @param harvestedAfter the harvested after
|
||||||
|
* @param page the page
|
||||||
|
* @return the pair
|
||||||
|
* @throws ScholixException the scholix exception
|
||||||
|
*/
|
||||||
|
@Timed(value = "scholix.index.request.links", description = "Time taken to request index")
|
||||||
|
public Pair<Long,List<Scholix>> linksFromPid ( final String linkProvider,
|
||||||
|
final String targetPid, final String targetPidType, final String targetPublisher,
|
||||||
|
final String targetType, final String sourcePid, final String sourcePidType,
|
||||||
|
final String sourcePublisher, final String sourceType, final String harvestedAfter,
|
||||||
|
final Integer page) throws ScholixException {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if (sourcePid==null && sourcePidType==null && targetPid==null && targetPidType==null && sourcePublisher==null && targetPublisher==null && linkProvider==null)
|
||||||
|
throw new ScholixException("One of sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider should be not null");
|
||||||
|
|
||||||
|
final List<QueryBuilder> queries = new ArrayList<>();
|
||||||
|
|
||||||
|
if (StringUtils.isNoneBlank(linkProvider)) {
|
||||||
|
myCounter.increment("linkProvider");
|
||||||
|
queries.add(createLinkProviderQuery(linkProvider));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNoneBlank(targetPid)) {
|
||||||
|
myCounter.increment("targetPid");
|
||||||
|
queries.add(createPidValueQuery(RelationPrefix.target, targetPid));
|
||||||
|
}
|
||||||
|
if (StringUtils.isNoneBlank(sourcePid)) {
|
||||||
|
myCounter.increment("sourcePid");
|
||||||
|
queries.add(createPidValueQuery(RelationPrefix.source, sourcePid));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNoneBlank(targetPidType)) {
|
||||||
|
assert targetPidType != null;
|
||||||
|
incrementPidCounter(RelationPrefix.target,targetPidType);
|
||||||
|
queries.add(createPidTypeQuery(RelationPrefix.target, targetPidType));
|
||||||
|
}
|
||||||
|
if (StringUtils.isNoneBlank(sourcePidType)) {
|
||||||
|
assert sourcePidType != null;
|
||||||
|
incrementPidCounter(RelationPrefix.source,sourcePidType);
|
||||||
|
queries.add(createPidTypeQuery(RelationPrefix.source, sourcePidType));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNoneBlank(targetType)) {
|
||||||
|
if ("dataset".equalsIgnoreCase(targetType) || "publication".equalsIgnoreCase(targetType))
|
||||||
|
myCounter.increment(String.format("targetType_%s", targetType));
|
||||||
|
queries.add(createObjectTypeQuery(RelationPrefix.target, targetType));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNoneBlank(sourceType)) {
|
||||||
|
if ("dataset".equalsIgnoreCase(sourceType) || "publication".equalsIgnoreCase(sourceType)) {
|
||||||
|
myCounter.increment(String.format("sourceType_%s", sourceType));
|
||||||
|
}
|
||||||
|
queries.add(createObjectTypeQuery(RelationPrefix.source, sourceType));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNoneBlank(targetPublisher)) {
|
||||||
|
myCounter.increment("targetPublisher");
|
||||||
|
|
||||||
|
queries.add(createLinkPublisherQuery(RelationPrefix.target,targetPublisher));
|
||||||
|
}
|
||||||
|
|
||||||
|
QueryBuilder result = createFinalQuery(queries);
|
||||||
|
|
||||||
|
NativeSearchQuery finalQuery = new NativeSearchQueryBuilder()
|
||||||
|
.withQuery(result)
|
||||||
|
.withPageable(PageRequest.of(page,10))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
|
||||||
|
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = connectionPool.getResource();
|
||||||
|
ElasticsearchRestTemplate client = resource.getValue();
|
||||||
|
|
||||||
|
long tt = client.count(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
|
||||||
|
|
||||||
|
SearchHits<Scholix> scholixRes = client.search(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
|
||||||
|
|
||||||
|
connectionPool.returnResource(resource);
|
||||||
|
|
||||||
|
return new ImmutablePair<>(tt,scholixRes.stream().map(SearchHit::getContent).collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
package eu.dnetlib.scholix.api.metrics;
|
||||||
|
|
||||||
|
import eu.dnetlib.common.metrics.MetricInfo;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
|
||||||
|
@Component("scholexplorer_test")
|
||||||
|
public class RequestCounter implements MetricInfo {
|
||||||
|
@Override
|
||||||
|
public double obtainValue() {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
spring.main.banner-mode = console
|
||||||
|
|
||||||
|
logging.level.root = INFO
|
||||||
|
dhp.swagger.api.host = localhost:8080
|
||||||
|
#dhp.swagger.api.host = api.scholexplorer.openaire.eu
|
||||||
|
dhp.swagger.api.basePath = /
|
||||||
|
|
||||||
|
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml
|
||||||
|
#
|
||||||
|
#spring.thymeleaf.cache=false
|
||||||
|
#
|
||||||
|
management.endpoints.web.exposure.include = prometheus,health
|
||||||
|
management.endpoints.web.base-path = /
|
||||||
|
management.endpoints.web.path-mapping.prometheus = metrics
|
||||||
|
management.endpoints.web.path-mapping.health = health
|
||||||
|
|
||||||
|
management.metrics.distribution.percentiles-histogram.http.server.requests=false
|
||||||
|
management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
|
||||||
|
management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
scholix.elastic.clusterNodes = localhost:9200
|
||||||
|
scholix.elastic.indexName = dli_shadow_scholix
|
||||||
|
scholix.elastic.socketTimeout = 60000
|
||||||
|
scholix.elastic.connectionTimeout= 60000
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
|
|
@ -0,0 +1,135 @@
|
||||||
|
<!-- HTML for static distribution bundle build -->
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>Custom Swagger UI</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/swagger-ui.css" >
|
||||||
|
<link rel="icon" type="image/png" href="./swagger-ui/swagger-favicon-32x32.png" sizes="32x32" />
|
||||||
|
<link rel="icon" type="image/png" href="./swagger-ui/swagger-favicon-16x16.png" sizes="16x16" />
|
||||||
|
<style>
|
||||||
|
html
|
||||||
|
{
|
||||||
|
box-sizing: border-box;
|
||||||
|
overflow: -moz-scrollbars-vertical;
|
||||||
|
overflow-y: scroll;
|
||||||
|
}
|
||||||
|
|
||||||
|
*,
|
||||||
|
*:before,
|
||||||
|
*:after
|
||||||
|
{
|
||||||
|
box-sizing: inherit;
|
||||||
|
}
|
||||||
|
|
||||||
|
body
|
||||||
|
{
|
||||||
|
margin:0;
|
||||||
|
background: #fafafa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.top-nav-bar{
|
||||||
|
position: fixed;
|
||||||
|
top: 0;
|
||||||
|
z-index: 99;
|
||||||
|
width: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
background: #333;
|
||||||
|
padding: 15px;
|
||||||
|
}
|
||||||
|
.nav-bar-icon{
|
||||||
|
margin-top: 1px;
|
||||||
|
float: left;
|
||||||
|
display: block;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.nav-bar-title{
|
||||||
|
float: left;
|
||||||
|
display: block;
|
||||||
|
text-decoration: none;
|
||||||
|
margin-top: 7px;
|
||||||
|
margin-left: 10px;
|
||||||
|
font-size: 18px;
|
||||||
|
color: #ffffff;
|
||||||
|
font-family: sans-serif;
|
||||||
|
}
|
||||||
|
.nav-bar-select{
|
||||||
|
width: 30%;
|
||||||
|
float: right;
|
||||||
|
font-family: sans-serif;
|
||||||
|
display: inline-block;
|
||||||
|
cursor: pointer;
|
||||||
|
padding: 10px 15px;
|
||||||
|
outline: 0;
|
||||||
|
border-radius: 2px;
|
||||||
|
border: none;
|
||||||
|
background: #fafafa;
|
||||||
|
color: #3b4151;
|
||||||
|
appearance: none;
|
||||||
|
-webkit-appearance: none;
|
||||||
|
-moz-appearance: none;
|
||||||
|
}
|
||||||
|
select.classic {
|
||||||
|
background-image: linear-gradient(45deg, transparent 50%, #111 50%), linear-gradient(135deg, #111 50%, transparent 50%);
|
||||||
|
background-position: calc(100% - 20px) calc(1em + 2px), calc(100% - 15px) calc(1em + 2px), 100% 0;
|
||||||
|
background-size: 5px 5px, 5px 5px, 3.5em 3.5em;
|
||||||
|
background-repeat: no-repeat;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<div class="top-nav-bar">
|
||||||
|
<a class="nav-bar-icon"><img src="swagger-favicon-32x32.png"></a>
|
||||||
|
<a class="nav-bar-title"><b>X name</b></a>
|
||||||
|
<select class="classic nav-bar-select" id="service-selector" onchange="changeSwaggerUI()">
|
||||||
|
<option value="./swagger.json">X service</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div style="margin-top: 100px" id="swagger-ui"></div>
|
||||||
|
|
||||||
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/swagger-ui-bundle.js"> </script>
|
||||||
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/swagger-ui-standalone-preset.js"> </script>
|
||||||
|
<script>
|
||||||
|
function changeSwaggerUI(){
|
||||||
|
let selected_service_swaggerURL = document.getElementById("service-selector").value;
|
||||||
|
loadUI(selected_service_swaggerURL);
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadUI(swaggerJsonURL){
|
||||||
|
// Begin Swagger UI call region
|
||||||
|
const ui = SwaggerUIBundle({
|
||||||
|
url: swaggerJsonURL,
|
||||||
|
validatorUrl: "",
|
||||||
|
dom_id: '#swagger-ui',
|
||||||
|
deepLinking: true,
|
||||||
|
docExpansion: 'none',
|
||||||
|
presets: [
|
||||||
|
SwaggerUIBundle.presets.apis,
|
||||||
|
SwaggerUIStandalonePreset
|
||||||
|
],
|
||||||
|
plugins: [
|
||||||
|
CustomTopbarPlugin
|
||||||
|
],
|
||||||
|
layout: "StandaloneLayout"
|
||||||
|
});
|
||||||
|
// End Swagger UI call region
|
||||||
|
|
||||||
|
window.ui = ui
|
||||||
|
}
|
||||||
|
|
||||||
|
function CustomTopbarPlugin() {
|
||||||
|
// this plugin overrides the Topbar component to return nothing
|
||||||
|
return {
|
||||||
|
components: {
|
||||||
|
Topbar: () => null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
window.onload = function() {
|
||||||
|
loadUI("./swagger-ui/springfox.js?v=3.0.0");
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
package eu.dnetlib.scholix.test;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
|
||||||
|
public class QueryTest {
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void featureToTest() {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
10
pom.xml
10
pom.xml
|
|
@ -142,6 +142,13 @@
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
<artifactId>commons-pool2</artifactId>
|
||||||
|
<version>2.11.1</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<dependencyManagement>
|
<dependencyManagement>
|
||||||
|
|
@ -233,6 +240,7 @@
|
||||||
<version>3.0.0</version>
|
<version>3.0.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
<!-- Hadoop -->
|
<!-- Hadoop -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
|
@ -390,7 +398,7 @@
|
||||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||||
<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
|
<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
|
||||||
<java.version>1.8</java.version>
|
<java.version>1.8</java.version>
|
||||||
<dhp-schemas-version>2.3.6</dhp-schemas-version>
|
<dhp-schemas-version>2.11.33</dhp-schemas-version>
|
||||||
<apache.solr.version>7.1.0</apache.solr.version>
|
<apache.solr.version>7.1.0</apache.solr.version>
|
||||||
<mongodb.driver.version>3.4.2</mongodb.driver.version>
|
<mongodb.driver.version>3.4.2</mongodb.driver.version>
|
||||||
<springfox-version>2.8.0</springfox-version>
|
<springfox-version>2.8.0</springfox-version>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue