sandro.labruzzo
/
scholexplorer-api
2
1
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

imported new version of the api

This commit is contained in:
Sandro La Bruzzo 2024-06-06 16:10:04 +02:00
parent 59b1ba3a2e
commit 0a23135d10
24 changed files with 1787 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,6 +2,8 @@
# Compiled class file # Compiled class file
*.class *.class
.idea
target
# Log file # Log file
*.log *.log

93
pom.xml Normal file
View File

@ -0,0 +1,93 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.3.0</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>eu.dnetlib</groupId>
<artifactId>scholexplorer-api</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>scholexplorer-api</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>17</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aspects</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-schemas</artifactId>
<version>6.1.3-FLAT-SCHOLIX</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<version>4.2.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-annotations</artifactId>
<version>2.2.22</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

26
src/main/java/eu/dnetlib/scholexplorer/api/RestClientConfig.java Normal file
View File

@ -0,0 +1,26 @@
package eu.dnetlib.scholexplorer.api;
import eu.dnetlib.scholexplorer.api.index.ElasticSearchPool;
import eu.dnetlib.scholexplorer.api.index.ElasticSearchProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class RestClientConfig {
@Autowired
private ElasticSearchProperties elasticSearchProperties;
@Bean
public ElasticSearchPool connectionPool() {
elasticSearchProperties.setMaxIdle(5);
elasticSearchProperties.setMaxTotal(10);
return new ElasticSearchPool(elasticSearchProperties);
}
}

122
src/main/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplication.java Normal file
View File

@ -0,0 +1,122 @@
package eu.dnetlib.scholexplorer.api;
import io.micrometer.core.aop.TimedAspect;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.info.License;
import org.apache.commons.lang3.StringUtils;
import org.springdoc.core.models.GroupedOpenApi;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import io.swagger.v3.oas.models.servers.Server;
import io.swagger.v3.oas.models.info.Info;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
@SpringBootApplication
public class ScholexplorerApiApplication {
@Value("${server.public_url}")
private String serverPublicUrl;
@Value("${server.public_desc}")
private String serverPublicDesc;
protected static final License AGPL_3_LICENSE =
new License().name("GNU Affero General Public License v3.0 or later").url("https://www.gnu.org/licenses/agpl-3.0.txt");
private final double scale = 1000000000;
private final double[] histogramValues = new double[] {
.005 * scale, .01 * scale, .25 * scale, .5 * scale, .75 * scale, scale, 2.5 * scale, 5.0 * scale, 7.5 * scale, 10.0 * scale
};
@Bean
public TimedAspect timedAspect(final MeterRegistry meterRegistry) {
final MeterFilter mf = new MeterFilter() {
@Override
public DistributionStatisticConfig configure(final Meter.Id id, final DistributionStatisticConfig config) {
if (id.getName().startsWith(ScholixAPIConstants.SCHOLIX_COUNTER_PREFIX)) {
return DistributionStatisticConfig.builder()
.percentilesHistogram(false)
.serviceLevelObjectives(histogramValues)
.build()
.merge(config);
}
return config;
}
};
meterRegistry.config().meterFilter(mf);
return new TimedAspect(meterRegistry);
}
@Bean
public TaggedCounter myCounter(final MeterRegistry meterRegistry) {
return new TaggedCounter(ScholixAPIConstants.SCHOLIX_MANAGER_COUNTER_NAME, ScholixAPIConstants.SCHOLIX_MANAGER_TAG_NAME, meterRegistry);
}
@Bean
public GroupedOpenApi publicApiV1() {
return GroupedOpenApi.builder()
.group(ScholixAPIConstants.API_V1_NAME)
.pathsToMatch("/v1/**")
.build();
}
@Bean
public GroupedOpenApi publicApiV2() {
return GroupedOpenApi.builder()
.group(ScholixAPIConstants.API_V2_NAME)
.pathsToMatch("/v2/**")
.build();
}
@Bean
public OpenAPI newSwaggerDocket() {
final List<Server> servers = new ArrayList<>();
if (StringUtils.isNotBlank(serverPublicUrl)) {
final Server server = new Server();
server.setUrl(serverPublicUrl);
server.setDescription(serverPublicDesc);
servers.add(server);
}
return new OpenAPI()
.servers(servers)
.info(getSwaggerInfo())
.tags(new ArrayList<>());
}
private Info getSwaggerInfo() {
return new Info()
.title(swaggerTitle())
.description(swaggerDesc())
.version("1.0")
.license(AGPL_3_LICENSE);
}
protected String swaggerTitle() {
return "ScholeExplorer APIs";
}
protected String swaggerDesc() {
return ScholixAPIConstants.API_DESCRIPTION;
}
public static void main(String[] args) {
SpringApplication.run(ScholexplorerApiApplication.class, args);
}
}

23
src/main/java/eu/dnetlib/scholexplorer/api/ScholixAPIConstants.java Normal file
View File

@ -0,0 +1,23 @@
package eu.dnetlib.scholexplorer.api;
public class ScholixAPIConstants {
public static final String API_V1_NAME = "Scholexplorer API V1.0";
public static final String API_V2_NAME = "Scholexplorer API V2.0";
public static String API_DESCRIPTION =" <p style=\"text-align:center;\"><img src=\"/logo.png\" alt=\"ScholeXplorer\"> </p>" +
"The Scholix Swagger API allows clients to run REST queries over the Scholexplorer index in order to fetch links matching given criteria. In the current version, clients can search for:" +
"<ul><li>Links whose source object has a given PID or PID type</li>" +
"<li>Links whose source object has been published by a given data source (\"data source as publisher\")</li>" +
"<li>Links that were collected from a given data source (\"data source as provider\").</li></ul>";
public static String SCHOLIX_MANAGER_COUNTER_NAME= "scholixLinkCounter";
public static final String SCHOLIX_MANAGER_TAG_NAME = "links";
public static String SCHOLIX_COUNTER_PREFIX = "scholix";
}

35
src/main/java/eu/dnetlib/scholexplorer/api/ScholixException.java Normal file
View File

@ -0,0 +1,35 @@
package eu.dnetlib.scholexplorer.api;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
public class ScholixException extends Exception{
private static final long serialVersionUID = -3414428892721711308L;
public ScholixException() {
super();
}
public ScholixException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public ScholixException(String message, Throwable cause) {
super(message, cause);
}
public ScholixException(String message) {
super(message);
}
public ScholixException(Throwable cause) {
super(cause);
}
}

33
src/main/java/eu/dnetlib/scholexplorer/api/TaggedCounter.java Normal file
View File

@ -0,0 +1,33 @@
package eu.dnetlib.scholexplorer.api;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import java.util.HashMap;
import java.util.Map;
public class TaggedCounter {
private final String name;
private final String tagName;
private final MeterRegistry registry;
private final Map<String, Counter> counters = new HashMap<>();
public TaggedCounter(String name, String tagName, MeterRegistry registry) {
this.name = name;
this.tagName = tagName;
this.registry = registry;
}
public void increment(String tagValue){
Counter counter = counters.get(tagValue);
if(counter == null) {
counter = Counter.builder(name).tags(tagName, tagValue).register(registry);
counters.put(tagValue, counter);
}
counter.increment();
}
}

39
src/main/java/eu/dnetlib/scholexplorer/api/controller/DatasourceV1.java Normal file
View File

@ -0,0 +1,39 @@
package eu.dnetlib.scholexplorer.api.controller;
import eu.dnetlib.dhp.schema.sx.api.model.v1.LinkPublisher;
import eu.dnetlib.scholexplorer.api.ScholixException;
import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager;
import io.micrometer.core.annotation.Timed;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v1")
@Tag(name = "Datasources")
public class DatasourceV1 {
@Autowired
private ScholixIndexManager manager;
@Timed(value = "scholix.v1.datasources", description = "Time taken to return all datasources on Version 1.0 of Scholix")
@Operation(summary = "Get all Datasources", description = "returns a list of all datasources")
@GetMapping("/listDatasources")
public List<LinkPublisher> getDatasources() throws ScholixException {
final List<Pair<String, Long>> result = manager.totalLinksByProvider(null);
if (result == null) { return new ArrayList<>(); }
return result.stream().map(p -> new LinkPublisher().name(p.getKey()).totalRelationships(p.getValue().intValue())).collect(Collectors.toList());
}
}

36
src/main/java/eu/dnetlib/scholexplorer/api/controller/HomeController.java Normal file
View File

@ -0,0 +1,36 @@
package eu.dnetlib.scholexplorer.api.controller;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
@Controller
public class HomeController {
@GetMapping({
"/doc", "/swagger"
})
public String apiDoc() {
return "redirect:swagger-ui/index.html";
}
@GetMapping({
"/v1/ui"
})
public String v1Doc() {
return "redirect:/swagger-ui/index.html?urls.primaryName=Scholexplorer%20API%20V1.0";
}
@GetMapping({
"/v2/ui"
})
public String v2Doc() {
return "redirect:/swagger-ui/index.html?urls.primaryName=Scholexplorer%20API%20V2.0";
}
}

42
src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkProviderV2.java Normal file
View File

@ -0,0 +1,42 @@
package eu.dnetlib.scholexplorer.api.controller;
import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType;
import eu.dnetlib.scholexplorer.api.ScholixException;
import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v2")
@Tag(name = "LinkProvider : Operation related to the Link Provider")
public class LinkProviderV2 {
@Autowired
ScholixIndexManager manager;
@Operation(summary = "Get all Link Providers", description = "Return a list of link provider and relative number of relations")
@GetMapping("/LinkProvider")
public List<LinkProviderType> getLinkProviders(
@Parameter(in = ParameterIn.QUERY, description = "Filter the link provider name") @RequestParam(required = false) final String name)
throws ScholixException {
final List<Pair<String, Long>> result = manager.totalLinksByProvider(name);
if (result == null) { return new ArrayList<>(); }
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
}
}

56
src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkPublisherV2.java Normal file
View File

@ -0,0 +1,56 @@
package eu.dnetlib.scholexplorer.api.controller;
import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType;
import eu.dnetlib.scholexplorer.api.ScholixException;
import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v2/LinkPublisher")
@Tag(name = "LinkPublisher : Operation related to the Link Publisher")
public class LinkPublisherV2 {
@Autowired
ScholixIndexManager manager;
@Operation(summary = "Get All Publishers that provide source object", description = "Return a List of all Publishers that provide source objects in Scholix "
+
"links and the total number of links where the source object comes from this publisher")
@GetMapping("/inSource")
public List<LinkProviderType> getInSource(
@Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) final String name)
throws ScholixException {
final List<Pair<String, Long>> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.source, name);
if (result == null) { return new ArrayList<>(); }
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
}
@Operation(summary = "Get All Publishers that provide target object", description = "Return a List of all Publishers that provide source objects in Scholix "
+
"links and the total number of links where the target object comes from this publisher")
@GetMapping("/inTarget")
public List<LinkProviderType> getInTarget(
@Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) final String name)
throws ScholixException {
final List<Pair<String, Long>> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.target, name);
if (result == null) { return new ArrayList<>(); }
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
}
}

78
src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixControllerV2.java Normal file
View File

@ -0,0 +1,78 @@
package eu.dnetlib.scholexplorer.api.controller;
import eu.dnetlib.dhp.schema.sx.api.model.v2.PageResultType;
import eu.dnetlib.dhp.schema.sx.api.model.v2.ScholixType;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.scholexplorer.api.ScholixException;
import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager;
import io.micrometer.core.annotation.Timed;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v2")
@Tag(name = "Links : Operation related to the Scholix Links")
public class ScholixControllerV2 {
@Autowired
ScholixIndexManager repository;
@Timed(value = "scholix.v2.links", description = "Time taken to return links on Version 2.0 of Scholix")
@Operation(summary = "Get Scholix Links")
@GetMapping("/Links")
public PageResultType links(
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships collected from a LinkProvider") final String linkProvider,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target pid") final String targetPid,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target pid type") final String targetPidType,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target published in a Publisher named targetPublisher") final String targetPublisher,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target type (literature, dataset, unknown)") final String targetType,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source pid") final String sourcePid,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source pid type") final String sourcePidType,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source published in a Publisher named sourcePublisher") final String sourcePublisher,
@RequestParam(required = false)
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source type (literature, dataset, unknown)") final String sourceType,
// @Parameter(in = ParameterIn.QUERY,
// description = "Filter scholix Links having collected after this date") String harvestedAfter,
@Parameter(in = ParameterIn.QUERY, description = "select page of result") final Integer page) throws Exception {
if (StringUtils.isEmpty(sourcePid) && StringUtils.isEmpty(targetPid) && StringUtils.isEmpty(sourcePublisher) && StringUtils.isEmpty(targetPublisher)&&StringUtils.isEmpty(sourceType)
&& StringUtils.isEmpty(linkProvider)) {
throw new ScholixException(
"The method requires one of the following parameters: sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider, sourceType");
}
try {
final int currentPage = page != null ? page : 0;
final Pair<Long, List<Scholix>> scholixResult = repository
.linksFromPid(linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, null,currentPage);
final PageResultType pageResult = new PageResultType();
pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10);
pageResult.setTotalLinks(scholixResult.getLeft().intValue());
pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList()));
return pageResult;
} catch (final Throwable e) {
throw new ScholixException("Error on requesting url ", e);
}
}
}

81
src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixLinkControllerV1.java Normal file
View File

@ -0,0 +1,81 @@
package eu.dnetlib.scholexplorer.api.controller;
import eu.dnetlib.dhp.schema.sx.api.model.v1.ScholixV1;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.scholexplorer.api.ScholixException;
import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager;
import io.micrometer.core.annotation.Timed;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.validation.constraints.NotNull;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v1")
public class ScholixLinkControllerV1 {
@Autowired
ScholixIndexManager repository;
@Operation(summary = "Get all Scholix relation collected from a publisher", description = "return a list of scholix object published from a specific publisher")
@GetMapping("/linksFromPublisher")
@Timed(value = "scholix.v1.linksFromPublisher", description = "Time taken to return links on Version 1.0 of Scholix collected from a publisher")
public List<ScholixV1> linksFromPublisher(
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships collected from a publisher", required = true) final String publisher,
@Parameter(in = ParameterIn.QUERY, description = "The page number") @RequestParam(required = false) final Integer page) throws ScholixException {
final int currentPage = page != null ? page : 0;
final Pair<Long, List<Scholix>> scholixResult = repository.linksFromPid(null, null, null, publisher, null, null, null, null, null,null, currentPage);
final List<Scholix> scholixData = scholixResult.getValue();
if (scholixData == null) { return null; }
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
}
@Operation(summary = "Get all Scholix relation collected from a datasource", description = "return a list of scholix object collected from a specific datasource")
@GetMapping("/linksFromDatasource")
@Timed(value = "scholix.v1.linksFromDatasource", description = "Time taken to return links on Version 1.0 of Scholix collected from a LinkProvider")
public List<ScholixV1> linksFromDatasource(
@Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships collected from a LinkProvider") @NotNull final String datasource,
@Parameter(in = ParameterIn.QUERY, description = "The page number") @RequestParam(required = false) final Integer page) throws ScholixException {
final int currentPage = page != null ? page : 0;
final Pair<Long, List<Scholix>> scholixResult = repository.linksFromPid(datasource, null, null, null, null, null, null, null, null, null,currentPage);
final List<Scholix> scholixData = scholixResult.getValue();
if (scholixData == null) { return null; }
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
}
@Operation(summary = "Retrieve all scholix links from a persistent identifier", description = "The linksFromPid endpoint returns a list of scholix object related from a specific persistent identifier")
@GetMapping("/linksFromPid")
@Timed(value = "scholix.v1.linksFromPid", description = "Time taken to return links on Version 1.0 of Scholix related from a specific persistent identifier")
public List<ScholixV1> linksFromPid(
@Parameter(in = ParameterIn.QUERY, description = "persistent Identifier") @NotNull final String pid,
@Parameter(in = ParameterIn.QUERY, description = "Persistent Identifier Type") @RequestParam(required = false) final String pidType,
@Parameter(in = ParameterIn.QUERY, description = "typology target filter should be publication, dataset or unknown") @RequestParam(required = false) final String typologyTarget,
@Parameter(in = ParameterIn.QUERY, description = "a datasource provenance filter of the target relation") @RequestParam(required = false) final String datasourceTarget,
@Parameter(in = ParameterIn.QUERY, description = "The page number") @RequestParam(required = false) final Integer page) throws ScholixException {
final int currentPage = page != null ? page : 0;
final Pair<Long, List<Scholix>> scholixResult =
repository.linksFromPid(datasourceTarget, null, null, null, typologyTarget, pid, pidType, null, null, null, currentPage);
final List<Scholix> scholixData = scholixResult.getValue();
if (scholixData == null) { return null; }
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
}
}

74
src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchClientFactory.java Normal file
View File

@ -0,0 +1,74 @@
package eu.dnetlib.scholexplorer.api.index;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
/**
* The type Elastic search client factory.
*/
public class ElasticSearchClientFactory implements PooledObjectFactory<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> {
private final ElasticSearchProperties elasticSearchProperties;
/**
* Instantiates a new Elastic search client factory.
*
* @param elasticSearchProperties the elastic search properties
*/
public ElasticSearchClientFactory(final ElasticSearchProperties elasticSearchProperties){
this.elasticSearchProperties = elasticSearchProperties;
}
public PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> makeObject() throws Exception {
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo(elasticSearchProperties.getClusterNodes().split(","))
.withConnectTimeout(elasticSearchProperties.getConnectionTimeout())
.withSocketTimeout(elasticSearchProperties.getSocketTimeout())
.build();
RestHighLevelClient cc = RestClients.create(clientConfiguration).rest();
return new DefaultPooledObject<>(new ImmutablePair<>(cc, new ElasticsearchRestTemplate(cc)));
}
public void destroyObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
RestHighLevelClient client = pooledObject.getObject().getLeft();
if(client!=null&&client.ping(RequestOptions.DEFAULT)){
try {
client.close();
}catch (Exception e){
//ignore
}
}
}
public boolean validateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) {
RestHighLevelClient client = pooledObject.getObject().getLeft();
try {
return client.ping(RequestOptions.DEFAULT);
}catch(Exception e){
return false;
}
}
public void activateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
RestHighLevelClient client = pooledObject.getObject().getLeft();
boolean response = client.ping(RequestOptions.DEFAULT);
}
public void passivateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
//nothing
}
}

32
src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchPool.java Normal file
View File

@ -0,0 +1,32 @@
package eu.dnetlib.scholexplorer.api.index;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
/**
* The type Elastic search pool.
*/
public class ElasticSearchPool extends Pool<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> {
private final ElasticSearchProperties elasticSearchProperties;
/**
* Instantiates a new Elastic search pool.
*
* @param elasticSearchProperties the elastic search properties
*/
public ElasticSearchPool(ElasticSearchProperties elasticSearchProperties){
super(elasticSearchProperties, new ElasticSearchClientFactory(elasticSearchProperties));
this.elasticSearchProperties = elasticSearchProperties;
}
/**
* Gets elastic search properties.
*
* @return the elastic search properties
*/
public ElasticSearchProperties getElasticSearchProperties() {
return elasticSearchProperties;
}
}

114
src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchProperties.java Normal file
View File

@ -0,0 +1,114 @@
package eu.dnetlib.scholexplorer.api.index;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import javax.validation.constraints.NotNull;
/**
* The type Elastic search properties.
*/
@Component("elasticSearchProperties")
@ConfigurationProperties(prefix = "scholix.elastic")
public class ElasticSearchProperties extends GenericObjectPoolConfig {
@NotNull
private String clusterNodes;
@NotNull
private String indexName;
@NotNull
private String indexResourceName;
@NotNull
private long connectionTimeout;
@NotNull
private long socketTimeout;
/**
* Gets cluster nodes.
*
* @return the cluster nodes
*/
public String getClusterNodes() {
return clusterNodes;
}
/**
* Sets cluster nodes.
*
* @param clusterNodes the cluster nodes
* @return the cluster nodes
*/
public ElasticSearchProperties setClusterNodes(String clusterNodes) {
this.clusterNodes = clusterNodes;
return this;
}
/**
* Gets index name.
*
* @return the index name
*/
public String getIndexName() {
return indexName;
}
/**
* Sets index name.
*
* @param indexName the index name
* @return the index name
*/
public ElasticSearchProperties setIndexName(String indexName) {
this.indexName = indexName;
return this;
}
public String getIndexResourceName() {
return indexResourceName;
}
public void setIndexResourceName(String indexResourceName) {
this.indexResourceName = indexResourceName;
}
/**
* Gets connection timeout.
*
* @return the connection timeout
*/
public long getConnectionTimeout() {
return connectionTimeout;
}
/**
* Sets connection timeout.
*
* @param connectionTimeout the connection timeout
* @return the connection timeout
*/
public ElasticSearchProperties setConnectionTimeout(long connectionTimeout) {
this.connectionTimeout = connectionTimeout;
return this;
}
/**
* Gets socket timeout.
*
* @return the socket timeout
*/
public long getSocketTimeout() {
return socketTimeout;
}
/**
* Sets socket timeout.
*
* @param socketTimeout the socket timeout
* @return the socket timeout
*/
public ElasticSearchProperties setSocketTimeout(long socketTimeout) {
this.socketTimeout = socketTimeout;
return this;
}
}

232
src/main/java/eu/dnetlib/scholexplorer/api/index/Pool.java Normal file
View File

@ -0,0 +1,232 @@
package eu.dnetlib.scholexplorer.api.index;
import eu.dnetlib.scholexplorer.api.ScholixException;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
/**
* When using the Java High Level REST Client provided by the Elasticsearch official website, it is found that there is no in the client API.
* Connecting to connect the pool, create a new connection every time, this is impact in high concurrency situation, so it is ready to be on the client
* API increases the concept of pool.
*
* Fortunately, we don't need to turn your weight to write the implementation of the connection pool, because Apache provides us with the general framework of the connection pool.
* Commons-pool2, and we only need to implement some logic according to the frame design. Used in the REDIS client API
* Jedispool is based on Commons-pool2 implementation.
*
* Let's take a look at how to achieve it.
*
* First we have to create a pool class, this pool introduces GenericObjectPool in Commons-pool2 through dependent manner. In this class
* In, we define how to borrow objects and returns objects from the pool.
*
* @param <T> the type parameter
*/
public class Pool<T> implements Cloneable {
/**
* The Internal pool.
*/
protected GenericObjectPool<T> internalPool ;
/**
* Instantiates a new Pool.
*/
public Pool(){
super();
}
/**
* Instantiates a new Pool.
*
* @param poolConfig the pool config
* @param factory the factory
*/
public Pool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory<T> factory){
initPool(poolConfig, factory);
}
/**
* Init pool.
*
* @param poolConfig the pool config
* @param factory the factory
*/
public void initPool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory<T> factory) {
if (this.internalPool != null) {
try {
closeInternalPool();
} catch (Exception e) {
}
}
this.internalPool = new GenericObjectPool<T>(factory, poolConfig);
}
/**
* Close internal pool.
*
* @throws ScholixException the scholix exception
*/
protected void closeInternalPool() throws ScholixException {
try {
internalPool.close();
} catch (Exception e) {
throw new ScholixException("Could not destroy the pool", e);
}
}
/**
* Gets resource.
*
* @return the resource
* @throws ScholixException the scholix exception
*/
public T getResource() throws ScholixException {
try {
return internalPool.borrowObject();
} catch (Exception e) {
throw new ScholixException("Could not get a resource from the pool", e);
}
}
/**
* Return resource.
*
* @param resource the resource
* @throws ScholixException the scholix exception
*/
public void returnResource(final T resource) throws ScholixException {
if (resource != null) {
returnResourceObject(resource);
}
}
private void returnResourceObject(final T resource) throws ScholixException {
if (resource == null) {
return;
}
try {
internalPool.returnObject(resource);
} catch (Exception e) {
throw new ScholixException("Could not return the resource to the pool", e);
}
}
/**
* Return broken resource.
*
* @param resource the resource
* @throws ScholixException the scholix exception
*/
public void returnBrokenResource(final T resource) throws ScholixException {
if (resource != null) {
returnBrokenResourceObject(resource);
}
}
private void returnBrokenResourceObject(T resource) throws ScholixException {
try {
internalPool.invalidateObject(resource);
} catch (Exception e) {
throw new ScholixException("Could not return the resource to the pool", e);
}
}
/**
* Destroy.
*
* @throws ScholixException the scholix exception
*/
public void destroy() throws ScholixException {
closeInternalPool();
}
/**
* Gets num active.
*
* @return the num active
*/
public int getNumActive() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getNumActive();
}
/**
* Gets num idle.
*
* @return the num idle
*/
public int getNumIdle() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getNumIdle();
}
/**
* Gets num waiters.
*
* @return the num waiters
*/
public int getNumWaiters() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getNumWaiters();
}
/**
* Gets mean borrow wait time millis.
*
* @return the mean borrow wait time millis
*/
public long getMeanBorrowWaitTimeMillis() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getMeanBorrowWaitTimeMillis();
}
/**
* Gets max borrow wait time millis.
*
* @return the max borrow wait time millis
*/
public long getMaxBorrowWaitTimeMillis() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getMaxBorrowWaitTimeMillis();
}
private boolean poolInactive() {
return this.internalPool == null || this.internalPool.isClosed();
}
/**
* Add objects.
*
* @param count the count
* @throws Exception the exception
*/
public void addObjects(int count) throws Exception {
try {
for (int i = 0; i < count; i++) {
this.internalPool.addObject();
}
} catch (Exception e) {
throw new Exception("Error trying to add idle objects", e);
}
}
}

401
src/main/java/eu/dnetlib/scholexplorer/api/index/ScholixIndexManager.java Normal file
View File

@ -0,0 +1,401 @@
package eu.dnetlib.scholexplorer.api.index;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.dhp.schema.sx.scholix.ScholixEntityId;
import eu.dnetlib.dhp.schema.sx.scholix.ScholixRelationship;
import eu.dnetlib.dhp.schema.sx.scholix.ScholixResource;
import eu.dnetlib.dhp.schema.sx.scholix.flat.ScholixFlat;
import eu.dnetlib.scholexplorer.api.ScholixException;
import eu.dnetlib.scholexplorer.api.TaggedCounter;
import eu.dnetlib.scholexplorer.api.model.Summary;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.stereotype.Component;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.elasticsearch.index.query.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class ScholixIndexManager {
@Autowired
ElasticSearchProperties elasticSearchProperties;
/**
* The Elasticsearch template.
*/
@Autowired
ElasticSearchPool connectionPool;
final ObjectMapper mapper = new ObjectMapper();
/**
* The enum Pid type prefix.
*/
public enum RelationPrefix {
/**
* Source pid type prefix.
*/
source,
/**
* Target pid type prefix.
*/
target
}
@Autowired
TaggedCounter myCounter;
private List<String> extractIdentifiersFromScholix(SearchHits<ScholixFlat> scholix) {
return scholix.stream()
.flatMap(s ->
Stream.of(
s.getContent().getSourceId(),
s.getContent().getTargetId()))
.distinct()
.toList();
}
private Map<String, ScholixResource> retrieveResources(ElasticsearchRestTemplate client, List<String> ids) {
final IdsQueryBuilder qb = new IdsQueryBuilder().addIds(ids.toArray(String[]::new));
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(qb)
.withPageable(PageRequest.of(0, ids.size()))
.build();
SearchHits<Summary> result = client.search(searchQuery, Summary.class, IndexCoordinates.of(elasticSearchProperties.getIndexResourceName()));
return result.stream().map(r -> {
try {
return mapper.readValue(r.getContent().getBody(), ScholixResource.class);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}).collect(Collectors.toMap(
ScholixResource::getDnetIdentifier,
v -> v,
(a, b) -> a
));
}
private Scholix generateScholix(ScholixFlat flat, ScholixResource source, ScholixResource target) throws ScholixException {
if (flat == null || source == null || target == null)
throw new ScholixException("Error generating scholix null input");
final Scholix scholix = new Scholix();
scholix.setSource(source);
scholix.setTarget(target);
scholix.setIdentifier(flat.getIdentifier());
final ScholixRelationship r = new ScholixRelationship();
r.setSchema("datacite");
r.setName(flat.getRelationType().toLowerCase());
scholix.setRelationship(r);
scholix.setPublicationDate(flat.getPublicationDate());
scholix.setLinkprovider(flat.getLinkProviders().stream().map(p -> {
final ScholixEntityId eid = new ScholixEntityId();
eid.setName(p);
return eid;
}).toList());
final Map<String, ScholixEntityId> publishers = new HashMap<>();
if (source.getPublisher() != null)
source.getPublisher().forEach(p -> publishers.put(p.getName(), p));
if (target.getPublisher() != null)
target.getPublisher().forEach(p -> publishers.put(p.getName(), p));
scholix.setPublisher(publishers.values().stream().toList());
return scholix;
}
private QueryBuilder createFinalQuery(final List<QueryBuilder> queries) throws ScholixException {
if (queries == null || queries.isEmpty())
throw new ScholixException("the list of queries must be not empty");
if (queries.size() == 1) {
return queries.get(0);
} else {
final BoolQueryBuilder b = new BoolQueryBuilder();
b.must().addAll(queries);
return b;
}
}
public List<Pair<String, Long>> totalLinksByProvider(final String filterName) throws ScholixException {
final QueryBuilder query = StringUtils.isNoneBlank(filterName) ? QueryBuilders.termQuery("linkProviders", filterName) : QueryBuilders.matchAllQuery();
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(query)
.withSearchType(SearchType.DEFAULT)
.withPageable(PageRequest.of(0, 10))
.addAggregation(
AggregationBuilders.terms("genres").field("linkProviders").size(100)
.minDocCount(1))
.build();
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = null;
try {
resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
final SearchHits<ScholixFlat> hits = client.search(searchQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
final Aggregations aggregations = hits.getAggregations();
if (aggregations == null)
return null;
return ((ParsedStringTerms) aggregations.get("genres")).getBuckets().stream().map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount())).collect(Collectors.toList());
} catch (ScholixException e) {
throw e;
} finally {
if (connectionPool != null) {
connectionPool.returnResource(resource);
}
}
}
private QueryBuilder createLinkPublisherQuery(final RelationPrefix prefix, final String publisher) throws ScholixException {
if (prefix == null) {
throw new ScholixException("prefix cannot be null");
}
return new NestedQueryBuilder(String.format("%s.publisher", prefix), new TermQueryBuilder(String.format("%s.publisher.name", prefix), publisher), ScoreMode.None);
}
public List<Pair<String, Long>> totalLinksPublisher(final RelationPrefix prefix, final String filterName) throws ScholixException {
final QueryBuilder query = StringUtils.isNoneBlank(filterName) ? createLinkPublisherQuery(prefix, filterName) : QueryBuilders.matchAllQuery();
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(query)
.withSearchType(SearchType.DEFAULT)
.withPageable(PageRequest.of(0, 10))
.addAggregation(
AggregationBuilders.terms("publishers").field(String.format("%sPublisher", prefix.toString())).size(100)
.minDocCount(1))
.build();
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = null;
try {
resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
final SearchHits<ScholixFlat> hits = client.search(searchQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
final Aggregations aggregations = hits.getAggregations();
if (aggregations == null)
return null;
return ((ParsedStringTerms) aggregations.get("publishers")).getBuckets().stream().map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount())).collect(Collectors.toList());
} catch (ScholixException e) {
throw e;
} finally {
if (connectionPool != null) {
connectionPool.returnResource(resource);
}
}
}
private void incrementPidCounter(RelationPrefix prefix, String value) {
switch (value.toLowerCase()) {
case "doi": {
myCounter.increment(String.format("%s_doi", prefix));
break;
}
case "pmc": {
myCounter.increment(String.format("%s_pmc", prefix));
break;
}
default:
myCounter.increment(String.format("%s_other", prefix));
}
}
/**
* Links from pid pair.
*
* @param linkProvider the link provider
* @param targetPid the target pid
* @param targetPidType the target pid type
* @param targetPublisher the target publisher
* @param targetType the target type
* @param sourcePid the source pid
* @param sourcePidType the source pid type
* @param sourcePublisher the source publisher
* @param sourceType the source type
* @param page the page
* @return the pair
* @throws ScholixException the scholix exception
*/
public Pair<Long, List<Scholix>> linksFromPid(final String linkProvider,
final String targetPid, final String targetPidType, final String targetPublisher,
final String targetType, final String sourcePid, final String sourcePidType,
final String sourcePublisher,
final String sourceType,
final String relation,
final Integer page) throws ScholixException {
if (sourcePid == null && sourcePidType == null && sourceType == null && targetType == null && targetPid == null && targetPidType == null && sourcePublisher == null && targetPublisher == null && linkProvider == null)
throw new ScholixException("One of sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider should be not null");
final List<QueryBuilder> queries = new ArrayList<>();
if (StringUtils.isNoneBlank(linkProvider)) {
myCounter.increment("linkProvider");
queries.add(QueryBuilders.termQuery("linkProviders", linkProvider));
}
if (StringUtils.isNoneBlank(targetPid)) {
myCounter.increment("targetPid");
queries.add(QueryBuilders.termQuery("targetPid", targetPid));
}
if (StringUtils.isNoneBlank(sourcePid)) {
myCounter.increment("sourcePid");
queries.add(QueryBuilders.termQuery("sourcePid", sourcePid));
}
if (StringUtils.isNoneBlank(targetPidType)) {
assert targetPidType != null;
incrementPidCounter(RelationPrefix.target, targetPidType);
queries.add(QueryBuilders.termQuery("targetPidType", targetPidType));
}
if (StringUtils.isNoneBlank(sourcePidType)) {
assert sourcePidType != null;
incrementPidCounter(RelationPrefix.source, sourcePidType);
queries.add(QueryBuilders.termQuery("sourcePidType", sourcePidType));
}
if (StringUtils.isNoneBlank(targetType)) {
myCounter.increment(String.format("targetType_%s", targetType));
queries.add(QueryBuilders.termQuery("targetType", targetType));
}
if (StringUtils.isNoneBlank(sourceType)) {
assert sourceType != null;
myCounter.increment(String.format("sourceType_%s", sourceType));
queries.add(QueryBuilders.termQuery("sourceType", sourceType));
}
if (StringUtils.isNoneBlank(targetPublisher)) {
myCounter.increment("targetPublisher");
queries.add(QueryBuilders.termQuery("targetPublisher", targetPublisher));
}
if (StringUtils.isNoneBlank(relation)) {
myCounter.increment("targetPublisher");
queries.add(QueryBuilders.termQuery("relationType", relation));
}
QueryBuilder result = createFinalQuery(queries);
NativeSearchQuery finalQuery = new NativeSearchQueryBuilder()
.withQuery(result)
.withPageable(PageRequest.of(page, 100))
.build();
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = null;
try {
resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
long tt = client.count(finalQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
SearchHits<ScholixFlat> scholixRes = client.search(finalQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
if (tt > 0) {
final Map<String, ScholixResource> idMap = retrieveResources(client, extractIdentifiersFromScholix(scholixRes));
return new ImmutablePair<>(tt, scholixRes.stream().map(SearchHit::getContent).map(s -> {
try {
return generateScholix(s, idMap.get(s.getSourceId()), idMap.get(s.getTargetId()));
} catch (ScholixException e) {
throw new RuntimeException(e);
}
}).toList());
} else return new ImmutablePair<>(tt, new ArrayList<>());
} catch (ScholixException e) {
throw e;
} finally {
if (connectionPool != null) {
connectionPool.returnResource(resource);
}
}
}
public List<Scholix> findPage(final String relType) throws ScholixException {
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = null;
try {
resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.termQuery("relationType", relType))
.withSearchType(SearchType.DEFAULT)
.withPageable(PageRequest.of(0, 20))
.build();
SearchHits<ScholixFlat> search = client.search(searchQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
final Map<String, ScholixResource> idMap = retrieveResources(client, extractIdentifiersFromScholix(search));
return search.stream().map(SearchHit::getContent).map(s -> {
try {
return generateScholix(s, idMap.get(s.getSourceId()), idMap.get(s.getTargetId()));
} catch (ScholixException e) {
throw new RuntimeException(e);
}
}).toList();
} catch (Throwable e) {
System.out.println(e.getMessage());
} finally {
if (connectionPool != null && resource != null) {
connectionPool.returnResource(resource);
}
}
return null;
}
}

13
src/main/java/eu/dnetlib/scholexplorer/api/model/Summary.java Normal file
View File

@ -0,0 +1,13 @@
package eu.dnetlib.scholexplorer.api.model;
public class Summary {
private String body;
public String getBody() {
return body;
}
public void setBody(String body) {
this.body = body;
}
}

43
src/main/resources/application.properties Normal file
View File

@ -0,0 +1,43 @@
spring.application.name=scholexplorer-api
spring.main.banner-mode = console
springdoc.api-docs.path=/api-docs
springdoc.swagger-ui.path=/scholexplorer-api
server.public_url =
server.public_desc = API Base URL
logging.level.root = INFO
dhp.swagger.api.host = localhost:8080
#dhp.swagger.api.host = api.scholexplorer.openaire.eu
dhp.swagger.api.basePath = /
management.endpoints.web.exposure.include = health, metrics, prometheus
management.metrics.tags.application=${spring.application.name}
#maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml
#
#spring.thymeleaf.cache=false
#
#spring.metrics.export.prometheus.enabled = true
#spring.metrics.export.prometheus.port = 8080
#management.endpoints.web.exposure.include = phealth, metrics, prometheus
#management.endpoints.web.base-path = /
#management.endpoints.web.path-mapping.prometheus = metrics
#management.endpoints.web.path-mapping.health = health
#management.endpoint.health.show-details = always
#
#management.metrics.distribution.percentiles-histogram.http.server.requests=false
#management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
#management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999
#scholix.elastic.clusterNodes = 10.19.65.51:9200,10.19.65.52:9200,10.19.65.53:9200,10.19.65.54:9200
scholix.elastic.clusterNodes = localhost:9200
scholix.elastic.indexName = scholix
scholix.elastic.indexResourceName = summary
scholix.elastic.socketTimeout = 60000
scholix.elastic.connectionTimeout= 60000

BIN
src/main/resources/static/logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

1
src/main/resources/static/logo.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

173
src/test/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplicationTests.java Normal file
View File

@ -0,0 +1,173 @@
package eu.dnetlib.scholexplorer.api;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.util.*;
class TestResult {
public long totalTimeMs;
public int totalItems;
public String relationName;
public long getTotalTimeMs() {
return totalTimeMs;
}
public TestResult setTotalTimeMs(long totalTimeMs) {
this.totalTimeMs = totalTimeMs;
return this;
}
public int getTotalItems() {
return totalItems;
}
public TestResult setTotalItems(int totalItems) {
this.totalItems = totalItems;
return this;
}
public String getRelationName() {
return relationName;
}
public TestResult setRelationName(String relationName) {
this.relationName = relationName;
return this;
}
}
@SpringBootTest
class ScholexplorerApiApplicationTests {
@Autowired
ScholixIndexManager scholixRepo;
final ObjectMapper mapper = new ObjectMapper();
final List<String> relations = Arrays.asList("IsDerivedFrom",
"Continues",
"References",
"HasVersion",
"IsVersionOf",
"IsSupplementTo",
"IsDocumentedBy",
"Documents",
"IsContinuedBy",
"IsSupplementedBy",
"IsSourceOf",
"Reviews",
"Cites",
"IsAmongTopNSimilarDocuments",
"IsPartOf",
"HasPart",
"IsIdenticalTo",
"IsNewVersionOf",
"IsRelatedTo",
"HasAmongTopNSimilarDocuments",
"IsCitedBy",
"IsPreviousVersionOf",
"IsReferencedBy",
"IsVariantFormOf",
"IsCompiledBy",
"IsReviewedBy",
"IsDescribedBy",
"Compiles",
"IsOriginalFormOf",
"Describes");
private List<TestResult> executeTest() throws Exception {
Random rand = new Random();
final List<TestResult> infos = new ArrayList<>();
String currentRel1 =relations.get(rand.nextInt(relations.size()));
long start = System.nanoTime();
List<Scholix> result = scholixRepo.findPage(currentRel1);
long total = (System.nanoTime() - start) /1000000;
int cnt = result.size();
infos.add(new TestResult().setRelationName(currentRel1).setTotalItems(cnt).setTotalTimeMs(total));
String currentRel2 =relations.get(rand.nextInt(relations.size()));
start = System.nanoTime();
result = scholixRepo.findPage(currentRel2);
long total2 = (System.nanoTime() - start) /1000000;
int cnt2 = result.size();
infos.add(new TestResult().setRelationName(currentRel2).setTotalItems(cnt2).setTotalTimeMs(total2));
String currentRel3 =relations.get(rand.nextInt(relations.size()));
start = System.nanoTime();
result = scholixRepo.findPage(currentRel3);
long total3 = (System.nanoTime() - start) /1000000;
int cnt3 = result.size();
infos.add(new TestResult().setRelationName(currentRel3).setTotalItems(cnt3).setTotalTimeMs(total3));
return infos;
}
@Test
void contextLoads() throws Exception {
final List<TestResult> infos = new ArrayList<>();
for (int i = 0; i < 200; i++) {
infos.addAll(executeTest());
}
LongSummaryStatistics summary = infos.stream().map(TestResult::getTotalTimeMs).mapToLong(Long::longValue).summaryStatistics();
System.out.println(summary.getMax());
System.out.println(summary.getMin());
System.out.println(summary.getAverage());
}
@Test
void testlinksFromPid() throws ScholixException {
Pair<Long, List<Scholix>> result = scholixRepo.linksFromPid(null, null, "doi", null, null, null, "pmid", null, null, null, 0);
result.getRight().forEach(
s -> Assertions.assertTrue(s.getTarget().getIdentifier().stream().anyMatch(p -> p.getSchema().equals("doi")))
);
result.getRight().forEach(
s -> Assertions.assertTrue(s.getSource().getIdentifier().stream().anyMatch(p -> p.getSchema().equals("pmid")))
);
result = scholixRepo.linksFromPid(null, null, null, null, "dataset", null, null, null, "publication","IsSupplementedBy", 0);
System.out.println(result.getLeft());
result.getRight().forEach(
s -> {
Assertions.assertEquals("dataset", s.getTarget().getObjectType());
Assertions.assertEquals("publication", s.getSource().getObjectType());
Assertions.assertEquals("issupplementedby", s.getRelationship().getName());
});
result = scholixRepo.linksFromPid(null, null, null, null, "publication", null, null, null, "publication","IsVersionOf", 0);
System.out.println(result.getLeft());
result.getRight().forEach(
s -> {
Assertions.assertEquals("publication", s.getTarget().getObjectType());
Assertions.assertEquals("publication", s.getSource().getObjectType());
Assertions.assertEquals("IsVersionOf".toLowerCase(), s.getRelationship().getName());
});
}
}

38
src/test/resources/application.properties Normal file
View File

@ -0,0 +1,38 @@
spring.application.name=scholexplorer-api
spring.main.banner-mode = console
server.public_url =
server.public_desc = API Base URL
logging.level.root = INFO
dhp.swagger.api.host = localhost:8080
#dhp.swagger.api.host = api.scholexplorer.openaire.eu
dhp.swagger.api.basePath = /
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml
#
#spring.thymeleaf.cache=false
#
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
management.endpoint.health.show-details = always
management.metrics.distribution.percentiles-histogram.http.server.requests=false
management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999
#scholix.elastic.clusterNodes = 10.19.65.51:9200,10.19.65.52:9200,10.19.65.53:9200,10.19.65.54:9200
scholix.elastic.clusterNodes = localhost:9200
scholix.elastic.indexName = scholix
scholix.elastic.indexResourceName = summary
scholix.elastic.socketTimeout = 60000
scholix.elastic.connectionTimeout= 60000