From 0a23135d10ebf8434d316d206efe3bca20cc3e24 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 6 Jun 2024 16:10:04 +0200 Subject: [PATCH] imported new version of the api --- .gitignore | 2 + pom.xml | 93 ++++ .../scholexplorer/api/RestClientConfig.java | 26 ++ .../api/ScholexplorerApiApplication.java | 122 ++++++ .../api/ScholixAPIConstants.java | 23 + .../scholexplorer/api/ScholixException.java | 35 ++ .../scholexplorer/api/TaggedCounter.java | 33 ++ .../api/controller/DatasourceV1.java | 39 ++ .../api/controller/HomeController.java | 36 ++ .../api/controller/LinkProviderV2.java | 42 ++ .../api/controller/LinkPublisherV2.java | 56 +++ .../api/controller/ScholixControllerV2.java | 78 ++++ .../controller/ScholixLinkControllerV1.java | 81 ++++ .../api/index/ElasticSearchClientFactory.java | 74 ++++ .../api/index/ElasticSearchPool.java | 32 ++ .../api/index/ElasticSearchProperties.java | 114 +++++ .../dnetlib/scholexplorer/api/index/Pool.java | 232 ++++++++++ .../api/index/ScholixIndexManager.java | 401 ++++++++++++++++++ .../scholexplorer/api/model/Summary.java | 13 + src/main/resources/application.properties | 43 ++ src/main/resources/static/logo.png | Bin 0 -> 13093 bytes src/main/resources/static/logo.svg | 1 + .../api/ScholexplorerApiApplicationTests.java | 173 ++++++++ src/test/resources/application.properties | 38 ++ 24 files changed, 1787 insertions(+) create mode 100644 pom.xml create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/RestClientConfig.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplication.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/ScholixAPIConstants.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/ScholixException.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/TaggedCounter.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/controller/DatasourceV1.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/controller/HomeController.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkProviderV2.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkPublisherV2.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixControllerV2.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixLinkControllerV1.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchClientFactory.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchPool.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchProperties.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/index/Pool.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/index/ScholixIndexManager.java create mode 100644 src/main/java/eu/dnetlib/scholexplorer/api/model/Summary.java create mode 100644 src/main/resources/application.properties create mode 100644 src/main/resources/static/logo.png create mode 100644 src/main/resources/static/logo.svg create mode 100644 src/test/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplicationTests.java create mode 100644 src/test/resources/application.properties diff --git a/.gitignore b/.gitignore index 9154f4c..9738e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ # Compiled class file *.class +.idea +target # Log file *.log diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..e64ccfd --- /dev/null +++ b/pom.xml @@ -0,0 +1,93 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.3.0 + + + eu.dnetlib + scholexplorer-api + 0.0.1-SNAPSHOT + scholexplorer-api + Demo project for Spring Boot + + 17 + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + runtime + + + org.springframework + spring-aspects + + + org.apache.commons + commons-pool2 + 2.11.1 + + + javax.annotation + javax.annotation-api + 1.3.2 + + + eu.dnetlib.dhp + dhp-schemas + 6.1.3-FLAT-SCHOLIX + + + org.springframework.data + spring-data-elasticsearch + 4.2.2 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 7.6.2 + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.5.0 + + + io.swagger.core.v3 + swagger-annotations + 2.2.22 + + + + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/RestClientConfig.java b/src/main/java/eu/dnetlib/scholexplorer/api/RestClientConfig.java new file mode 100644 index 0000000..4d2b49a --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/RestClientConfig.java @@ -0,0 +1,26 @@ +package eu.dnetlib.scholexplorer.api; + + +import eu.dnetlib.scholexplorer.api.index.ElasticSearchPool; +import eu.dnetlib.scholexplorer.api.index.ElasticSearchProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RestClientConfig { + + @Autowired + private ElasticSearchProperties elasticSearchProperties; + + + @Bean + public ElasticSearchPool connectionPool() { + + elasticSearchProperties.setMaxIdle(5); + elasticSearchProperties.setMaxTotal(10); + return new ElasticSearchPool(elasticSearchProperties); + } + + +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplication.java b/src/main/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplication.java new file mode 100644 index 0000000..7fb87a7 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplication.java @@ -0,0 +1,122 @@ +package eu.dnetlib.scholexplorer.api; + +import io.micrometer.core.aop.TimedAspect; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.License; +import org.apache.commons.lang3.StringUtils; +import org.springdoc.core.models.GroupedOpenApi; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import io.swagger.v3.oas.models.servers.Server; +import io.swagger.v3.oas.models.info.Info; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; + +@SpringBootApplication +public class ScholexplorerApiApplication { + + @Value("${server.public_url}") + private String serverPublicUrl; + + @Value("${server.public_desc}") + private String serverPublicDesc; + + protected static final License AGPL_3_LICENSE = + new License().name("GNU Affero General Public License v3.0 or later").url("https://www.gnu.org/licenses/agpl-3.0.txt"); + + private final double scale = 1000000000; + + private final double[] histogramValues = new double[] { + .005 * scale, .01 * scale, .25 * scale, .5 * scale, .75 * scale, scale, 2.5 * scale, 5.0 * scale, 7.5 * scale, 10.0 * scale + }; + + @Bean + public TimedAspect timedAspect(final MeterRegistry meterRegistry) { + final MeterFilter mf = new MeterFilter() { + + @Override + public DistributionStatisticConfig configure(final Meter.Id id, final DistributionStatisticConfig config) { + if (id.getName().startsWith(ScholixAPIConstants.SCHOLIX_COUNTER_PREFIX)) { + + return DistributionStatisticConfig.builder() + .percentilesHistogram(false) + .serviceLevelObjectives(histogramValues) + .build() + .merge(config); + } + return config; + } + }; + meterRegistry.config().meterFilter(mf); + return new TimedAspect(meterRegistry); + } + + + @Bean + public TaggedCounter myCounter(final MeterRegistry meterRegistry) { + + return new TaggedCounter(ScholixAPIConstants.SCHOLIX_MANAGER_COUNTER_NAME, ScholixAPIConstants.SCHOLIX_MANAGER_TAG_NAME, meterRegistry); + } + + @Bean + public GroupedOpenApi publicApiV1() { + return GroupedOpenApi.builder() + .group(ScholixAPIConstants.API_V1_NAME) + .pathsToMatch("/v1/**") + .build(); + } + + @Bean + public GroupedOpenApi publicApiV2() { + return GroupedOpenApi.builder() + .group(ScholixAPIConstants.API_V2_NAME) + .pathsToMatch("/v2/**") + .build(); + } + + + @Bean + public OpenAPI newSwaggerDocket() { + final List servers = new ArrayList<>(); + if (StringUtils.isNotBlank(serverPublicUrl)) { + final Server server = new Server(); + server.setUrl(serverPublicUrl); + server.setDescription(serverPublicDesc); + servers.add(server); + } + + return new OpenAPI() + .servers(servers) + .info(getSwaggerInfo()) + .tags(new ArrayList<>()); + } + + private Info getSwaggerInfo() { + return new Info() + .title(swaggerTitle()) + .description(swaggerDesc()) + .version("1.0") + .license(AGPL_3_LICENSE); + } + + protected String swaggerTitle() { + return "ScholeExplorer APIs"; + } + + protected String swaggerDesc() { + return ScholixAPIConstants.API_DESCRIPTION; + } + + public static void main(String[] args) { + SpringApplication.run(ScholexplorerApiApplication.class, args); + } + +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/ScholixAPIConstants.java b/src/main/java/eu/dnetlib/scholexplorer/api/ScholixAPIConstants.java new file mode 100644 index 0000000..94d6773 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/ScholixAPIConstants.java @@ -0,0 +1,23 @@ +package eu.dnetlib.scholexplorer.api; + +public class ScholixAPIConstants { + + public static final String API_V1_NAME = "Scholexplorer API V1.0"; + public static final String API_V2_NAME = "Scholexplorer API V2.0"; + + public static String API_DESCRIPTION ="

\"ScholeXplorer\"

" + + "The Scholix Swagger API allows clients to run REST queries over the Scholexplorer index in order to fetch links matching given criteria. In the current version, clients can search for:" + + ""; + + + public static String SCHOLIX_MANAGER_COUNTER_NAME= "scholixLinkCounter"; + public static final String SCHOLIX_MANAGER_TAG_NAME = "links"; + + public static String SCHOLIX_COUNTER_PREFIX = "scholix"; + + + + +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/ScholixException.java b/src/main/java/eu/dnetlib/scholexplorer/api/ScholixException.java new file mode 100644 index 0000000..b59a4f8 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/ScholixException.java @@ -0,0 +1,35 @@ +package eu.dnetlib.scholexplorer.api; + + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) +public class ScholixException extends Exception{ + + private static final long serialVersionUID = -3414428892721711308L; + + + public ScholixException() { + super(); + } + + public ScholixException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public ScholixException(String message, Throwable cause) { + super(message, cause); + } + + public ScholixException(String message) { + super(message); + } + + public ScholixException(Throwable cause) { + super(cause); + } + + +} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/TaggedCounter.java b/src/main/java/eu/dnetlib/scholexplorer/api/TaggedCounter.java new file mode 100644 index 0000000..088dd90 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/TaggedCounter.java @@ -0,0 +1,33 @@ +package eu.dnetlib.scholexplorer.api; + +import io.micrometer.core.instrument.Counter; + +import io.micrometer.core.instrument.MeterRegistry; +import java.util.HashMap; +import java.util.Map; + + +public class TaggedCounter { + + private final String name; + private final String tagName; + private final MeterRegistry registry; + private final Map counters = new HashMap<>(); + + + public TaggedCounter(String name, String tagName, MeterRegistry registry) { + this.name = name; + this.tagName = tagName; + this.registry = registry; + } + + + public void increment(String tagValue){ + Counter counter = counters.get(tagValue); + if(counter == null) { + counter = Counter.builder(name).tags(tagName, tagValue).register(registry); + counters.put(tagValue, counter); + } + counter.increment(); + } +} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/controller/DatasourceV1.java b/src/main/java/eu/dnetlib/scholexplorer/api/controller/DatasourceV1.java new file mode 100644 index 0000000..3510f9f --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/controller/DatasourceV1.java @@ -0,0 +1,39 @@ +package eu.dnetlib.scholexplorer.api.controller; + +import eu.dnetlib.dhp.schema.sx.api.model.v1.LinkPublisher; +import eu.dnetlib.scholexplorer.api.ScholixException; +import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager; +import io.micrometer.core.annotation.Timed; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/v1") +@Tag(name = "Datasources") +public class DatasourceV1 { + + @Autowired + private ScholixIndexManager manager; + + @Timed(value = "scholix.v1.datasources", description = "Time taken to return all datasources on Version 1.0 of Scholix") + @Operation(summary = "Get all Datasources", description = "returns a list of all datasources") + @GetMapping("/listDatasources") + public List getDatasources() throws ScholixException { + + final List> result = manager.totalLinksByProvider(null); + + if (result == null) { return new ArrayList<>(); } + + return result.stream().map(p -> new LinkPublisher().name(p.getKey()).totalRelationships(p.getValue().intValue())).collect(Collectors.toList()); + + } +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/controller/HomeController.java b/src/main/java/eu/dnetlib/scholexplorer/api/controller/HomeController.java new file mode 100644 index 0000000..2699e63 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/controller/HomeController.java @@ -0,0 +1,36 @@ +package eu.dnetlib.scholexplorer.api.controller; + + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; + +@Controller +public class HomeController { + + @GetMapping({ + "/doc", "/swagger" + }) + public String apiDoc() { + return "redirect:swagger-ui/index.html"; + } + + @GetMapping({ + "/v1/ui" + }) + public String v1Doc() { + return "redirect:/swagger-ui/index.html?urls.primaryName=Scholexplorer%20API%20V1.0"; + } + + + @GetMapping({ + "/v2/ui" + }) + public String v2Doc() { + return "redirect:/swagger-ui/index.html?urls.primaryName=Scholexplorer%20API%20V2.0"; + } + + + + + +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkProviderV2.java b/src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkProviderV2.java new file mode 100644 index 0000000..233ea7a --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkProviderV2.java @@ -0,0 +1,42 @@ +package eu.dnetlib.scholexplorer.api.controller; + +import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType; +import eu.dnetlib.scholexplorer.api.ScholixException; +import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/v2") +@Tag(name = "LinkProvider : Operation related to the Link Provider") +public class LinkProviderV2 { + + @Autowired + ScholixIndexManager manager; + + @Operation(summary = "Get all Link Providers", description = "Return a list of link provider and relative number of relations") + @GetMapping("/LinkProvider") + public List getLinkProviders( + @Parameter(in = ParameterIn.QUERY, description = "Filter the link provider name") @RequestParam(required = false) final String name) + throws ScholixException { + + final List> result = manager.totalLinksByProvider(name); + + if (result == null) { return new ArrayList<>(); } + + return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList()); + + } +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkPublisherV2.java b/src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkPublisherV2.java new file mode 100644 index 0000000..e52a7d7 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/controller/LinkPublisherV2.java @@ -0,0 +1,56 @@ +package eu.dnetlib.scholexplorer.api.controller; + +import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType; +import eu.dnetlib.scholexplorer.api.ScholixException; +import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/v2/LinkPublisher") +@Tag(name = "LinkPublisher : Operation related to the Link Publisher") +public class LinkPublisherV2 { + + @Autowired + ScholixIndexManager manager; + + @Operation(summary = "Get All Publishers that provide source object", description = "Return a List of all Publishers that provide source objects in Scholix " + + + "links and the total number of links where the source object comes from this publisher") + @GetMapping("/inSource") + public List getInSource( + @Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) final String name) + throws ScholixException { + final List> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.source, name); + + if (result == null) { return new ArrayList<>(); } + + return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList()); + } + + @Operation(summary = "Get All Publishers that provide target object", description = "Return a List of all Publishers that provide source objects in Scholix " + + + "links and the total number of links where the target object comes from this publisher") + @GetMapping("/inTarget") + public List getInTarget( + @Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) final String name) + throws ScholixException { + final List> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.target, name); + + if (result == null) { return new ArrayList<>(); } + + return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList()); + } +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixControllerV2.java b/src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixControllerV2.java new file mode 100644 index 0000000..e2236f1 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixControllerV2.java @@ -0,0 +1,78 @@ +package eu.dnetlib.scholexplorer.api.controller; + +import eu.dnetlib.dhp.schema.sx.api.model.v2.PageResultType; +import eu.dnetlib.dhp.schema.sx.api.model.v2.ScholixType; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import eu.dnetlib.scholexplorer.api.ScholixException; +import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager; +import io.micrometer.core.annotation.Timed; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/v2") +@Tag(name = "Links : Operation related to the Scholix Links") +public class ScholixControllerV2 { + + @Autowired + ScholixIndexManager repository; + + + @Timed(value = "scholix.v2.links", description = "Time taken to return links on Version 2.0 of Scholix") + @Operation(summary = "Get Scholix Links") + @GetMapping("/Links") + public PageResultType links( + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships collected from a LinkProvider") final String linkProvider, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target pid") final String targetPid, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target pid type") final String targetPidType, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target published in a Publisher named targetPublisher") final String targetPublisher, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a target type (literature, dataset, unknown)") final String targetType, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source pid") final String sourcePid, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source pid type") final String sourcePidType, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source published in a Publisher named sourcePublisher") final String sourcePublisher, + @RequestParam(required = false) + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships having a source type (literature, dataset, unknown)") final String sourceType, + // @Parameter(in = ParameterIn.QUERY, + // description = "Filter scholix Links having collected after this date") String harvestedAfter, + @Parameter(in = ParameterIn.QUERY, description = "select page of result") final Integer page) throws Exception { + + if (StringUtils.isEmpty(sourcePid) && StringUtils.isEmpty(targetPid) && StringUtils.isEmpty(sourcePublisher) && StringUtils.isEmpty(targetPublisher)&&StringUtils.isEmpty(sourceType) + && StringUtils.isEmpty(linkProvider)) { + throw new ScholixException( + "The method requires one of the following parameters: sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider, sourceType"); + } + + try { + final int currentPage = page != null ? page : 0; + final Pair> scholixResult = repository + .linksFromPid(linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, null,currentPage); + final PageResultType pageResult = new PageResultType(); + pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10); + pageResult.setTotalLinks(scholixResult.getLeft().intValue()); + pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList())); + return pageResult; + } catch (final Throwable e) { + throw new ScholixException("Error on requesting url ", e); + } + } +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixLinkControllerV1.java b/src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixLinkControllerV1.java new file mode 100644 index 0000000..9d7bffd --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/controller/ScholixLinkControllerV1.java @@ -0,0 +1,81 @@ +package eu.dnetlib.scholexplorer.api.controller; + + +import eu.dnetlib.dhp.schema.sx.api.model.v1.ScholixV1; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import eu.dnetlib.scholexplorer.api.ScholixException; +import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager; +import io.micrometer.core.annotation.Timed; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.NotNull; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/v1") +public class ScholixLinkControllerV1 { + + @Autowired + ScholixIndexManager repository; + + @Operation(summary = "Get all Scholix relation collected from a publisher", description = "return a list of scholix object published from a specific publisher") + @GetMapping("/linksFromPublisher") + @Timed(value = "scholix.v1.linksFromPublisher", description = "Time taken to return links on Version 1.0 of Scholix collected from a publisher") + public List linksFromPublisher( + + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships collected from a publisher", required = true) final String publisher, + @Parameter(in = ParameterIn.QUERY, description = "The page number") @RequestParam(required = false) final Integer page) throws ScholixException { + + final int currentPage = page != null ? page : 0; + + final Pair> scholixResult = repository.linksFromPid(null, null, null, publisher, null, null, null, null, null,null, currentPage); + final List scholixData = scholixResult.getValue(); + if (scholixData == null) { return null; } + return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList()); + } + + @Operation(summary = "Get all Scholix relation collected from a datasource", description = "return a list of scholix object collected from a specific datasource") + @GetMapping("/linksFromDatasource") + @Timed(value = "scholix.v1.linksFromDatasource", description = "Time taken to return links on Version 1.0 of Scholix collected from a LinkProvider") + public List linksFromDatasource( + @Parameter(in = ParameterIn.QUERY, description = "Filter Scholix relationships collected from a LinkProvider") @NotNull final String datasource, + @Parameter(in = ParameterIn.QUERY, description = "The page number") @RequestParam(required = false) final Integer page) throws ScholixException { + + final int currentPage = page != null ? page : 0; + final Pair> scholixResult = repository.linksFromPid(datasource, null, null, null, null, null, null, null, null, null,currentPage); + final List scholixData = scholixResult.getValue(); + if (scholixData == null) { return null; } + return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList()); + } + + + + @Operation(summary = "Retrieve all scholix links from a persistent identifier", description = "The linksFromPid endpoint returns a list of scholix object related from a specific persistent identifier") + @GetMapping("/linksFromPid") + @Timed(value = "scholix.v1.linksFromPid", description = "Time taken to return links on Version 1.0 of Scholix related from a specific persistent identifier") + public List linksFromPid( + @Parameter(in = ParameterIn.QUERY, description = "persistent Identifier") @NotNull final String pid, + @Parameter(in = ParameterIn.QUERY, description = "Persistent Identifier Type") @RequestParam(required = false) final String pidType, + @Parameter(in = ParameterIn.QUERY, description = "typology target filter should be publication, dataset or unknown") @RequestParam(required = false) final String typologyTarget, + @Parameter(in = ParameterIn.QUERY, description = "a datasource provenance filter of the target relation") @RequestParam(required = false) final String datasourceTarget, + @Parameter(in = ParameterIn.QUERY, description = "The page number") @RequestParam(required = false) final Integer page) throws ScholixException { + + final int currentPage = page != null ? page : 0; + final Pair> scholixResult = + repository.linksFromPid(datasourceTarget, null, null, null, typologyTarget, pid, pidType, null, null, null, currentPage); + final List scholixData = scholixResult.getValue(); + if (scholixData == null) { return null; } + return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList()); + } + + +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchClientFactory.java b/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchClientFactory.java new file mode 100644 index 0000000..ceb9fff --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchClientFactory.java @@ -0,0 +1,74 @@ +package eu.dnetlib.scholexplorer.api.index; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.client.RestClients; +import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; + +/** + * The type Elastic search client factory. + */ +public class ElasticSearchClientFactory implements PooledObjectFactory> { + + private final ElasticSearchProperties elasticSearchProperties; + + + /** + * Instantiates a new Elastic search client factory. + * + * @param elasticSearchProperties the elastic search properties + */ + public ElasticSearchClientFactory(final ElasticSearchProperties elasticSearchProperties){ + this.elasticSearchProperties = elasticSearchProperties; + + } + + public PooledObject> makeObject() throws Exception { + + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(elasticSearchProperties.getClusterNodes().split(",")) + .withConnectTimeout(elasticSearchProperties.getConnectionTimeout()) + .withSocketTimeout(elasticSearchProperties.getSocketTimeout()) + .build(); + RestHighLevelClient cc = RestClients.create(clientConfiguration).rest(); + + return new DefaultPooledObject<>(new ImmutablePair<>(cc, new ElasticsearchRestTemplate(cc))); + } + + public void destroyObject(PooledObject> pooledObject) throws Exception { + RestHighLevelClient client = pooledObject.getObject().getLeft(); + if(client!=null&&client.ping(RequestOptions.DEFAULT)){ + try { + client.close(); + }catch (Exception e){ + //ignore + } + } + } + + public boolean validateObject(PooledObject> pooledObject) { + RestHighLevelClient client = pooledObject.getObject().getLeft(); + try { + return client.ping(RequestOptions.DEFAULT); + }catch(Exception e){ + return false; + } + } + + public void activateObject(PooledObject> pooledObject) throws Exception { + RestHighLevelClient client = pooledObject.getObject().getLeft(); + boolean response = client.ping(RequestOptions.DEFAULT); + } + + public void passivateObject(PooledObject> pooledObject) throws Exception { + //nothing + } + + +} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchPool.java b/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchPool.java new file mode 100644 index 0000000..9fccd3e --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchPool.java @@ -0,0 +1,32 @@ +package eu.dnetlib.scholexplorer.api.index; + +import org.apache.commons.lang3.tuple.Pair; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; + +/** + * The type Elastic search pool. + */ +public class ElasticSearchPool extends Pool> { + + private final ElasticSearchProperties elasticSearchProperties; + + /** + * Instantiates a new Elastic search pool. + * + * @param elasticSearchProperties the elastic search properties + */ + public ElasticSearchPool(ElasticSearchProperties elasticSearchProperties){ + super(elasticSearchProperties, new ElasticSearchClientFactory(elasticSearchProperties)); + this.elasticSearchProperties = elasticSearchProperties; + } + + /** + * Gets elastic search properties. + * + * @return the elastic search properties + */ + public ElasticSearchProperties getElasticSearchProperties() { + return elasticSearchProperties; + } +} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchProperties.java b/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchProperties.java new file mode 100644 index 0000000..9a9beec --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/index/ElasticSearchProperties.java @@ -0,0 +1,114 @@ +package eu.dnetlib.scholexplorer.api.index; + +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import javax.validation.constraints.NotNull; + +/** + * The type Elastic search properties. + */ +@Component("elasticSearchProperties") +@ConfigurationProperties(prefix = "scholix.elastic") +public class ElasticSearchProperties extends GenericObjectPoolConfig { + + @NotNull + private String clusterNodes; + @NotNull + private String indexName; + @NotNull + private String indexResourceName; + @NotNull + private long connectionTimeout; + @NotNull + private long socketTimeout; + + /** + * Gets cluster nodes. + * + * @return the cluster nodes + */ + public String getClusterNodes() { + return clusterNodes; + } + + /** + * Sets cluster nodes. + * + * @param clusterNodes the cluster nodes + * @return the cluster nodes + */ + public ElasticSearchProperties setClusterNodes(String clusterNodes) { + this.clusterNodes = clusterNodes; + return this; + } + + /** + * Gets index name. + * + * @return the index name + */ + public String getIndexName() { + return indexName; + } + + /** + * Sets index name. + * + * @param indexName the index name + * @return the index name + */ + public ElasticSearchProperties setIndexName(String indexName) { + this.indexName = indexName; + return this; + } + + public String getIndexResourceName() { + return indexResourceName; + } + + public void setIndexResourceName(String indexResourceName) { + this.indexResourceName = indexResourceName; + } + + /** + * Gets connection timeout. + * + * @return the connection timeout + */ + public long getConnectionTimeout() { + return connectionTimeout; + } + + /** + * Sets connection timeout. + * + * @param connectionTimeout the connection timeout + * @return the connection timeout + */ + public ElasticSearchProperties setConnectionTimeout(long connectionTimeout) { + this.connectionTimeout = connectionTimeout; + return this; + } + + /** + * Gets socket timeout. + * + * @return the socket timeout + */ + public long getSocketTimeout() { + return socketTimeout; + } + + /** + * Sets socket timeout. + * + * @param socketTimeout the socket timeout + * @return the socket timeout + */ + public ElasticSearchProperties setSocketTimeout(long socketTimeout) { + this.socketTimeout = socketTimeout; + return this; + } +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/index/Pool.java b/src/main/java/eu/dnetlib/scholexplorer/api/index/Pool.java new file mode 100644 index 0000000..78f1c82 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/index/Pool.java @@ -0,0 +1,232 @@ +package eu.dnetlib.scholexplorer.api.index; + + +import eu.dnetlib.scholexplorer.api.ScholixException; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; + +/** + * When using the Java High Level REST Client provided by the Elasticsearch official website, it is found that there is no in the client API. + * Connecting to connect the pool, create a new connection every time, this is impact in high concurrency situation, so it is ready to be on the client + * API increases the concept of pool. + * + * Fortunately, we don't need to turn your weight to write the implementation of the connection pool, because Apache provides us with the general framework of the connection pool. + * Commons-pool2, and we only need to implement some logic according to the frame design. Used in the REDIS client API + * Jedispool is based on Commons-pool2 implementation. + * + * Let's take a look at how to achieve it. + * + * First we have to create a pool class, this pool introduces GenericObjectPool in Commons-pool2 through dependent manner. In this class + * In, we define how to borrow objects and returns objects from the pool. + * + * @param the type parameter + */ +public class Pool implements Cloneable { + + /** + * The Internal pool. + */ + protected GenericObjectPool internalPool ; + + /** + * Instantiates a new Pool. + */ + public Pool(){ + super(); + } + + /** + * Instantiates a new Pool. + * + * @param poolConfig the pool config + * @param factory the factory + */ + public Pool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory factory){ + initPool(poolConfig, factory); + } + + /** + * Init pool. + * + * @param poolConfig the pool config + * @param factory the factory + */ + public void initPool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory factory) { + + if (this.internalPool != null) { + try { + closeInternalPool(); + } catch (Exception e) { + } + } + + this.internalPool = new GenericObjectPool(factory, poolConfig); + } + + /** + * Close internal pool. + * + * @throws ScholixException the scholix exception + */ + protected void closeInternalPool() throws ScholixException { + try { + internalPool.close(); + } catch (Exception e) { + throw new ScholixException("Could not destroy the pool", e); + } + } + + /** + * Gets resource. + * + * @return the resource + * @throws ScholixException the scholix exception + */ + public T getResource() throws ScholixException { + try { + return internalPool.borrowObject(); + } catch (Exception e) { + throw new ScholixException("Could not get a resource from the pool", e); + } + } + + + /** + * Return resource. + * + * @param resource the resource + * @throws ScholixException the scholix exception + */ + public void returnResource(final T resource) throws ScholixException { + if (resource != null) { + returnResourceObject(resource); + } + } + + private void returnResourceObject(final T resource) throws ScholixException { + if (resource == null) { + return; + } + try { + internalPool.returnObject(resource); + } catch (Exception e) { + throw new ScholixException("Could not return the resource to the pool", e); + } + } + + /** + * Return broken resource. + * + * @param resource the resource + * @throws ScholixException the scholix exception + */ + public void returnBrokenResource(final T resource) throws ScholixException { + if (resource != null) { + returnBrokenResourceObject(resource); + } + } + + private void returnBrokenResourceObject(T resource) throws ScholixException { + try { + internalPool.invalidateObject(resource); + } catch (Exception e) { + throw new ScholixException("Could not return the resource to the pool", e); + } + } + + /** + * Destroy. + * + * @throws ScholixException the scholix exception + */ + public void destroy() throws ScholixException { + closeInternalPool(); + } + + + /** + * Gets num active. + * + * @return the num active + */ + public int getNumActive() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getNumActive(); + } + + /** + * Gets num idle. + * + * @return the num idle + */ + public int getNumIdle() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getNumIdle(); + } + + /** + * Gets num waiters. + * + * @return the num waiters + */ + public int getNumWaiters() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getNumWaiters(); + } + + /** + * Gets mean borrow wait time millis. + * + * @return the mean borrow wait time millis + */ + public long getMeanBorrowWaitTimeMillis() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getMeanBorrowWaitTimeMillis(); + } + + /** + * Gets max borrow wait time millis. + * + * @return the max borrow wait time millis + */ + public long getMaxBorrowWaitTimeMillis() { + if (poolInactive()) { + return -1; + } + + return this.internalPool.getMaxBorrowWaitTimeMillis(); + } + + private boolean poolInactive() { + return this.internalPool == null || this.internalPool.isClosed(); + } + + /** + * Add objects. + * + * @param count the count + * @throws Exception the exception + */ + public void addObjects(int count) throws Exception { + try { + for (int i = 0; i < count; i++) { + this.internalPool.addObject(); + } + } catch (Exception e) { + throw new Exception("Error trying to add idle objects", e); + } + } +} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/index/ScholixIndexManager.java b/src/main/java/eu/dnetlib/scholexplorer/api/index/ScholixIndexManager.java new file mode 100644 index 0000000..2882444 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/index/ScholixIndexManager.java @@ -0,0 +1,401 @@ +package eu.dnetlib.scholexplorer.api.index; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixEntityId; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixRelationship; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixResource; +import eu.dnetlib.dhp.schema.sx.scholix.flat.ScholixFlat; +import eu.dnetlib.scholexplorer.api.ScholixException; +import eu.dnetlib.scholexplorer.api.TaggedCounter; +import eu.dnetlib.scholexplorer.api.model.Summary; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; +import org.springframework.data.elasticsearch.core.SearchHit; +import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.stereotype.Component; +import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; +import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; +import org.elasticsearch.index.query.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component +public class ScholixIndexManager { + @Autowired + ElasticSearchProperties elasticSearchProperties; + + /** + * The Elasticsearch template. + */ + @Autowired + ElasticSearchPool connectionPool; + + final ObjectMapper mapper = new ObjectMapper(); + + /** + * The enum Pid type prefix. + */ + public enum RelationPrefix { + /** + * Source pid type prefix. + */ + source, + /** + * Target pid type prefix. + */ + target + } + + @Autowired + TaggedCounter myCounter; + + + private List extractIdentifiersFromScholix(SearchHits scholix) { + return scholix.stream() + .flatMap(s -> + Stream.of( + s.getContent().getSourceId(), + s.getContent().getTargetId())) + .distinct() + .toList(); + } + + + private Map retrieveResources(ElasticsearchRestTemplate client, List ids) { + final IdsQueryBuilder qb = new IdsQueryBuilder().addIds(ids.toArray(String[]::new)); + + + final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() + .withQuery(qb) + .withPageable(PageRequest.of(0, ids.size())) + .build(); + + + SearchHits result = client.search(searchQuery, Summary.class, IndexCoordinates.of(elasticSearchProperties.getIndexResourceName())); + return result.stream().map(r -> { + try { + return mapper.readValue(r.getContent().getBody(), ScholixResource.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toMap( + ScholixResource::getDnetIdentifier, + v -> v, + (a, b) -> a + )); + + } + + private Scholix generateScholix(ScholixFlat flat, ScholixResource source, ScholixResource target) throws ScholixException { + if (flat == null || source == null || target == null) + throw new ScholixException("Error generating scholix null input"); + + final Scholix scholix = new Scholix(); + scholix.setSource(source); + scholix.setTarget(target); + scholix.setIdentifier(flat.getIdentifier()); + final ScholixRelationship r = new ScholixRelationship(); + r.setSchema("datacite"); + r.setName(flat.getRelationType().toLowerCase()); + scholix.setRelationship(r); + scholix.setPublicationDate(flat.getPublicationDate()); + scholix.setLinkprovider(flat.getLinkProviders().stream().map(p -> { + final ScholixEntityId eid = new ScholixEntityId(); + eid.setName(p); + return eid; + }).toList()); + + final Map publishers = new HashMap<>(); + if (source.getPublisher() != null) + source.getPublisher().forEach(p -> publishers.put(p.getName(), p)); + if (target.getPublisher() != null) + target.getPublisher().forEach(p -> publishers.put(p.getName(), p)); + + scholix.setPublisher(publishers.values().stream().toList()); + return scholix; + } + + private QueryBuilder createFinalQuery(final List queries) throws ScholixException { + + if (queries == null || queries.isEmpty()) + throw new ScholixException("the list of queries must be not empty"); + + + if (queries.size() == 1) { + return queries.get(0); + } else { + final BoolQueryBuilder b = new BoolQueryBuilder(); + b.must().addAll(queries); + + return b; + } + + } + + public List> totalLinksByProvider(final String filterName) throws ScholixException { + + + final QueryBuilder query = StringUtils.isNoneBlank(filterName) ? QueryBuilders.termQuery("linkProviders", filterName) : QueryBuilders.matchAllQuery(); + + final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() + .withQuery(query) + .withSearchType(SearchType.DEFAULT) + .withPageable(PageRequest.of(0, 10)) + .addAggregation( + AggregationBuilders.terms("genres").field("linkProviders").size(100) + .minDocCount(1)) + .build(); + + + Pair resource = null; + try { + resource = connectionPool.getResource(); + ElasticsearchRestTemplate client = resource.getValue(); + final SearchHits hits = client.search(searchQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + + final Aggregations aggregations = hits.getAggregations(); + if (aggregations == null) + return null; + + return ((ParsedStringTerms) aggregations.get("genres")).getBuckets().stream().map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount())).collect(Collectors.toList()); + } catch (ScholixException e) { + throw e; + } finally { + if (connectionPool != null) { + connectionPool.returnResource(resource); + } + } + + + } + + + private QueryBuilder createLinkPublisherQuery(final RelationPrefix prefix, final String publisher) throws ScholixException { + if (prefix == null) { + throw new ScholixException("prefix cannot be null"); + } + return new NestedQueryBuilder(String.format("%s.publisher", prefix), new TermQueryBuilder(String.format("%s.publisher.name", prefix), publisher), ScoreMode.None); + } + + + public List> totalLinksPublisher(final RelationPrefix prefix, final String filterName) throws ScholixException { + + + final QueryBuilder query = StringUtils.isNoneBlank(filterName) ? createLinkPublisherQuery(prefix, filterName) : QueryBuilders.matchAllQuery(); + + final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() + .withQuery(query) + .withSearchType(SearchType.DEFAULT) + .withPageable(PageRequest.of(0, 10)) + .addAggregation( + AggregationBuilders.terms("publishers").field(String.format("%sPublisher", prefix.toString())).size(100) + .minDocCount(1)) + .build(); + + + Pair resource = null; + try { + resource = connectionPool.getResource(); + ElasticsearchRestTemplate client = resource.getValue(); + final SearchHits hits = client.search(searchQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + + final Aggregations aggregations = hits.getAggregations(); + if (aggregations == null) + return null; + + return ((ParsedStringTerms) aggregations.get("publishers")).getBuckets().stream().map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount())).collect(Collectors.toList()); + } catch (ScholixException e) { + throw e; + } finally { + if (connectionPool != null) { + connectionPool.returnResource(resource); + } + } + } + + private void incrementPidCounter(RelationPrefix prefix, String value) { + switch (value.toLowerCase()) { + case "doi": { + myCounter.increment(String.format("%s_doi", prefix)); + break; + } + case "pmc": { + myCounter.increment(String.format("%s_pmc", prefix)); + break; + } + default: + myCounter.increment(String.format("%s_other", prefix)); + } + } + + + /** + * Links from pid pair. + * + * @param linkProvider the link provider + * @param targetPid the target pid + * @param targetPidType the target pid type + * @param targetPublisher the target publisher + * @param targetType the target type + * @param sourcePid the source pid + * @param sourcePidType the source pid type + * @param sourcePublisher the source publisher + * @param sourceType the source type + * @param page the page + * @return the pair + * @throws ScholixException the scholix exception + */ + public Pair> linksFromPid(final String linkProvider, + final String targetPid, final String targetPidType, final String targetPublisher, + final String targetType, final String sourcePid, final String sourcePidType, + final String sourcePublisher, + final String sourceType, + final String relation, + final Integer page) throws ScholixException { + + + if (sourcePid == null && sourcePidType == null && sourceType == null && targetType == null && targetPid == null && targetPidType == null && sourcePublisher == null && targetPublisher == null && linkProvider == null) + throw new ScholixException("One of sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider should be not null"); + + final List queries = new ArrayList<>(); + + if (StringUtils.isNoneBlank(linkProvider)) { + myCounter.increment("linkProvider"); + queries.add(QueryBuilders.termQuery("linkProviders", linkProvider)); + } + + if (StringUtils.isNoneBlank(targetPid)) { + myCounter.increment("targetPid"); + queries.add(QueryBuilders.termQuery("targetPid", targetPid)); + } + if (StringUtils.isNoneBlank(sourcePid)) { + myCounter.increment("sourcePid"); + queries.add(QueryBuilders.termQuery("sourcePid", sourcePid)); + } + + if (StringUtils.isNoneBlank(targetPidType)) { + assert targetPidType != null; + incrementPidCounter(RelationPrefix.target, targetPidType); + queries.add(QueryBuilders.termQuery("targetPidType", targetPidType)); + } + if (StringUtils.isNoneBlank(sourcePidType)) { + assert sourcePidType != null; + incrementPidCounter(RelationPrefix.source, sourcePidType); + queries.add(QueryBuilders.termQuery("sourcePidType", sourcePidType)); + } + + if (StringUtils.isNoneBlank(targetType)) { + myCounter.increment(String.format("targetType_%s", targetType)); + queries.add(QueryBuilders.termQuery("targetType", targetType)); + } + + if (StringUtils.isNoneBlank(sourceType)) { + assert sourceType != null; + myCounter.increment(String.format("sourceType_%s", sourceType)); + queries.add(QueryBuilders.termQuery("sourceType", sourceType)); + } + + if (StringUtils.isNoneBlank(targetPublisher)) { + myCounter.increment("targetPublisher"); + queries.add(QueryBuilders.termQuery("targetPublisher", targetPublisher)); + } + + if (StringUtils.isNoneBlank(relation)) { + myCounter.increment("targetPublisher"); + queries.add(QueryBuilders.termQuery("relationType", relation)); + } + QueryBuilder result = createFinalQuery(queries); + + NativeSearchQuery finalQuery = new NativeSearchQueryBuilder() + .withQuery(result) + .withPageable(PageRequest.of(page, 100)) + .build(); + + + Pair resource = null; + try { + resource = connectionPool.getResource(); + ElasticsearchRestTemplate client = resource.getValue(); + + + long tt = client.count(finalQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + + SearchHits scholixRes = client.search(finalQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + + + if (tt > 0) { + final Map idMap = retrieveResources(client, extractIdentifiersFromScholix(scholixRes)); + + return new ImmutablePair<>(tt, scholixRes.stream().map(SearchHit::getContent).map(s -> { + try { + return generateScholix(s, idMap.get(s.getSourceId()), idMap.get(s.getTargetId())); + } catch (ScholixException e) { + throw new RuntimeException(e); + } + }).toList()); + } else return new ImmutablePair<>(tt, new ArrayList<>()); + } catch (ScholixException e) { + throw e; + } finally { + if (connectionPool != null) { + connectionPool.returnResource(resource); + } + + } + } + + public List findPage(final String relType) throws ScholixException { + Pair resource = null; + try { + resource = connectionPool.getResource(); + ElasticsearchRestTemplate client = resource.getValue(); + final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() + + .withQuery(QueryBuilders.termQuery("relationType", relType)) + + .withSearchType(SearchType.DEFAULT) + .withPageable(PageRequest.of(0, 20)) + .build(); + + SearchHits search = client.search(searchQuery, ScholixFlat.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + + + final Map idMap = retrieveResources(client, extractIdentifiersFromScholix(search)); + + return search.stream().map(SearchHit::getContent).map(s -> { + try { + return generateScholix(s, idMap.get(s.getSourceId()), idMap.get(s.getTargetId())); + } catch (ScholixException e) { + throw new RuntimeException(e); + } + }).toList(); + } catch (Throwable e) { + System.out.println(e.getMessage()); + } finally { + if (connectionPool != null && resource != null) { + connectionPool.returnResource(resource); + } + } + return null; + } + +} diff --git a/src/main/java/eu/dnetlib/scholexplorer/api/model/Summary.java b/src/main/java/eu/dnetlib/scholexplorer/api/model/Summary.java new file mode 100644 index 0000000..5472f83 --- /dev/null +++ b/src/main/java/eu/dnetlib/scholexplorer/api/model/Summary.java @@ -0,0 +1,13 @@ +package eu.dnetlib.scholexplorer.api.model; + +public class Summary { + private String body; + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..9b7468a --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,43 @@ +spring.application.name=scholexplorer-api +spring.main.banner-mode = console +springdoc.api-docs.path=/api-docs +springdoc.swagger-ui.path=/scholexplorer-api +server.public_url = +server.public_desc = API Base URL + +logging.level.root = INFO +dhp.swagger.api.host = localhost:8080 +#dhp.swagger.api.host = api.scholexplorer.openaire.eu +dhp.swagger.api.basePath = / + + +management.endpoints.web.exposure.include = health, metrics, prometheus +management.metrics.tags.application=${spring.application.name} +#maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml + +# +#spring.thymeleaf.cache=false +# +#spring.metrics.export.prometheus.enabled = true +#spring.metrics.export.prometheus.port = 8080 +#management.endpoints.web.exposure.include = phealth, metrics, prometheus +#management.endpoints.web.base-path = / +#management.endpoints.web.path-mapping.prometheus = metrics +#management.endpoints.web.path-mapping.health = health +#management.endpoint.health.show-details = always +# +#management.metrics.distribution.percentiles-histogram.http.server.requests=false +#management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms +#management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999 + + + +#scholix.elastic.clusterNodes = 10.19.65.51:9200,10.19.65.52:9200,10.19.65.53:9200,10.19.65.54:9200 +scholix.elastic.clusterNodes = localhost:9200 +scholix.elastic.indexName = scholix +scholix.elastic.indexResourceName = summary +scholix.elastic.socketTimeout = 60000 +scholix.elastic.connectionTimeout= 60000 + + + diff --git a/src/main/resources/static/logo.png b/src/main/resources/static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..52911830cf0638a60567a27f1c66b4a8b4766d2b GIT binary patch literal 13093 zcmcJ0S6oxi6R%1K>C&b5E+8OC3%&O$gdqG7G4$S%ARPjThzLSxf}o)Z(ouRRfEbV- zS}0Nj3H?U@pZj)S??ZA<_RP+FcXrO~&dwys)L4g-oR$36ty`3Ox|-&QXvRL zh$l!ZlLrRByxa_Y)|S;?|Jc>vT@RKPV`8aaP!^F7=1gU9vU$%_pfL zj}9GC7ClS!OMD$AI1-dmj|pAQ)^8A{@5W?@Ev}mRrgx`mQ+l+q?c$S*ulWPrq!XRl+xtG7A5nSW#&D;HZbi?zV3ighmlg+ zqj37Dtb`jMY2PaAv_j7w)4EFqamAU|}@B{I70K4}%Jm+QpX@oeY=eSLG zqW7a%mCX!LIOnOOi-+poQC`n6RCl61q3OT4g_ERl?cdktyOH=&-X$(!)Kw}>qj9Kq zdJr(?(F5kyY7fx3M>lqsbOWVvAdK~m&nxra;8wZ+^aiBM7DoL~0To1aqprNKPWOAv zZil80tjkpuo$We2%hL2uA+3!R(Dc}vKTGUP9dHX@Qu^0H_YU1C5Z~c@V#KeMfz3AQ za2~Iq@B=FEH_ti~BaBH$6aG!f6Rrs*5!3%bf@4g%NNR%Pmi7_Gq>Zx}o{JRbLg9t` zJvIc#=MjdE|8%Swt9g8>mU(5I)+S+;&6Kq9A%-X1_w&-5)6DN}d#R%-|5yrTJYQrM zssmMLBBf$NuRi}}xPT|r|GdN#LGo?Cf1~%W9<=Z2K9r=sH=8Sa^q%n@l5^N2^Ldn@ z>3`*Y=SJ?{WFtF^Gj0DylBh00(Df>Dw;H?_XyD|y|7MnMrG9)r`6O>rh5a3}p*B?N zFC8sjL){u$a@4kU&(>t3B?N!QZOHaLlQeY#7<%OK{`A(b*Vc4&i{I$}CtW_1aESTKLqs6Es^3$8UWGHTfzKuxie?|%G$k)(1RqZ(;IGW+hky>&gj0~IL zAlp#Mgj|#Rl7Tw4E{(l@DE~w1^piMgLxWcR;+Z4u;AoBVTU?H=6iKwzSK%(V4|$io zzREdE0b_eS5?^;MYOyi@cLMUCX!z`NQ_<-vHZmW2I6HPXZm@pzq#JF0L8^%3oEEZe zM9Z_=$AtJ(&bd7dt^OxUh=^>%T(+f079^me*H(!0;o(N+^B=J$Z4k>6EvywLs`4$G zMDWD4TeY;L|C>BDyat56e>$T^b$#sQ@v3_=)mB?|(eLzNu@^SvwhoVEu7uU)r8Oz} z?!hDVhyuXamp(R!no}T=3o9#RC=zcztJK?FoN(Ahtu0dJ0rS2hX|p#bz9%-xhLk1e zoYIPHbh;tBor)XVdrj&ajpFawrv+K8%eR1U%poFMQ(Xj79Yi2&qcfBpS!>Q zVQ>~tUa_0QYs^)k)|QJ~b)fnrhG_6keM%Kp`-}V34hMHvCJ1OZpaE4>zG^wG&57xw zI^68}COULL0KCX$4aGs9e>n`l@H+ONvI$Ht*u5=v^NDoEEmG*^P`D#1ILX^`KI^RY zG)pQ~O>)AyMFmgkS=9di@A027 z{V;cw0Wc+)#rrWjQNt*DizAhBWiIdI8YsyiII#2mo$QkfrM-R#@M4ge7{-K-?Wv+` z&+w+L=7;_{Z!w1uwR0?ffl!%hwM!fp0#m-Fbr!S@by7dR?$9!f-cs?HD}s6doQ`}! zCx%Y#=X|7?ObE{Ie-&Cnr#R zO#oDgSq$!YwElfHxnnYt#Uw&uH3Es91DkG`w&4ssujFM~GL!cgP;G6`D86o45 z=G6|#ZXkSAr$y(2tl9I$;bME`c1nuXm5gn!97Mv?3aO1MGY;T(~!-i*kFFp9|$~ z`|?{Nwul(dNSaiEc!RfYb9%&09JQI)>A&XxtP%{S_y=|It}bB^@8a~E3e|=j-0&X~ z_PQrXq{7c1_^}dm^L93iOqmFb9rwJBC@{!1vB%p62_xni!1=*8XBJ~eJ(*dw3zBFl zG-T;Zhi3FR@+6STss<+WT5Vud(}7{@gmOKNG=Q4%==?c-s4V3_*xn>a&Z8}jsIw&5 z_pEL@q;sW|H^CIP2o^oH>@`LX2KN%$_XF2NB8B6!xh7fv6L|V^^PuBOKxNiaUg?mj zd0yBf0Etwz{Ei2Chg;tMQ>7-zYcSm?NuzNXiZ^~+#EwSx%G;j$#B}mz%(0MJ-QC@d z*mMDi*F(WNwPT!i`Tc~AVmR3CelXg){1c}I-ejM-6Q#+JBd=L>E5DiK>Qql&A_+Z{kzLe(-d@fiBGH_I4L|udizMv2U%P6Eopayr<{#G!t{J_Ji}f5J~zj%u#@%rQR+CR z+fRC^wY|gHQ-cZxqvD(NDLL7g0{Brh2)*gB5TG@(*E<(y-w%E^d| zc$4Q}l~z`A2KQ;ekq&0(-ik*v)jle$rSJy`>=pLEux69v3POSD=9h9suatTvOgq0? zMKGL6u<@WVH)*f#ucjVS=FNy@*NkYBv8Z->x~B@c0OiN!{1WNQPT=Np*i7)}+2P2~&u**CV>|Dn zo^FC2JPiLS{ZOS#s z<)z6W;fJxjJe+1X@-!RaSA7utf&yEd?3R0-A2xE5aut-GeMH;~qe>?gUz!(38`X_Z z^_N17H>1kpz^$U+3)hu)0f-BNNT0EKnEwXv_2-qk8ZT>0)VMO0{RMrFVEl;t+3Afm zjOqM!qW&#l)w{3$uQNbQC6SUs{6b5uPd?*nF8dX=#gqhk$$!Kj;I|k@x>I8F=>vFlcFhsZ(O9cN)Jm@kG+Vl(tEIBxOaw#tu~4$ zZ5!F^n-NySJ8f;*P8_j)i_Bo?n zdg`7e++2fk!togizfgB5NwX+y66uwuW? z9JwlcuQU@>xgpmwC>IoQB|(GG$a~N@Wx3AFb8gB{!$P&raf)9xl4N1i*~_2pte`ZT1HeH{@yPb%hMR(ATF$)mAT(&PI}J3 zA?HP3Kzrd+kO7Evki#C;+SEj980?Ty(;atCI@u>?j|Pmk{sXBSalUS1aSn33jWxLazVdm-iNb`Jjwyh!`mdNYUE$z zDiTJA6rMGQ-(6T6Y#@(uL~n^(bG&;VWcVr_9Qr#r*&*r5|1KNo$Z(YxZTtFO8%A?L z!YSv^9J}1P(!S2Y5!XuR-Rc10^a^1#W-ggytG0e=*Hun`cT8aIfqC`CLG-ZHOUi>N5_p z&pVgLxekn9J%(PBpj=eNlyc(8Q~@BF$sFpCnW(;}tH#u9KX&>lFW`x>%fbv>JLKoh z(65p8R1b~Lg%Dvs3y6iOK9z+4<63`gpVcE`0MX1bBp z)-x5NXt=KP@)|2Lq%PrVauG3KJ}s zbGFOw=VT9^rd--)T@+$(99e)|o6q{2v#H0MD}!)Ga{50WvtQpwKjoeM$lXCRN^!nN zby00oy0Dme3hHt(|B#TbVo>wBUQLSvNa*pYp%#*nC0He=E&OGKJm@q%M6g5P zJ{!rW&D93N{Ur&lxdP`^4Uec=Hwg{zv8>^SMtVP%(Gj=&#?*f6Y@1_@wNvk{a3nIV>g;aA}q7yyd`-x{l+oY!Y#b^HFG9{?3N- zvBFeldLXvFfH8pW)!qJNyXk%k1-q?@#G!R;PcubOQI#T)o4>tCgKFQ? zi!aGv;mj+;z2~lj!~8<3B6>DDi0EbA1SdrodW7t*8Ox8zC!2KCh9XAax zx{O}Wp9Lzdjkr+1EI(e#OVFVp_yTMn`e=aYq(_x>-z+Faus{9F325IYko)}O(fj3r zopS!I-}f-(_e1|=Qxm3t>SnutX3N{Zu3?1mr149`l69$?l7|@n=zG+Ern-M{9%hc% z!bw8|s{23;c-@&BZmT6hU`aqxBmV80|BKR_i?_3u47L>m zQIdT(4twGp?~xk}r@t@NL9^XR2Jx+8LggHp-Kt5aX1joPmqBkbo2Y+y-Vr~gS8D0$ zERsaCss~L(&(=di;Tb`o{xfVq2(VoQy`wnEAD(F$MG^j{8&>wRxBwO`zV<%n@z67m zmtS(g^RY7(>1A1Z{tA8UPm1g|4Vp48zI>AKOCS&TBtMVf=%9Lnde={JE)9JctsCem zqB_DA90y?k3RUv9L9{aZ8G#8@IgKH3?)l-GhpJAczP(;p@;g(A6HFznRkPbAb|sNqm$c*HpnSFkquk{uVO9uW7E0f(U*~*_wf>t#HT?v7klk^-%_iGh1PM(ck(~=Qm&d?)`=BL2CFrjf5=21w=zB4a=fn|8cckCxJIeVf^m2`qr(?t6+3Ht>35T74KqW)jJq7FTB-@ z&)87m>cgazG2bKfyE#Pv+8GDO^_Zugyye?3f7S+e&Rz$2lDCMeO_W-OYMmpYuqlIf z0G-D5ndhe2#IAS#zQvZ#=w~Azab;n&$QRL?k;U35yCXkOiWg}tAPdB}Z>j$tErM|% zM}{Dd5$i=}GN<0e=IyG~bhOMK;P+7=6lMkE@rc5Cb_2BG*eSO zGrp)GmU-c+XUBKCbh^zwxc<7jocHJ*x@OKAZL`~T(#Ts~Wia`R7V`-jkS@@o60qNi zSC0k1`yAfWOAan2IkrBRi)kdh0<026TiMFF>r!%KNrJ89N+{gNj&$kUDW9i-w_E{B zzq{pWck(jRP1NYYn$_5FBEpT z0Q#k8BaJ>O9AfnFqr>38Ve zzc=+ARc(Nncvn&1;}wK5QNC>Oawzur$$;rro;3O5n-&~8Be~$xMgR!z%_rd;6sNzJ zr#%JPYxNzH?U^j&zI~STa>&DU#KpB+-bdS#I`&;4Jydw_o`yOi=k$R3qL@T{i^hDU z`lJTf%whfWUV!T=3pE=c>nr)Gs(PE`MEJpBeBAE@=Eoq}pTU(rr+c@hNX9W3BtM!9 zQnpKL#%P-pSykTYBC`9FAwW|%ygQRane>N2qtNIyN5|C{{rhu_0oj`r98o$tOl4o~ z*Zj&`hr$M5!&w03IcKGb`pYj+thAv;g3Bql&qjP^j?Z`^bdu8c#a7IEcEsfJC0o$G z02?`#%@y7-a9{ZOt6DJf8uPv|U`C%CofmrUyzubH(+gc*<+lZ8G$#o>i=A(j#D4`> z25|hb9Vz6dA~&@}Ri(qgj@kC=Id5lrXHKxnf2_4@KZ~M=`aW^UA$)#p$d5H_1&k@t zD2?w_U50GyBEDfUS9#6+ExzQ}U2W~>7igJCvqw2^uy8*9Vz)oV^ ze6c?@YFx=v!G;h*=L@5@A$ z@k;fNgo5{hN!8pCDjpt=#=iR)*}UOL$802|AD*r2m=xg) z`kGFJ)83W@q@c>buZFPd=x7&Q77p!221#s2-UnP}@r!Q=h4iWovhDeGek~y{OD=Pf zB3jZwVj#A6FfUQOG42MzC3G5Uh-`KwG%$9Tj#FT<8b`mflrb4V+(@%U$j)>20u~)i z+N{CyER9oVSI|*-{#eO8@qC+h-7e=K+%#m&)U>zWaSyt)K-eCZaaUF2Zzd=SE2{FQ zwY9-BV61nLp0uTIRVK)T4!cj%C#%BZYjUAbI`t&10^2+#(&?m!r;$HB$G#Y|XLo8UKF zj_3EA-xbsAnddZOg4NiJdWU>=Zc(r(oGYgL{Scd{B&-=Y_ zj1agUZJdH_50FhiDr>jhQ-_9j{xa8kKFJbR*R?s(!^DQ6TsY2_%wkrDc+K4XFh|p6 zK@h1Yf*#uVtv+`0e4DjU(2Lc2JNp{Sw>bB*kd%b2fp3DT`M$*sK!agz|A>|C#aa}k z!^F8zk^&?32_~8+FDvw<2#Lhsazep1N@^n@ABpIZxy5{~0Z$G|?;4(mNx2=1^bR5s z1?jk1;>*@9nHz3F^w1sy<(pX*+9*Dz0N3C*87rw4j+}SU`OMoX<(;;~i9n)4qqQAs z&B6Z=0&3Y4fab?a6_U{=N_;UU5`bu{56GPITy${ItMox4;H)QV7-Jhe7JG?yKxA|7 zZ1|-r1?D>BV7p42L`=16lIaFjnBI)v?uzCIWF|^JrIh*o(64!Z%M_K8na$gQ zX-BDsoUihuw;eujb_h=HbJEHuV`^yuHjT8o;|&D4)SWtF!0EMvB0<81gUwV;btq{h zpgrINSqmAr{C8429WUNzl;ihkcV>96C_g;FN+0qRBNczDM`>6i(v+;}{X#(l859sg zcK&UXyI@O1TEr=%pR?_Ne@vMxdeFAr>pD|KvZRN-{;-g%LoK3gnYZPJcAB;xW3$s` z!dBt;n=1z##93AGy=HgIzS)89-sN5>ZkY~aBj#T7kP7YS41TI^ zxMyi6#<%v_!GrwJRjP(?_p!PJnT%*I(uzM;`90YCns!UF&upB=Z}e(4-aDPOgO>_t z48$R;U`Mu@O|f)!BzJktS>*<1#Xhb~iPHErGqduLzQKIylHGQ@Q>FUC>SwB5k;v(E zA}2o_VP3EZLi2C+ZxR^1Gj#SoD45~oy4O6t%7<|Xyt^OCgXwZ%^{|1*=Cgs9;+5{cfkPJa{i+*eD`OVOumP?T3SulsM*r75z zj)B55;~=vC<-_yE(dS*?a{dEZ02cLP`qX1-=FYc5iCeYcSe`nq*r3+|#d_n)s*lCzV z3s9(Wb)$<)WDU|Fyn1QakrK#C^Xa9wN<_?lmW*BDXu%+sn)9|!#gb38{GS0GHq-uM zN4BK~-qfNbibRl4{!}(>{NvG8z~KJ#n3Bo1v=A4@#Wc_EhpyGaJ)~g_W8?q1E~d;p z$U?SQBcX&`*{~AYc(5v0j5y~#B4q}3!uA|kXMlXwBNAm$e^oR~>eHAaF*NIAx2={> zwuq@1q7OvvqOh^1k6=OO)W#KF^qd<3L`BfoK|pW?wljBP)#fD7U+E+9Dme4jQUnqE zBb^WSseW;LebbiL^bGyBlP0E}z}NUAx92`yt4aZ7a3C&{thy>QkrqQMQh1W)^0?B4 zS)v)fJnj*PK}YqNLO9?;dH257=~GL+AFpvUPJHrYcCs^Vn0WlDE$Ax9P@wwc!Rn>! zv_eYNO-^9HH{2udduesr^wLfRX3lof>m8+1VH39Zr~+byIJ~>VEfnDDolEI3?i}ZD zW7#jo_X@1q|0wt~ziON6V^NTAyMH!rI)^f#8(z6<^dRaR*}2^8vb5%Jvu*GB%|BMx z%-Dkd*ZWPM9T10YXt>8Mtfy$%F*U7FPU~$I?Y+O1#~4s3 znSL_eti#rS;0OE!1$~U@(7n2mnf)x8dpf0k)djoLYRo?6>Jzu(V(03?qm9@!cZWWN zZjqVDy=UMTc}Yx#3(!(&FFuItU;5^vsOkAAH zw@Pr&aqx#a|Dqwbbt@0pOb24+!f_})Ep^pG=1k??BfGOS?O&;ezX;r139^(t{Xqu~ zdzq9!C1vnoyLq&?hn}0WP2i%Voo%c!0cC#G8_IdMH*5t8N^|~PF%InVL^P91_V;{=re2X8WRI_oRCs>S!kL0_(hBm7YRl`2?{ed#^ zkJJg-sS$O3z7d|Fk5QtHLh=K|nz|J197B;m4}r=N6toy3N=_qKoMMZQp-sz_SP+$I z=P3A$Qt~Kh_OpzOoBZVN){2`50QtSPPQuxevEmkjR3=>eAn?`D)+-O|LnM|3;(DR} zQkgV)mgYNhanrJepJLM{u(_XHvFCO+sP?tvlPl=Z&o?0b=HM-$J2b1!L|0;?nm*GYpeA!eX(SugwMbwfT!R`@ucSsxgPmj()TkdNtgBF zj#D5ic}MK7z_w~v2-N>j*8TLBM@2;QL$-Mt7~89X{s=GbpfH}#1_&nW#=uqfvsEx# zn$<(UlwuEOU^Z2nR{h>@(JZV-Vb|FcglgVwxU`S5joWE_ui5NpNvzPl}tdt=Q54utION+D0&qwhRwxm@?FJE3spE)!rVkmNL`}k znsg|Xxoz3a#Y`Sb9Hg>?oGq~Cxl#&WVL793LE`n)BwecW7W0CUsvZ!iWaQ5r3z^3w z7Y(25nu7G!oREA9 zbOD5jqTdCtjJWW!etf0FIeJ8*j#bwd0rCX7z!NT|P=W*!wjaDxxMD0ne zafy=lg~T@5uxg9f8|to0ns|2)NO5bx?OLrTY*xKEa(SN9N*VU7X_lhAA8|Ne0P~1~ z@kLtU8#&Dz``P+WhbwoW*)q`pd8u;!ltqPzQOD1_YV#wD48YT>>=`>AHnFBw*VqL9 zWVP;=M@+idd+#ZY@?iU@ry!T$K(E<%SKHfIN@WkJcrNo1KOO$3LDO5-%6(qv;y+y! z;S$?40FYE(pqTz@Y;I~mVDgW_ZRRl$%%{Tuq568T*1)q$HH(3p@Oeu@p(j}jTx@)J z7kf+7H3*v7$B!1Gw`6{1=iD_Hk_~^9Q!EZZW-Eooof{;E`-Aly1u6;KJ?zci94f@Q zSV2^**3C9{W@nv>4203WUV)TXi|^?f8ekQSK9)ya!y@MzH4TLJcbayM8mA)=!C8ILjg^C_Ws@QTg@uzwz zT={Om^A#n$YrU_Or2>Ka-fU4V{jiK_Vpfh9AdF-Wyw-VfzR~JI3CwvH03jdA--YNi zGHmX?eb_RwGPe$uc+r;8?|Ql}*L3Yjhv}`~mI!96B#a#PI0z~4Q@3k^?Z7*wZw~lZ zMA3>ju?8M-FiJqbO#C3QakK6&kV3IwF01~>9S@#3*iN6XovyVq#~(V(+5JxNatb5X zeu)AY9M>~{KpU`}Vfag@0*SGW@Fn|Xga`z#*XIzkHjvg?$!?`frB1wpmB=dcW`XPi zyNsXZ+9RY25|Swv411X;D;$RUC{CIzb7AAaM;&T%O(fq5Md0|ja^rOX4a@voUF~lk zJmTW>Q8{kU|E$H)y)4V$D@Ly4if=DP60v_jPDpNkG^KLy6VXCr>2bFmP6p8cOCMWn zjZn8sI_L?0ZFxD&OHu# z=qSG?*8U2b?ehJ4ePU}~I}DUOWA>~~V4?v+df}3mFPTBR>If+drZkqhT`$XbG=Ri6 zvuZIn?{Yh|Rxa^NNfeWYJM_UMLofKl>xZo6B1{u3ck{?>39<$+;Ie|eK1;kW?2lbO0*6d);KKP9u^S)8ek;O+}!;oHsIXDzU6 zH*C~LjK=q~D&KpV zRJgHCGBVkJ^#GZ(qX@X?0n2o!i0@(6ey)$WZHWn~p@9{F%sMk%LC>^C0%*?Oj;d&{ zhdhx()O_zwO=ju}4|v9enb-~s;CgRS7sJHBK%luHlk32nQXL_70w`lSkK5B|Fa4~K z2xKbt`r(p4=a5Xm)-n+b-XA9xF?^FIjDhR(IFbg(xydnueBSD+<(8lP#C2BOi;HJyaUWNcR_I3%hY|ZzC4JrVhi^>-Ner ze8R3b#zo1-94Tp@rEtU-`xK#Y?^@no8^XGdBDU{hBGr96X^QrNE6zx-Qk*)kt+o#i^ zJ4PrWvi>$ebHO!_`l!f*3sXgLT;V^w!PJcB0hG~@T-~qBCNZe+Ke$}^ZT750IpV*+ zv1y4f`nz-+^!>jYZ|=M7Kxhu|Uzz&E>h}kP8>ddHr2c0@@%az-(tJ-^s@(n;)jqtN z%DL0dnB3E&T3GwyU(C&{d`Eefj{fYyr|Dn{2V&lfXg)8vnctS@HU6(IxAe4(HR0+` GkpBaFVb{h0 literal 0 HcmV?d00001 diff --git a/src/main/resources/static/logo.svg b/src/main/resources/static/logo.svg new file mode 100644 index 0000000..08ab481 --- /dev/null +++ b/src/main/resources/static/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplicationTests.java b/src/test/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplicationTests.java new file mode 100644 index 0000000..e1cfa3f --- /dev/null +++ b/src/test/java/eu/dnetlib/scholexplorer/api/ScholexplorerApiApplicationTests.java @@ -0,0 +1,173 @@ +package eu.dnetlib.scholexplorer.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import eu.dnetlib.scholexplorer.api.index.ScholixIndexManager; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.*; + + +class TestResult { + public long totalTimeMs; + public int totalItems; + public String relationName; + + public long getTotalTimeMs() { + return totalTimeMs; + } + + public TestResult setTotalTimeMs(long totalTimeMs) { + this.totalTimeMs = totalTimeMs; + return this; + } + + public int getTotalItems() { + return totalItems; + } + + public TestResult setTotalItems(int totalItems) { + this.totalItems = totalItems; + return this; + } + + public String getRelationName() { + return relationName; + } + + public TestResult setRelationName(String relationName) { + this.relationName = relationName; + return this; + } +} + +@SpringBootTest +class ScholexplorerApiApplicationTests { + + @Autowired + ScholixIndexManager scholixRepo; + + final ObjectMapper mapper = new ObjectMapper(); + + + final List relations = Arrays.asList("IsDerivedFrom", + "Continues", + "References", + "HasVersion", + "IsVersionOf", + "IsSupplementTo", + "IsDocumentedBy", + "Documents", + "IsContinuedBy", + "IsSupplementedBy", + "IsSourceOf", + "Reviews", + "Cites", + "IsAmongTopNSimilarDocuments", + "IsPartOf", + "HasPart", + "IsIdenticalTo", + "IsNewVersionOf", + "IsRelatedTo", + "HasAmongTopNSimilarDocuments", + "IsCitedBy", + "IsPreviousVersionOf", + "IsReferencedBy", + "IsVariantFormOf", + "IsCompiledBy", + "IsReviewedBy", + "IsDescribedBy", + "Compiles", + "IsOriginalFormOf", + "Describes"); + + private List executeTest() throws Exception { + Random rand = new Random(); + + final List infos = new ArrayList<>(); + String currentRel1 =relations.get(rand.nextInt(relations.size())); + long start = System.nanoTime(); + List result = scholixRepo.findPage(currentRel1); + long total = (System.nanoTime() - start) /1000000; + int cnt = result.size(); + + infos.add(new TestResult().setRelationName(currentRel1).setTotalItems(cnt).setTotalTimeMs(total)); + + String currentRel2 =relations.get(rand.nextInt(relations.size())); + start = System.nanoTime(); + result = scholixRepo.findPage(currentRel2); + long total2 = (System.nanoTime() - start) /1000000; + int cnt2 = result.size(); + infos.add(new TestResult().setRelationName(currentRel2).setTotalItems(cnt2).setTotalTimeMs(total2)); + + String currentRel3 =relations.get(rand.nextInt(relations.size())); + start = System.nanoTime(); + result = scholixRepo.findPage(currentRel3); + long total3 = (System.nanoTime() - start) /1000000; + int cnt3 = result.size(); + infos.add(new TestResult().setRelationName(currentRel3).setTotalItems(cnt3).setTotalTimeMs(total3)); + return infos; + } + + @Test + void contextLoads() throws Exception { + final List infos = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + infos.addAll(executeTest()); + } + + + LongSummaryStatistics summary = infos.stream().map(TestResult::getTotalTimeMs).mapToLong(Long::longValue).summaryStatistics(); + + System.out.println(summary.getMax()); + System.out.println(summary.getMin()); + System.out.println(summary.getAverage()); + } + + + + @Test + void testlinksFromPid() throws ScholixException { + Pair> result = scholixRepo.linksFromPid(null, null, "doi", null, null, null, "pmid", null, null, null, 0); + result.getRight().forEach( + s -> Assertions.assertTrue(s.getTarget().getIdentifier().stream().anyMatch(p -> p.getSchema().equals("doi"))) + ); + + result.getRight().forEach( + s -> Assertions.assertTrue(s.getSource().getIdentifier().stream().anyMatch(p -> p.getSchema().equals("pmid"))) + ); + + + result = scholixRepo.linksFromPid(null, null, null, null, "dataset", null, null, null, "publication","IsSupplementedBy", 0); + System.out.println(result.getLeft()); + result.getRight().forEach( + s -> { + Assertions.assertEquals("dataset", s.getTarget().getObjectType()); + Assertions.assertEquals("publication", s.getSource().getObjectType()); + Assertions.assertEquals("issupplementedby", s.getRelationship().getName()); + + }); + + result = scholixRepo.linksFromPid(null, null, null, null, "publication", null, null, null, "publication","IsVersionOf", 0); + System.out.println(result.getLeft()); + result.getRight().forEach( + s -> { + Assertions.assertEquals("publication", s.getTarget().getObjectType()); + Assertions.assertEquals("publication", s.getSource().getObjectType()); + Assertions.assertEquals("IsVersionOf".toLowerCase(), s.getRelationship().getName()); + + }); + + + + } + + + + +} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties new file mode 100644 index 0000000..5ee330f --- /dev/null +++ b/src/test/resources/application.properties @@ -0,0 +1,38 @@ +spring.application.name=scholexplorer-api +spring.main.banner-mode = console + +server.public_url = +server.public_desc = API Base URL + +logging.level.root = INFO +dhp.swagger.api.host = localhost:8080 +#dhp.swagger.api.host = api.scholexplorer.openaire.eu +dhp.swagger.api.basePath = / + +maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml + +# +#spring.thymeleaf.cache=false +# +management.endpoints.web.exposure.include = prometheus,health +management.endpoints.web.base-path = / +management.endpoints.web.path-mapping.prometheus = metrics +management.endpoints.web.path-mapping.health = health +management.endpoint.health.show-details = always + +management.metrics.distribution.percentiles-histogram.http.server.requests=false +management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms +management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999 + + + +#scholix.elastic.clusterNodes = 10.19.65.51:9200,10.19.65.52:9200,10.19.65.53:9200,10.19.65.54:9200 +scholix.elastic.clusterNodes = localhost:9200 +scholix.elastic.indexName = scholix +scholix.elastic.indexResourceName = summary +scholix.elastic.socketTimeout = 60000 +scholix.elastic.connectionTimeout= 60000 + + + +