D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix merge

This commit is contained in:
Michele Artini 2022-03-29 12:22:54 +02:00
parent 3db226bea6 d7824d5071
commit 9308ab35f0
40 changed files with 1646 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,6 +5,7 @@
*.iml
*.ipr
*.iws
*.java-version
*~
/**/*.sh
/**/my_application.properties

4
apps/dhp-broker-application/pom.xml
View File

@ -4,8 +4,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.4-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

4
apps/dhp-broker-public-application/pom.xml
View File

@ -4,8 +4,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.4-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

4
apps/dhp-mdstore-manager/pom.xml
View File

@ -4,8 +4,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.4-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

2
apps/dnet-orgs-database-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

0
apps/dnet-orgs-database-application/import_certificates.sh → apps/dnet-orgs-database-application/scripts/import_certificates.sh
View File

10
apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql
View File

@ -14,11 +14,19 @@ UPDATE tmp_dedup_events SET local_id = oa_original_id WHERE local_id = '' OR lo
UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN' WHERE oa_country = '' OR oa_country IS NULL;
UPDATE tmp_dedup_events SET oa_name = oa_acronym WHERE oa_name = '' OR oa_name IS NULL;
DELETE FROM tmp_dedup_events WHERE oa_name = '' OR oa_name IS NULL;
-- delete invalid relations (a raw org can not be suggested as duplicate and as new org)
DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
SELECT oa_original_id
FROM tmp_dedup_events
GROUP BY oa_original_id HAVING count(oa_original_id) > 1)
AND (local_id = '' OR local_id is NULL);
-- delete invalid relations (a raw org can not be suggested to multiple orgs)
DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
SELECT oa_original_id
FROM tmp_dedup_events
GROUP BY oa_original_id HAVING count(*) > 1)
GROUP BY oa_original_id HAVING count(oa_original_id) > 1)
AND local_id NOT LIKE 'openorgs____::%';
-- IMPORT MISSING TERMS

7
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -648,7 +648,6 @@ GROUP BY o.id, o.name, o.city, o.country
ORDER BY o.name;
CREATE TABLE org_index_search(id text PRIMARY KEY, txt tsvector);
CREATE INDEX org_index_search_txt_idx ON org_index_search(txt);
CREATE INDEX org_index_search_txt_gin_idx ON org_index_search USING gin(txt);
CREATE OR REPLACE FUNCTION refresh_index_search() RETURNS bigint AS $$
@ -656,9 +655,10 @@ CREATE OR REPLACE FUNCTION refresh_index_search() RETURNS bigint AS $$
WITH d as (
INSERT INTO org_index_search(id, txt) SELECT
o.id,
to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
FROM organizations o
LEFT OUTER JOIN other_names n on (o.id = n.id)
LEFT OUTER JOIN other_ids i on (o.id = i.id)
LEFT OUTER JOIN acronyms a on (o.id = a.id)
LEFT OUTER JOIN urls u on (o.id = u.id)
GROUP BY o.id, o.name RETURNING *
@ -680,9 +680,10 @@ CREATE OR REPLACE FUNCTION insert_or_update_index_search_trigger() RETURNS trigg
BEGIN
INSERT INTO org_index_search(id, txt) (SELECT
o.id,
to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
FROM organizations o
LEFT OUTER JOIN other_names n on (o.id = n.id)
LEFT OUTER JOIN other_ids i on (o.id = i.id)
LEFT OUTER JOIN acronyms a on (o.id = a.id)
LEFT OUTER JOIN urls u on (o.id = u.id)
WHERE o.id = new.id

4
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_metadata.form.html
View File

@ -59,7 +59,7 @@
required="required"
ng-class="{'is-invalid' : organizationForm.org_cntr.$error.required}">
<option disabled="disabled" value='' ng-if="!org.country">country...</option>
<option ng-repeat="c in vocabularies.countries"value="{{c.value}}">{{c.name}}</option>
<option ng-repeat="c in vocabularies.countries" value="{{c.value}}">{{c.name}}</option>
</select>
<div class="input-group-append input-group-prepend">
<div class="input-group-text bg-primary text-white" ng-class="{'bg-light text-dark' : mode == 'readonly' || mode == 'not_authorized', 'bg-warning' : mode == 'approve'}">lat</div>
@ -252,7 +252,7 @@
<tr ng-if="org.relations.length == 0">
<td class="text-center text-muted" colspan="4">No relations</td>
</tr>
<tr ng-repeat="r in org.relations | orderBy:['type','relatedOrgName']">
<tr ng-repeat="r in org.relations">
<td class="pl-3" colspan="3">This organizazion
<!-- NB: The showed value MUST be the inverse -->
<span ng-switch="r.type">

7
apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js
View File

@ -513,7 +513,7 @@ orgsModule.controller('byTypeCtrl', function ($scope, $http, $routeParams, $loca
});
orgsModule.controller('showEditCtrl', function ($scope, $http, $routeParams, $route, $location, $timeout, $window, vocabulariesService) {
orgsModule.controller('showEditCtrl', function ($scope, $http, $routeParams, $route, $location, $timeout, $window, $filter, vocabulariesService) {
$scope.orgId = $routeParams.id;
$scope.org = {};
$scope.duplicates = [];
@ -529,7 +529,10 @@ orgsModule.controller('showEditCtrl', function ($scope, $http, $routeParams, $ro
$scope.gotoTab = function(tab) {
$scope.org = {};
call_http_get($http, 'api/organizations/get?id=' + $scope.orgId, function(res) { $scope.org = res.data; });
call_http_get($http, 'api/organizations/get?id=' + $scope.orgId, function(res) {
res.data.relations = $filter('orderBy')(res.data.relations, ['type','relatedOrgName'], false);
$scope.org = res.data;
});
if (tab == 2) {
$scope.duplicates = [];

5
apps/pom.xml
View File

@ -3,8 +3,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.4-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -17,6 +17,7 @@
<module>dhp-mdstore-manager</module>
<module>dnet-orgs-database-application</module>
<module>dnet-exporter-api</module>
<module>scholexplorer-api</module>
</modules>
<dependencies>

48
apps/scholexplorer-api/pom.xml Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
<artifactId>scholexplorer-api</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-schemas</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-help-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

124
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java Normal file
View File

@ -0,0 +1,124 @@
package eu.dnetlib.scholix.api;
import eu.dnetlib.common.app.AbstractDnetApp;
import io.micrometer.core.aop.TimedAspect;
import io.micrometer.core.instrument.*;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
@SpringBootApplication
@EnableSwagger2
@EnableCaching
@EnableScheduling
@ComponentScan(basePackages = "eu.dnetlib")
public class MainApplication extends AbstractDnetApp {
private double scale = 1000000000;
private double[] histogramValues =new double[]{.005 * scale, .01 * scale, .25 * scale, .5 * scale, .75 * scale, scale, 2.5 * scale, 5.0 * scale, 7.5 * scale, 10.0 * scale};
@Value("${dhp.swagger.api.host}")
private String swaggetHost;
@Value("${dhp.swagger.api.basePath}")
private String swaggerPath;
public static void main(final String[] args) {
SpringApplication.run(MainApplication.class, args);
}
@Bean
public TaggedCounter myCounter(MeterRegistry meterRegistry) {
return new TaggedCounter(ScholixAPIConstants.SCHOLIX_MANAGER_COUNTER_NAME, ScholixAPIConstants.SCHOLIX_MANAGER_TAG_NAME,meterRegistry);
}
@Bean
public TimedAspect timedAspect(MeterRegistry meterRegistry) {
MeterFilter mf = new MeterFilter() {
@Override
public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) {
if (id.getName().startsWith(ScholixAPIConstants.SCHOLIX_COUNTER_PREFIX)) {
return DistributionStatisticConfig.builder()
.percentilesHistogram(false)
.serviceLevelObjectives( histogramValues)
.build()
.merge(config);
}
return config;
}
};
meterRegistry.config().meterFilter(mf);
return new TimedAspect(meterRegistry);
}
@Override
protected void configSwagger(final Docket docket) {
docket
.host(swaggetHost)
.pathMapping(swaggerPath)
.groupName(ScholixAPIConstants.API_V1_NAME)
.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/v1"))
.build()
.apiInfo(new ApiInfoBuilder()
.title(ScholixAPIConstants.API_V1_NAME)
.description(ScholixAPIConstants.API_DESCRIPTION)
.version("1.0")
.contact(ApiInfo.DEFAULT_CONTACT)
.license("Apache 2.0")
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
.build());
}
@Bean (name = "SpringDocketv2")
public Docket v2Docket() {
final Docket docket = new Docket(DocumentationType.SWAGGER_2);
docket
.host(swaggetHost)
.pathMapping(swaggerPath)
.groupName(ScholixAPIConstants.API_V2_NAME)
.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/v2"))
.build()
.apiInfo(new ApiInfoBuilder()
.title(ScholixAPIConstants.API_V2_NAME)
.description(ScholixAPIConstants.API_DESCRIPTION)
.version("2.0")
.contact(ApiInfo.DEFAULT_CONTACT)
.license("Apache 2.0")
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
.build());
return docket;
}
}

26
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java Normal file
View File

@ -0,0 +1,26 @@
package eu.dnetlib.scholix.api;
import eu.dnetlib.scholix.api.index.ElasticSearchPool;
import eu.dnetlib.scholix.api.index.ElasticSearchProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class RestClientConfig {
@Autowired
private ElasticSearchProperties elasticSearchProperties;
@Bean
public ElasticSearchPool connectionPool() {
elasticSearchProperties.setMaxIdle(5);
elasticSearchProperties.setMaxTotal(10);
ElasticSearchPool pool = new ElasticSearchPool(elasticSearchProperties);
return pool;
}
}

23
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIConstants.java Normal file
View File

@ -0,0 +1,23 @@
package eu.dnetlib.scholix.api;
public class ScholixAPIConstants {
public static final String API_V1_NAME = "Scholexplorer API V1.0";
public static final String API_V2_NAME = "Scholexplorer API V2.0";
public static String API_DESCRIPTION =" <p style=\"text-align:center;\"><img src=\"/logo.png\" alt=\"ScholeXplorer\"> </p>" +
"The Scholix Swagger API allows clients to run REST queries over the Scholexplorer index in order to fetch links matching given criteria. In the current version, clients can search for:" +
"<ul><li>Links whose source object has a given PID or PID type</li>" +
"<li>Links whose source object has been published by a given data source (\"data source as publisher\")</li>" +
"<li>Links that were collected from a given data source (\"data source as provider\").</li></ul>";
public static String SCHOLIX_MANAGER_COUNTER_NAME= "scholixLinkCounter";
public static final String SCHOLIX_MANAGER_TAG_NAME = "links";
public static String SCHOLIX_COUNTER_PREFIX = "scholix";
}

8
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIVersion.java Normal file
View File

@ -0,0 +1,8 @@
package eu.dnetlib.scholix.api;
public enum ScholixAPIVersion {
V1,
V2
}

35
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixException.java Normal file
View File

@ -0,0 +1,35 @@
package eu.dnetlib.scholix.api;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
public class ScholixException extends Exception{
private static final long serialVersionUID = -3414428892721711308L;
public ScholixException() {
super();
}
public ScholixException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public ScholixException(String message, Throwable cause) {
super(message, cause);
}
public ScholixException(String message) {
super(message);
}
public ScholixException(Throwable cause) {
super(cause);
}
}

33
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/TaggedCounter.java Normal file
View File

@ -0,0 +1,33 @@
package eu.dnetlib.scholix.api;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import java.util.HashMap;
import java.util.Map;
public class TaggedCounter {
private String name;
private String tagName;
private MeterRegistry registry;
private Map<String, Counter> counters = new HashMap<>();
public TaggedCounter(String name, String tagName, MeterRegistry registry) {
this.name = name;
this.tagName = tagName;
this.registry = registry;
}
public void increment(String tagValue){
Counter counter = counters.get(tagValue);
if(counter == null) {
counter = Counter.builder(name).tags(tagName, tagValue).register(registry);
counters.put(tagValue, counter);
}
counter.increment();
}
}

50
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/DatasourceV1.java Normal file
View File

@ -0,0 +1,50 @@
package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.sx.api.model.v1.LinkPublisher;
import eu.dnetlib.scholix.api.ScholixException;
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
import io.micrometer.core.annotation.Timed;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v1")
@Api(tags = {
"Datasources"
})
public class DatasourceV1 extends AbstractDnetController {
@Autowired
private ScholixIndexManager manager;
@Timed(value = "scholix.v1.datasources", description = "Time taken to return all datasources on Version 1.0 of Scholix")
@Operation(
summary = "Get all Datasources",
description = "returns a list of all datasources")
@GetMapping("/listDatasources")
public List<LinkPublisher> getDatasources() throws ScholixException {
final List<Pair<String, Long>> result = manager.totalLinksByProvider(null);
if (result == null)
return new ArrayList<>();
return result.stream().map(p -> new LinkPublisher().name(p.getKey()).totalRelationships(p.getValue().intValue())).collect(Collectors.toList());
}
}

48
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/LinkProviderV2.java Normal file
View File

@ -0,0 +1,48 @@
package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.scholix.api.ScholixException;
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v2")
@Api(tags = {
"LinkProvider : Operation related to the Link Provider"
})
public class LinkProviderV2 {
@Autowired
ScholixIndexManager manager;
@Operation(
summary = "Get all Link Providers",
description = "Return a list of link provider and relative number of relations")
@GetMapping("/LinkProvider")
public List<LinkProviderType> getLinkProviders(
@Parameter(in = ParameterIn.QUERY, description = "Filter the link provider name") @RequestParam(required = false) String name
) throws ScholixException {
List<Pair<String, Long>> result = manager.totalLinksByProvider(name);
if (result==null)
return new ArrayList<>();
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
}
}

61
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/LinkPublisherV2.java Normal file
View File

@ -0,0 +1,61 @@
package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.scholix.api.ScholixException;
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
import eu.dnetlib.dhp.schema.sx.api.model.v2.LinkProviderType;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v2/LinkPublisher")
@Api(tags = {
"LinkPublisher : Operation related to the Link Publisher"
})
public class LinkPublisherV2 {
@Autowired
ScholixIndexManager manager;
@Operation(
summary = "Get All Publishers that provide source object",
description = "Return a List of all Publishers that provide source objects in Scholix " +
"links and the total number of links where the source object comes from this publisher")
@GetMapping("/inSource")
public List<LinkProviderType> getInSource(
@Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) String name
) throws ScholixException {
List<Pair<String, Long>> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.source,name);
if (result==null)
return new ArrayList<>();
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
}
@Operation(
summary = "Get All Publishers that provide target object",
description = "Return a List of all Publishers that provide source objects in Scholix " +
"links and the total number of links where the target object comes from this publisher")
@GetMapping("/inTarget")
public List<LinkProviderType> getInTarget(
@Parameter(in = ParameterIn.QUERY, description = "Filter the link publisher name") @RequestParam(required = false) String name
) throws ScholixException {
List<Pair<String, Long>> result = manager.totalLinksPublisher(ScholixIndexManager.RelationPrefix.target,name);
if (result==null)
return new ArrayList<>();
return result.stream().map(s -> new LinkProviderType().name(s.getLeft()).totalRelationships(s.getValue().intValue())).collect(Collectors.toList());
}
}

114
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV1.java Normal file
View File

@ -0,0 +1,114 @@
package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.scholix.api.ScholixException;
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
import eu.dnetlib.dhp.schema.sx.api.model.v1.ScholixV1;
import io.micrometer.core.annotation.Timed;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.media.Schema;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v1")
@Api(tags = {"Scholix"})
public class ScholixControllerV1 extends AbstractDnetController {
@Autowired
ScholixIndexManager manager;
@Operation(
summary = "Get all Scholix relation collected from a publisher",
description = "return a list of scholix object published from a specific publisher"
)
@GetMapping("/linksFromPublisher")
@Timed(value = "scholix.v1.linksFromPublisher", description = "Time taken to return links on Version 1.0 of Scholix collected from a publisher")
public List<ScholixV1> linksFromPublisher(
@Parameter(
in = ParameterIn.QUERY,
description = "Filter Scholix relationships collected from a publisher",
schema = @Schema(), required = true) String publisher,
@Parameter(in = ParameterIn.QUERY,
description = "The page number") @RequestParam(required = false) Integer page
) throws ScholixException {
final int currentPage = page != null ? page : 0;
Pair<Long, List<Scholix>> scholixResult =manager.linksFromPid(null,null,null,publisher,
null,null,null,null,null,null, currentPage
);
List<Scholix> scholixData = scholixResult.getValue();
if (scholixData== null)
return null;
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
}
@Operation(
summary = "Get all Scholix relation collected from a datasource",
description = "return a list of scholix object collected from a specific datasource"
)
@GetMapping("/linksFromDatasource")
@Timed(value = "scholix.v1.linksFromDatasource", description = "Time taken to return links on Version 1.0 of Scholix collected from a LinkProvider")
public List<ScholixV1> linksFromDatasource(
@Parameter(
in = ParameterIn.QUERY,
description = "Filter Scholix relationships collected from a LinkProvider",
schema = @Schema()) @NotNull String datasource,
@Parameter(in = ParameterIn.QUERY,
description = "The page number") @RequestParam(required = false) Integer page
) throws ScholixException {
final int currentPage = page != null ? page : 0;
Pair<Long, List<Scholix>> scholixResult =manager.linksFromPid(datasource,null,null,null,
null,null,null,null,null,null, currentPage
);
List<Scholix> scholixData = scholixResult.getValue();
if (scholixData== null)
return null;
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
}
@Operation(
summary = "Retrieve all scholix links from a persistent identifier",
description = "The linksFromPid endpoint returns a list of scholix object related from a specific persistent identifier"
)
@GetMapping("/linksFromPid")
@Timed(value = "scholix.v1.linksFromPid", description = "Time taken to return links on Version 1.0 of Scholix related from a specific persistent identifier")
public List<ScholixV1> linksFromPid(
@Parameter(in = ParameterIn.QUERY, description = "persistent Identifier") @NotNull String pid,
@Parameter(in = ParameterIn.QUERY, description = "Persistent Identifier Type") @RequestParam(required = false) String pidType,
@Parameter(in = ParameterIn.QUERY, description = "typology target filter should be publication, dataset or unknown") @RequestParam(required = false) String typologyTarget,
@Parameter(in = ParameterIn.QUERY, description = "a datasource provenance filter of the target relation") @RequestParam(required = false) String datasourceTarget,
@Parameter(in = ParameterIn.QUERY,
description = "The page number") @RequestParam(required = false) Integer page
) throws ScholixException {
final int currentPage = page != null ? page : 0;
Pair<Long, List<Scholix>> scholixResult =manager.linksFromPid(datasourceTarget,null,null,null,
typologyTarget,pid,pidType,null,null,null, currentPage
);
List<Scholix> scholixData = scholixResult.getValue();
if (scholixData== null)
return null;
return scholixData.stream().map(ScholixV1::fromScholix).collect(Collectors.toList());
}
}

77
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java Normal file
View File

@ -0,0 +1,77 @@
package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.sx.api.model.v2.PageResultType;
import eu.dnetlib.dhp.schema.sx.api.model.v2.ScholixType;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.scholix.api.ScholixException;
import eu.dnetlib.scholix.api.index.ScholixIndexManager;
import io.micrometer.core.annotation.Timed;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/v2")
@Api(tags = {
"Links : Operation related to the Scholix Links"
})
public class ScholixControllerV2 extends AbstractDnetController {
@Autowired
private ScholixIndexManager manager;
@Timed(value = "scholix.v2.links", description = "Time taken to return links on Version 2.0 of Scholix")
@ApiOperation("Get Scholix Links")
@GetMapping("/Links")
public PageResultType links(
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships collected from a LinkProvider") String linkProvider,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a target pid") String targetPid,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a target pid type") String targetPidType,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a target published in a Publisher named targetPublisher") String targetPublisher,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a target type (literature, dataset, unknown)") String targetType,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a source pid") String sourcePid,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a source pid type") String sourcePidType,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a source published in a Publisher named sourcePublisher") String sourcePublisher,
@Parameter(in = ParameterIn.QUERY,
description = "Filter Scholix relationships having a source type (literature, dataset, unknown)") String sourceType,
@Parameter(in = ParameterIn.QUERY,
description = "Filter scholix Links having collected after this date") String harvestedAfter,
@Parameter(in = ParameterIn.QUERY,
description = "select page of result") Integer page) throws Exception {
if (StringUtils.isEmpty(sourcePid) && StringUtils.isEmpty(targetPid) && StringUtils.isEmpty(sourcePublisher)&& StringUtils.isEmpty(targetPublisher)&& StringUtils.isEmpty(linkProvider))
throw new ScholixException("The method requires one of the following parameters: sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider");
try {
final int currentPage = page != null ? page : 0;
Pair<Long, List<Scholix>> scholixResult = manager.linksFromPid( linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, harvestedAfter, currentPage);
final PageResultType pageResult = new PageResultType();
pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10);
pageResult.setTotalLinks(scholixResult.getLeft().intValue());
pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList()));
return pageResult;
} catch (Throwable e) {
throw new ScholixException("Error on requesting url ", e);
}
}
}

74
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchClientFactory.java Normal file
View File

@ -0,0 +1,74 @@
package eu.dnetlib.scholix.api.index;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
/**
* The type Elastic search client factory.
*/
public class ElasticSearchClientFactory implements PooledObjectFactory<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> {
private final ElasticSearchProperties elasticSearchProperties;
/**
* Instantiates a new Elastic search client factory.
*
* @param elasticSearchProperties the elastic search properties
*/
public ElasticSearchClientFactory(final ElasticSearchProperties elasticSearchProperties){
this.elasticSearchProperties = elasticSearchProperties;
}
public PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> makeObject() throws Exception {
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo(elasticSearchProperties.getClusterNodes().split(","))
.withConnectTimeout(elasticSearchProperties.getConnectionTimeout())
.withSocketTimeout(elasticSearchProperties.getSocketTimeout())
.build();
RestHighLevelClient cc = RestClients.create(clientConfiguration).rest();
return new DefaultPooledObject<>(new ImmutablePair<>(cc, new ElasticsearchRestTemplate(cc)));
}
public void destroyObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
RestHighLevelClient client = pooledObject.getObject().getLeft();
if(client!=null&&client.ping(RequestOptions.DEFAULT)){
try {
client.close();
}catch (Exception e){
//ignore
}
}
}
public boolean validateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) {
RestHighLevelClient client = pooledObject.getObject().getLeft();
try {
return client.ping(RequestOptions.DEFAULT);
}catch(Exception e){
return false;
}
}
public void activateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
RestHighLevelClient client = pooledObject.getObject().getLeft();
boolean response = client.ping(RequestOptions.DEFAULT);
}
public void passivateObject(PooledObject<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> pooledObject) throws Exception {
//nothing
}
}

34
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchPool.java Normal file
View File

@ -0,0 +1,34 @@
package eu.dnetlib.scholix.api.index;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
/**
* The type Elastic search pool.
*/
public class ElasticSearchPool extends Pool<Pair<RestHighLevelClient, ElasticsearchRestTemplate>> {
private final ElasticSearchProperties elasticSearchProperties;
/**
* Instantiates a new Elastic search pool.
*
* @param elasticSearchProperties the elastic search properties
*/
public ElasticSearchPool(ElasticSearchProperties elasticSearchProperties){
super(elasticSearchProperties, new ElasticSearchClientFactory(elasticSearchProperties));
this.elasticSearchProperties = elasticSearchProperties;
}
/**
* Gets elastic search properties.
*
* @return the elastic search properties
*/
public ElasticSearchProperties getElasticSearchProperties() {
return elasticSearchProperties;
}
}

104
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java Normal file
View File

@ -0,0 +1,104 @@
package eu.dnetlib.scholix.api.index;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import javax.validation.constraints.NotNull;
/**
* The type Elastic search properties.
*/
@Component("elasticSearchProperties")
@ConfigurationProperties(prefix = "scholix.elastic")
public class ElasticSearchProperties extends GenericObjectPoolConfig {
@NotNull
private String clusterNodes;
@NotNull
private String indexName;
@NotNull
private long connectionTimeout;
@NotNull
private long socketTimeout;
/**
* Gets cluster nodes.
*
* @return the cluster nodes
*/
public String getClusterNodes() {
return clusterNodes;
}
/**
* Sets cluster nodes.
*
* @param clusterNodes the cluster nodes
* @return the cluster nodes
*/
public ElasticSearchProperties setClusterNodes(String clusterNodes) {
this.clusterNodes = clusterNodes;
return this;
}
/**
* Gets index name.
*
* @return the index name
*/
public String getIndexName() {
return indexName;
}
/**
* Sets index name.
*
* @param indexName the index name
* @return the index name
*/
public ElasticSearchProperties setIndexName(String indexName) {
this.indexName = indexName;
return this;
}
/**
* Gets connection timeout.
*
* @return the connection timeout
*/
public long getConnectionTimeout() {
return connectionTimeout;
}
/**
* Sets connection timeout.
*
* @param connectionTimeout the connection timeout
* @return the connection timeout
*/
public ElasticSearchProperties setConnectionTimeout(long connectionTimeout) {
this.connectionTimeout = connectionTimeout;
return this;
}
/**
* Gets socket timeout.
*
* @return the socket timeout
*/
public long getSocketTimeout() {
return socketTimeout;
}
/**
* Sets socket timeout.
*
* @param socketTimeout the socket timeout
* @return the socket timeout
*/
public ElasticSearchProperties setSocketTimeout(long socketTimeout) {
this.socketTimeout = socketTimeout;
return this;
}
}

231
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/Pool.java Normal file
View File

@ -0,0 +1,231 @@
package eu.dnetlib.scholix.api.index;
import eu.dnetlib.scholix.api.ScholixException;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
/**
* When using the Java High Level REST Client provided by the Elasticsearch official website, it is found that there is no in the client API.
* Connecting to connect the pool, create a new connection every time, this is impact in high concurrency situation, so it is ready to be on the client
* API increases the concept of pool.
*
* Fortunately, we don't need to turn your weight to write the implementation of the connection pool, because Apache provides us with the general framework of the connection pool.
* Commons-pool2, and we only need to implement some logic according to the frame design. Used in the REDIS client API
* Jedispool is based on Commons-pool2 implementation.
*
* Let's take a look at how to achieve it.
*
* First we have to create a pool class, this pool introduces GenericObjectPool in Commons-pool2 through dependent manner. In this class
* In, we define how to borrow objects and returns objects from the pool.
*
* @param <T> the type parameter
*/
public class Pool<T> implements Cloneable {
/**
* The Internal pool.
*/
protected GenericObjectPool<T> internalPool ;
/**
* Instantiates a new Pool.
*/
public Pool(){
super();
}
/**
* Instantiates a new Pool.
*
* @param poolConfig the pool config
* @param factory the factory
*/
public Pool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory<T> factory){
initPool(poolConfig, factory);
}
/**
* Init pool.
*
* @param poolConfig the pool config
* @param factory the factory
*/
public void initPool(final GenericObjectPoolConfig poolConfig, PooledObjectFactory<T> factory) {
if (this.internalPool != null) {
try {
closeInternalPool();
} catch (Exception e) {
}
}
this.internalPool = new GenericObjectPool<T>(factory, poolConfig);
}
/**
* Close internal pool.
*
* @throws ScholixException the scholix exception
*/
protected void closeInternalPool() throws ScholixException {
try {
internalPool.close();
} catch (Exception e) {
throw new ScholixException("Could not destroy the pool", e);
}
}
/**
* Gets resource.
*
* @return the resource
* @throws ScholixException the scholix exception
*/
public T getResource() throws ScholixException {
try {
return internalPool.borrowObject();
} catch (Exception e) {
throw new ScholixException("Could not get a resource from the pool", e);
}
}
/**
* Return resource.
*
* @param resource the resource
* @throws ScholixException the scholix exception
*/
public void returnResource(final T resource) throws ScholixException {
if (resource != null) {
returnResourceObject(resource);
}
}
private void returnResourceObject(final T resource) throws ScholixException {
if (resource == null) {
return;
}
try {
internalPool.returnObject(resource);
} catch (Exception e) {
throw new ScholixException("Could not return the resource to the pool", e);
}
}
/**
* Return broken resource.
*
* @param resource the resource
* @throws ScholixException the scholix exception
*/
public void returnBrokenResource(final T resource) throws ScholixException {
if (resource != null) {
returnBrokenResourceObject(resource);
}
}
private void returnBrokenResourceObject(T resource) throws ScholixException {
try {
internalPool.invalidateObject(resource);
} catch (Exception e) {
throw new ScholixException("Could not return the resource to the pool", e);
}
}
/**
* Destroy.
*
* @throws ScholixException the scholix exception
*/
public void destroy() throws ScholixException {
closeInternalPool();
}
/**
* Gets num active.
*
* @return the num active
*/
public int getNumActive() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getNumActive();
}
/**
* Gets num idle.
*
* @return the num idle
*/
public int getNumIdle() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getNumIdle();
}
/**
* Gets num waiters.
*
* @return the num waiters
*/
public int getNumWaiters() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getNumWaiters();
}
/**
* Gets mean borrow wait time millis.
*
* @return the mean borrow wait time millis
*/
public long getMeanBorrowWaitTimeMillis() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getMeanBorrowWaitTimeMillis();
}
/**
* Gets max borrow wait time millis.
*
* @return the max borrow wait time millis
*/
public long getMaxBorrowWaitTimeMillis() {
if (poolInactive()) {
return -1;
}
return this.internalPool.getMaxBorrowWaitTimeMillis();
}
private boolean poolInactive() {
return this.internalPool == null || this.internalPool.isClosed();
}
/**
* Add objects.
*
* @param count the count
* @throws Exception the exception
*/
public void addObjects(int count) throws Exception {
try {
for (int i = 0; i < count; i++) {
this.internalPool.addObject();
}
} catch (Exception e) {
throw new Exception("Error trying to add idle objects", e);
}
}
}

315
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java Normal file
View File

@ -0,0 +1,315 @@
package eu.dnetlib.scholix.api.index;
import eu.dnetlib.dhp.schema.sx.scholix.Scholix;
import eu.dnetlib.scholix.api.ScholixException;
import eu.dnetlib.scholix.api.TaggedCounter;
import io.micrometer.core.annotation.Timed;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* The type Scholix index manager.
*/
@Component
public class ScholixIndexManager {
/**
* The Elastic search properties.
*/
@Autowired
ElasticSearchProperties elasticSearchProperties;
/**
* The Elasticsearch template.
*/
@Autowired
ElasticSearchPool connectionPool;
/**
* The My counter.
*/
@Autowired
TaggedCounter myCounter;
/**
* The enum Pid type prefix.
*/
public enum RelationPrefix {
/**
* Source pid type prefix.
*/
source,
/**
* Target pid type prefix.
*/
target
}
private QueryBuilder createObjectTypeQuery(final RelationPrefix prefix, final String objectType ) throws ScholixException{
if (prefix == null){
throw new ScholixException("prefix cannot be null");
}
return new NestedQueryBuilder(String.format("%s", prefix), new TermQueryBuilder(String.format("%s.objectType",prefix), objectType), ScoreMode.None);
}
private QueryBuilder createPidTypeQuery(final RelationPrefix prefix, final String pidTypeValue ) throws ScholixException{
if (prefix == null){
throw new ScholixException("prefix cannot be null");
}
return new NestedQueryBuilder(String.format("%s.identifier", prefix), new TermQueryBuilder(String.format("%s.identifier.schema",prefix), pidTypeValue), ScoreMode.None);
}
private QueryBuilder createLinkProviderQuery(final String providerName ) throws ScholixException{
if (providerName == null){
throw new ScholixException("prefix cannot be null");
}
return new NestedQueryBuilder("linkprovider", new TermQueryBuilder("linkprovider.name",providerName), ScoreMode.None);
}
private QueryBuilder createLinkPublisherQuery(final RelationPrefix prefix, final String publisher ) throws ScholixException{
if (prefix == null){
throw new ScholixException("prefix cannot be null");
}
return new NestedQueryBuilder(String.format("%s.publisher", prefix), new TermQueryBuilder(String.format("%s.publisher.name",prefix), publisher), ScoreMode.None);
}
private QueryBuilder createPidValueQuery(final RelationPrefix prefix, final String pidValue ) throws ScholixException{
if (prefix == null){
throw new ScholixException("prefix cannot be null");
}
return new NestedQueryBuilder(String.format("%s.identifier", prefix), new TermQueryBuilder(String.format("%s.identifier.identifier",prefix), pidValue), ScoreMode.None);
}
private QueryBuilder createFinalQuery(final List<QueryBuilder> queries) throws ScholixException{
if (queries == null || queries.isEmpty())
throw new ScholixException("the list of queries must be not empty");
if (queries.size() ==1) {
return queries.get(0);
}
else {
final BoolQueryBuilder b = new BoolQueryBuilder();
b.must().addAll(queries);
return b;
}
}
private void incrementPidCounter(RelationPrefix prefix, String value) {
switch (value.toLowerCase()){
case "doi": {
myCounter.increment(String.format("%s_doi", prefix));
break;
}
case "pmc": {
myCounter.increment(String.format("%s_pmc", prefix));
break;
}
default:
myCounter.increment(String.format("%s_other", prefix));
}
}
public List<Pair<String, Long>> totalLinksByProvider(final String filterName) throws ScholixException {
final QueryBuilder query = StringUtils.isNoneBlank(filterName)?createLinkProviderQuery(filterName):QueryBuilders.matchAllQuery();
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(query)
.withSearchType(SearchType.DEFAULT)
.withPageable(PageRequest.of(0,10))
.addAggregation(AggregationBuilders.nested("nested", "linkprovider")
.subAggregation(AggregationBuilders.terms("by_map").field("linkprovider.name").size(100).minDocCount(1)))
.build();
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
final SearchHits<Scholix> hits = client.search(searchQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
final Aggregations aggregations = hits.getAggregations();
connectionPool.returnResource(resource);
if(aggregations == null)
return null;
final Aggregation aggByMap = ((ParsedNested) aggregations.asMap().get("nested")).getAggregations().asMap().get("by_map");
return ((ParsedStringTerms) aggByMap).getBuckets()
.stream()
.map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount()))
.collect(Collectors.toList());
}
public List<Pair<String, Long>> totalLinksPublisher(final RelationPrefix prefix, final String filterName) throws ScholixException {
final QueryBuilder query = StringUtils.isNoneBlank(filterName)?createLinkPublisherQuery(prefix,filterName):QueryBuilders.matchAllQuery();
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(query)
.withSearchType(SearchType.DEFAULT)
.withPageable(PageRequest.of(0,10))
.addAggregation(AggregationBuilders.nested("nested", String.format("%s.publisher", prefix ))
.subAggregation(AggregationBuilders.terms("by_map").field(String.format("%s.publisher.name", prefix )).size(100).minDocCount(1)))
.build();
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
final SearchHits<Scholix> hits = client.search(searchQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
final Aggregations aggregations = hits.getAggregations();
connectionPool.returnResource(resource);
if(aggregations == null)
return null;
final Aggregation aggByMap = ((ParsedNested) aggregations.asMap().get("nested")).getAggregations().asMap().get("by_map");
return ((ParsedStringTerms) aggByMap).getBuckets()
.stream()
.map(b -> new ImmutablePair<>(b.getKeyAsString(), b.getDocCount()))
.collect(Collectors.toList());
}
/**
* Links from pid pair.
*
* @param linkProvider the link provider
* @param targetPid the target pid
* @param targetPidType the target pid type
* @param targetPublisher the target publisher
* @param targetType the target type
* @param sourcePid the source pid
* @param sourcePidType the source pid type
* @param sourcePublisher the source publisher
* @param sourceType the source type
* @param harvestedAfter the harvested after
* @param page the page
* @return the pair
* @throws ScholixException the scholix exception
*/
@Timed(value = "scholix.index.request.links", description = "Time taken to request index")
public Pair<Long,List<Scholix>> linksFromPid ( final String linkProvider,
final String targetPid, final String targetPidType, final String targetPublisher,
final String targetType, final String sourcePid, final String sourcePidType,
final String sourcePublisher, final String sourceType, final String harvestedAfter,
final Integer page) throws ScholixException {
if (sourcePid==null && sourcePidType==null && targetPid==null && targetPidType==null && sourcePublisher==null && targetPublisher==null && linkProvider==null)
throw new ScholixException("One of sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider should be not null");
final List<QueryBuilder> queries = new ArrayList<>();
if (StringUtils.isNoneBlank(linkProvider)) {
myCounter.increment("linkProvider");
queries.add(createLinkProviderQuery(linkProvider));
}
if (StringUtils.isNoneBlank(targetPid)) {
myCounter.increment("targetPid");
queries.add(createPidValueQuery(RelationPrefix.target, targetPid));
}
if (StringUtils.isNoneBlank(sourcePid)) {
myCounter.increment("sourcePid");
queries.add(createPidValueQuery(RelationPrefix.source, sourcePid));
}
if (StringUtils.isNoneBlank(targetPidType)) {
assert targetPidType != null;
incrementPidCounter(RelationPrefix.target,targetPidType);
queries.add(createPidTypeQuery(RelationPrefix.target, targetPidType));
}
if (StringUtils.isNoneBlank(sourcePidType)) {
assert sourcePidType != null;
incrementPidCounter(RelationPrefix.source,sourcePidType);
queries.add(createPidTypeQuery(RelationPrefix.source, sourcePidType));
}
if (StringUtils.isNoneBlank(targetType)) {
if ("dataset".equalsIgnoreCase(targetType) || "publication".equalsIgnoreCase(targetType))
myCounter.increment(String.format("targetType_%s", targetType));
queries.add(createObjectTypeQuery(RelationPrefix.target, targetType));
}
if (StringUtils.isNoneBlank(sourceType)) {
if ("dataset".equalsIgnoreCase(sourceType) || "publication".equalsIgnoreCase(sourceType)) {
myCounter.increment(String.format("sourceType_%s", sourceType));
}
queries.add(createObjectTypeQuery(RelationPrefix.source, sourceType));
}
if (StringUtils.isNoneBlank(targetPublisher)) {
myCounter.increment("targetPublisher");
queries.add(createLinkPublisherQuery(RelationPrefix.target,targetPublisher));
}
QueryBuilder result = createFinalQuery(queries);
NativeSearchQuery finalQuery = new NativeSearchQueryBuilder()
.withQuery(result)
.withPageable(PageRequest.of(page,10))
.build();
Pair<RestHighLevelClient, ElasticsearchRestTemplate> resource = connectionPool.getResource();
ElasticsearchRestTemplate client = resource.getValue();
long tt = client.count(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
SearchHits<Scholix> scholixRes = client.search(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName()));
connectionPool.returnResource(resource);
return new ImmutablePair<>(tt,scholixRes.stream().map(SearchHit::getContent).collect(Collectors.toList()));
}
}

13
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/metrics/RequestCounter.java Normal file
View File

@ -0,0 +1,13 @@
package eu.dnetlib.scholix.api.metrics;
import eu.dnetlib.common.metrics.MetricInfo;
import org.springframework.stereotype.Component;
@Component("scholexplorer_test")
public class RequestCounter implements MetricInfo {
@Override
public double obtainValue() {
return 0L;
}
}

26
apps/scholexplorer-api/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,26 @@
spring.main.banner-mode = console
logging.level.root = INFO
dhp.swagger.api.host = localhost:8080
#dhp.swagger.api.host = api.scholexplorer.openaire.eu
dhp.swagger.api.basePath = /
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml
#
#spring.thymeleaf.cache=false
#
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
management.metrics.distribution.percentiles-histogram.http.server.requests=false
management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999
scholix.elastic.clusterNodes = localhost:9200
scholix.elastic.indexName = dli_shadow_scholix
scholix.elastic.socketTimeout = 60000
scholix.elastic.connectionTimeout= 60000

BIN
apps/scholexplorer-api/src/main/resources/static/logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 43 KiB

135
apps/scholexplorer-api/src/main/resources/static/new_swagger.html Normal file
View File

@ -0,0 +1,135 @@
<!-- HTML for static distribution bundle build -->
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Custom Swagger UI</title>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/swagger-ui.css" >
<link rel="icon" type="image/png" href="./swagger-ui/swagger-favicon-32x32.png" sizes="32x32" />
<link rel="icon" type="image/png" href="./swagger-ui/swagger-favicon-16x16.png" sizes="16x16" />
<style>
html
{
box-sizing: border-box;
overflow: -moz-scrollbars-vertical;
overflow-y: scroll;
}
*,
*:before,
*:after
{
box-sizing: inherit;
}
body
{
margin:0;
background: #fafafa;
}
.top-nav-bar{
position: fixed;
top: 0;
z-index: 99;
width: 100%;
overflow: hidden;
background: #333;
padding: 15px;
}
.nav-bar-icon{
margin-top: 1px;
float: left;
display: block;
text-decoration: none;
}
.nav-bar-title{
float: left;
display: block;
text-decoration: none;
margin-top: 7px;
margin-left: 10px;
font-size: 18px;
color: #ffffff;
font-family: sans-serif;
}
.nav-bar-select{
width: 30%;
float: right;
font-family: sans-serif;
display: inline-block;
cursor: pointer;
padding: 10px 15px;
outline: 0;
border-radius: 2px;
border: none;
background: #fafafa;
color: #3b4151;
appearance: none;
-webkit-appearance: none;
-moz-appearance: none;
}
select.classic {
background-image: linear-gradient(45deg, transparent 50%, #111 50%), linear-gradient(135deg, #111 50%, transparent 50%);
background-position: calc(100% - 20px) calc(1em + 2px), calc(100% - 15px) calc(1em + 2px), 100% 0;
background-size: 5px 5px, 5px 5px, 3.5em 3.5em;
background-repeat: no-repeat;
}
</style>
</head>
<body>
<div class="top-nav-bar">
<a class="nav-bar-icon"><img src="swagger-favicon-32x32.png"></a>
<a class="nav-bar-title"><b>X name</b></a>
<select class="classic nav-bar-select" id="service-selector" onchange="changeSwaggerUI()">
<option value="./swagger.json">X service</option>
</select>
</div>
<div style="margin-top: 100px" id="swagger-ui"></div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/swagger-ui-bundle.js"> </script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/swagger-ui-standalone-preset.js"> </script>
<script>
function changeSwaggerUI(){
let selected_service_swaggerURL = document.getElementById("service-selector").value;
loadUI(selected_service_swaggerURL);
}
function loadUI(swaggerJsonURL){
// Begin Swagger UI call region
const ui = SwaggerUIBundle({
url: swaggerJsonURL,
validatorUrl: "",
dom_id: '#swagger-ui',
deepLinking: true,
docExpansion: 'none',
presets: [
SwaggerUIBundle.presets.apis,
SwaggerUIStandalonePreset
],
plugins: [
CustomTopbarPlugin
],
layout: "StandaloneLayout"
});
// End Swagger UI call region
window.ui = ui
}
function CustomTopbarPlugin() {
// this plugin overrides the Topbar component to return nothing
return {
components: {
Topbar: () => null
}
}
}
window.onload = function() {
loadUI("./swagger-ui/springfox.js?v=3.0.0");
}
</script>
</body>
</html>

18
apps/scholexplorer-api/src/test/java/eu/dnetlib/scholix/test/QueryTest.java Normal file
View File

@ -0,0 +1,18 @@
package eu.dnetlib.scholix.test;
import org.junit.jupiter.api.Test;
public class QueryTest {
@Test
public void featureToTest() {
}
}

2
cmd-line-apps/dhp-broker-client/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>cmd-line-apps</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
cmd-line-apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-broker-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-openaire-broker-common/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

14
pom.xml
View File

@ -10,7 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>
@ -28,7 +28,7 @@
<module>libs</module>
<module>apps</module>
<module>cmd-line-apps</module>
</modules>
</modules>
<issueManagement>
<system>Redmine</system>
@ -142,6 +142,13 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.11.1</version>
</dependency>
</dependencies>
<dependencyManagement>
@ -233,6 +240,7 @@
<version>3.0.0</version>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -390,7 +398,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
<java.version>1.8</java.version>
<dhp-schemas-version>2.3.6</dhp-schemas-version>
<dhp-schemas-version>2.11.33</dhp-schemas-version>
<apache.solr.version>7.1.0</apache.solr.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version>
<springfox-version>2.8.0</springfox-version>