diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java index 500691c6..2ce535e4 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/MainApplication.java @@ -2,10 +2,7 @@ package eu.dnetlib.scholix.api; import eu.dnetlib.common.app.AbstractDnetApp; import io.micrometer.core.aop.TimedAspect; -import io.micrometer.core.instrument.DistributionSummary; -import io.micrometer.core.instrument.Meter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.*; import io.micrometer.core.instrument.config.MeterFilter; import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import org.springframework.beans.factory.annotation.Value; @@ -23,6 +20,8 @@ import springfox.documentation.spring.web.plugins.Docket; import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.time.Duration; +import java.util.Arrays; +import java.util.List; @SpringBootApplication @EnableSwagger2 @@ -31,6 +30,12 @@ import java.time.Duration; @ComponentScan(basePackages = "eu.dnetlib") public class MainApplication extends AbstractDnetApp { + private double scale = 1000000000; + + private double[] histogramValues =new double[]{.005 * scale, .01 * scale, .25 * scale, .5 * scale, .75 * scale, scale, 2.5 * scale, 5.0 * scale, 7.5 * scale, 10.0 * scale}; + + + @Value("${dhp.swagger.api.host}") private String swaggetHost; @@ -48,10 +53,17 @@ public class MainApplication extends AbstractDnetApp { } + + @Bean + public TaggedCounter myCounter(MeterRegistry meterRegistry) { + + return new TaggedCounter("scholixLinkCounter", "links",meterRegistry); + } + @Bean public TimedAspect timedAspect(MeterRegistry meterRegistry) { - double scale = 1000000; + MeterFilter mf = new MeterFilter() { @Override public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { @@ -60,10 +72,7 @@ public class MainApplication extends AbstractDnetApp { return DistributionStatisticConfig.builder() .percentiles(0.20, 0.50, 0.75,0.95) .percentilesHistogram(false) - .serviceLevelObjectives(5.0*scale, 10.0*scale, 250*scale, 500*scale, 750*scale ,1000.0*scale,2500.0*scale,5000.0*scale,7500.0*scale,10000.0*scale ) - -// .minimumExpectedValue(0.008) -//// .maximumExpectedValue(1000000.0) + .serviceLevelObjectives( histogramValues) .build() .merge(config); } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java index 2abcd0f3..078e489b 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/RestClientConfig.java @@ -1,6 +1,8 @@ package eu.dnetlib.scholix.api; +import eu.dnetlib.scholix.api.index.ElasticSearchProperties; import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.elasticsearch.client.ClientConfiguration; @@ -12,21 +14,18 @@ import java.time.Duration; @Configuration public class RestClientConfig extends AbstractElasticsearchConfiguration { + @Autowired + private ElasticSearchProperties elasticSearchProperties; + @Override @Bean - public RestHighLevelClient elasticsearchClient() { + public RestHighLevelClient elasticsearchClient() { final ClientConfiguration clientConfiguration = ClientConfiguration.builder() - .connectedTo("localhost:9200") - .withSocketTimeout(Duration.ofSeconds(60)) + .connectedTo(elasticSearchProperties.getClusterNodes().split(",")) + .withConnectTimeout(elasticSearchProperties.getConnectionTimeout()) + .withSocketTimeout(elasticSearchProperties.getSocketTimeout()) .build(); - return RestClients.create(clientConfiguration).rest(); - } - - - - - } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIVersion.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIVersion.java new file mode 100644 index 00000000..265d33eb --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixAPIVersion.java @@ -0,0 +1,8 @@ +package eu.dnetlib.scholix.api; + +public enum ScholixAPIVersion { + + V1, + V2 + +} diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixException.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixException.java new file mode 100644 index 00000000..21d63b71 --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/ScholixException.java @@ -0,0 +1,35 @@ +package eu.dnetlib.scholix.api; + + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) +public class ScholixException extends Exception{ + + private static final long serialVersionUID = -3414428892721711308L; + + + public ScholixException() { + super(); + } + + public ScholixException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public ScholixException(String message, Throwable cause) { + super(message, cause); + } + + public ScholixException(String message) { + super(message); + } + + public ScholixException(Throwable cause) { + super(cause); + } + + +} diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/TaggedCounter.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/TaggedCounter.java new file mode 100644 index 00000000..8cf48fba --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/TaggedCounter.java @@ -0,0 +1,33 @@ +package eu.dnetlib.scholix.api; + +import io.micrometer.core.instrument.Counter; + +import io.micrometer.core.instrument.MeterRegistry; +import java.util.HashMap; +import java.util.Map; + + +public class TaggedCounter { + + private String name; + private String tagName; + private MeterRegistry registry; + private Map counters = new HashMap<>(); + + + public TaggedCounter(String name, String tagName, MeterRegistry registry) { + this.name = name; + this.tagName = tagName; + this.registry = registry; + } + + + public void increment(String tagValue){ + Counter counter = counters.get(tagValue); + if(counter == null) { + counter = Counter.builder(name).tags(tagName, tagValue).register(registry); + counters.put(tagValue, counter); + } + counter.increment(); + } +} \ No newline at end of file diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java index e6fd4f84..116a1e77 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/ScholixControllerV2.java @@ -3,13 +3,17 @@ package eu.dnetlib.scholix.api.controller; import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.dhp.schema.sx.scholix.ScholixIdentifier; +import eu.dnetlib.scholix.api.ScholixAPIVersion; +import eu.dnetlib.scholix.api.index.ScholixIndexManager; import eu.dnetlib.scholix.api.model.v2.PageResultType; +import eu.dnetlib.scholix.api.model.v2.ScholixType; import io.micrometer.core.annotation.Timed; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; @@ -30,6 +34,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; @RestController @RequestMapping("/v2") @@ -39,7 +45,7 @@ import java.util.Collections; public class ScholixControllerV2 extends AbstractDnetController { @Autowired - ElasticsearchOperations elasticsearchTemplate; + private ScholixIndexManager manager; @@ -72,23 +78,16 @@ public class ScholixControllerV2 extends AbstractDnetController { description = "select page of result") Integer page) throws Exception { - NestedQueryBuilder nb = new NestedQueryBuilder("source.identifier", new TermQueryBuilder("source.identifier.schema", "pdb"), ScoreMode.None); + final int currentPage = page!= null? page : 0; - - NativeSearchQuery doiquery = new NativeSearchQueryBuilder() - .withQuery(nb) - .withPageable(PageRequest.of(1,10)) - .build(); - - long tt = elasticsearchTemplate.count(doiquery, Scholix.class, IndexCoordinates.of("dli_scholix")); - System.out.println(tt); - - SearchHits result = elasticsearchTemplate.search(doiquery, Scholix.class, IndexCoordinates.of("dli_scholix")); - + System.out.println(currentPage); + Pair> scholixResult = manager.linksFromPid(ScholixAPIVersion.V2, linkProvider, targetPid, targetPidType, targetPublisher, targetType, sourcePid, sourcePidType, sourcePublisher, sourceType, harvestedAfter, currentPage); final PageResultType pageResult = new PageResultType(); - pageResult.setTotalPages((int) tt ); + pageResult.setTotalPages(scholixResult.getLeft().intValue() / 10); + pageResult.setTotalLinks(scholixResult.getLeft().intValue()); + pageResult.setResult(scholixResult.getRight().stream().map(ScholixType::fromScholix).collect(Collectors.toList())); return pageResult; } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java new file mode 100644 index 00000000..38e7016e --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ElasticSearchProperties.java @@ -0,0 +1,103 @@ +package eu.dnetlib.scholix.api.index; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import javax.validation.constraints.NotNull; + +/** + * The type Elastic search properties. + */ +@Component("elasticSearchProperties") +@ConfigurationProperties(prefix = "scholix.elastic") +public class ElasticSearchProperties { + + @NotNull + private String clusterNodes; + @NotNull + private String indexName; + @NotNull + private long connectionTimeout; + @NotNull + private long socketTimeout; + + /** + * Gets cluster nodes. + * + * @return the cluster nodes + */ + public String getClusterNodes() { + return clusterNodes; + } + + /** + * Sets cluster nodes. + * + * @param clusterNodes the cluster nodes + * @return the cluster nodes + */ + public ElasticSearchProperties setClusterNodes(String clusterNodes) { + this.clusterNodes = clusterNodes; + return this; + } + + /** + * Gets index name. + * + * @return the index name + */ + public String getIndexName() { + return indexName; + } + + /** + * Sets index name. + * + * @param indexName the index name + * @return the index name + */ + public ElasticSearchProperties setIndexName(String indexName) { + this.indexName = indexName; + return this; + } + + /** + * Gets connection timeout. + * + * @return the connection timeout + */ + public long getConnectionTimeout() { + return connectionTimeout; + } + + /** + * Sets connection timeout. + * + * @param connectionTimeout the connection timeout + * @return the connection timeout + */ + public ElasticSearchProperties setConnectionTimeout(long connectionTimeout) { + this.connectionTimeout = connectionTimeout; + return this; + } + + /** + * Gets socket timeout. + * + * @return the socket timeout + */ + public long getSocketTimeout() { + return socketTimeout; + } + + /** + * Sets socket timeout. + * + * @param socketTimeout the socket timeout + * @return the socket timeout + */ + public ElasticSearchProperties setSocketTimeout(long socketTimeout) { + this.socketTimeout = socketTimeout; + return this; + } +} diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java new file mode 100644 index 00000000..daefcab3 --- /dev/null +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/index/ScholixIndexManager.java @@ -0,0 +1,222 @@ +package eu.dnetlib.scholix.api.index; + + +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; +import eu.dnetlib.scholix.api.ScholixAPIVersion; +import eu.dnetlib.scholix.api.ScholixException; +import eu.dnetlib.scholix.api.TaggedCounter; + +import io.micrometer.core.annotation.Timed; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.SearchHit; +import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; +import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * The type Scholix index manager. + */ +@Component +public class ScholixIndexManager { + + /** + * The Elastic search properties. + */ + @Autowired + ElasticSearchProperties elasticSearchProperties; + + /** + * The Elasticsearch template. + */ + @Autowired + ElasticsearchOperations elasticsearchTemplate; + + /** + * The My counter. + */ + @Autowired + TaggedCounter myCounter; + + + /** + * The enum Pid type prefix. + */ + enum pidTypePrefix { + /** + * Source pid type prefix. + */ + source, + /** + * Target pid type prefix. + */ + target + } + + + private QueryBuilder createObjectTypeQuery(final pidTypePrefix prefix, final String objectType ) throws ScholixException{ + if (prefix == null){ + throw new ScholixException("prefix cannot be null"); + } + return new NestedQueryBuilder(String.format("%s", prefix), new TermQueryBuilder(String.format("%s.objectType",prefix), objectType), ScoreMode.None); + } + + + private QueryBuilder createPidTypeQuery(final pidTypePrefix prefix, final String pidTypeValue ) throws ScholixException{ + if (prefix == null){ + throw new ScholixException("prefix cannot be null"); + } + return new NestedQueryBuilder(String.format("%s.identifier", prefix), new TermQueryBuilder(String.format("%s.identifier.schema",prefix), pidTypeValue), ScoreMode.None); + } + + + private QueryBuilder createPidValueQuery(final pidTypePrefix prefix, final String pidValue ) throws ScholixException{ + if (prefix == null){ + throw new ScholixException("prefix cannot be null"); + } + return new NestedQueryBuilder(String.format("%s.identifier", prefix), new TermQueryBuilder(String.format("%s.identifier.identifier",prefix), pidValue), ScoreMode.None); + } + + + private QueryBuilder createFinalQuery(final List queries) throws ScholixException{ + + if (queries == null || queries.isEmpty()) + throw new ScholixException("the list of queries must be not empty"); + + + if (queries.size() ==1) { + return queries.get(0); + } + + else { + final BoolQueryBuilder b = new BoolQueryBuilder(); + b.must().addAll(queries); + + return b; + } + + } + + private void incrementPidCounter(pidTypePrefix prefix, String value) { + + + switch (value.toLowerCase()){ + case "doi": { + myCounter.increment(String.format("%s_doi", prefix)); + break; + } + case "pmc": { + myCounter.increment(String.format("%s_pmc", prefix)); + break; + } + default: + myCounter.increment(String.format("%s_other", prefix)); + + } + + } + + + /** + * Links from pid pair. + * + * @param outputVersion the output version + * @param linkProvider the link provider + * @param targetPid the target pid + * @param targetPidType the target pid type + * @param targetPublisher the target publisher + * @param targetType the target type + * @param sourcePid the source pid + * @param sourcePidType the source pid type + * @param sourcePublisher the source publisher + * @param sourceType the source type + * @param harvestedAfter the harvested after + * @param page the page + * @return the pair + * @throws ScholixException the scholix exception + */ + @Timed(value = "scholix.index.request.links", description = "Time taken to request index") + public Pair> linksFromPid (final ScholixAPIVersion outputVersion, final String linkProvider, + final String targetPid, final String targetPidType, final String targetPublisher, + final String targetType, final String sourcePid, final String sourcePidType, + final String sourcePublisher, final String sourceType, final String harvestedAfter, + final Integer page) throws ScholixException { + + if(outputVersion == null) + throw new ScholixException("Error outputVersion not be empty"); + + if (sourcePid==null && sourcePidType==null && targetPid==null && targetPidType==null && sourcePublisher==null && targetPublisher==null && linkProvider==null) + throw new ScholixException("One of sourcePid, targetPid, sourcePublisher, targetPublisher, linkProvider should be not null"); + + final List queries = new ArrayList<>(); + + if (StringUtils.isNoneBlank(targetPid)) { + myCounter.increment("targetPid"); + queries.add(createPidValueQuery(pidTypePrefix.target, targetPid)); + } + if (StringUtils.isNoneBlank(sourcePid)) { + myCounter.increment("sourcePid"); + queries.add(createPidValueQuery(pidTypePrefix.source, sourcePid)); + } + + if (StringUtils.isNoneBlank(targetPidType)) { + assert targetPidType != null; + incrementPidCounter(pidTypePrefix.target,targetPidType); + queries.add(createPidTypeQuery(pidTypePrefix.target, targetPidType)); + } + if (StringUtils.isNoneBlank(sourcePidType)) { + assert sourcePidType != null; + incrementPidCounter(pidTypePrefix.source,sourcePidType); + queries.add(createPidTypeQuery(pidTypePrefix.source, sourcePidType)); + } + + if (StringUtils.isNoneBlank(targetType)) { + if ("dataset".equalsIgnoreCase(targetType) || "publication".equalsIgnoreCase(targetType)) + myCounter.increment(String.format("targetType_%s", targetType)); + queries.add(createObjectTypeQuery(pidTypePrefix.target, targetType)); + } + + if (StringUtils.isNoneBlank(sourceType)) { + if ("dataset".equalsIgnoreCase(sourceType) || "publication".equalsIgnoreCase(sourceType)) { + myCounter.increment(String.format("sourceType_%s", sourceType)); + } + queries.add(createObjectTypeQuery(pidTypePrefix.source, sourceType)); + } + + QueryBuilder result = createFinalQuery(queries); + + NativeSearchQuery finalQuery = new NativeSearchQueryBuilder() + .withQuery(result) + .withPageable(PageRequest.of(page,10)) + .build(); + + long tt = elasticsearchTemplate.count(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + System.out.println(tt); + + SearchHits scholixRes = elasticsearchTemplate.search(finalQuery, Scholix.class, IndexCoordinates.of(elasticSearchProperties.getIndexName())); + + System.out.println("SIZE OF HITS ->"+scholixRes.getSearchHits().size()); + + return new ImmutablePair<>(tt,scholixRes.stream().map(SearchHit::getContent).collect(Collectors.toList())); + } + + + + + +} diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v1/ScholixV1.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v1/ScholixV1.java index b12e453c..3e9c5978 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v1/ScholixV1.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v1/ScholixV1.java @@ -1,14 +1,16 @@ package eu.dnetlib.scholix.api.model.v1; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; import io.swagger.v3.oas.annotations.media.Schema; import javax.validation.constraints.NotBlank; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Objects; -public class ScholixV1 { +public class ScholixV1 implements Serializable { @JsonProperty("linkProvider") private List linkProvider; @@ -156,4 +158,9 @@ public class ScholixV1 { } return o.toString().replace("\n", "\n "); } + + + public static ScholixV1 fromScholix(Scholix input) { + return new ScholixV1(); + } } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/LinkProviderType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/LinkProviderType.java index 4957b4f7..d4a5697e 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/LinkProviderType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/LinkProviderType.java @@ -1,6 +1,7 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixEntityId; import io.swagger.v3.oas.annotations.media.Schema; import javax.validation.constraints.NotBlank; diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/RelationshipType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/RelationshipType.java index 87166b25..6651c1b9 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/RelationshipType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/RelationshipType.java @@ -1,9 +1,15 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixRelationship; import io.swagger.v3.oas.annotations.media.Schema; import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * The type Relationship type. @@ -21,6 +27,14 @@ public class RelationshipType { private String subTypeSchema; + private static Map relationMapping = Stream.of(new String[][] { + { "issupplementto", "IsSupplementTo" }, + { "issupplementedby", "IsSupplementedBy" }, + { "references", "References" }, + { "isreferencedby", "IsReferencedBy" }, + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + + /** * Gets The relationship type chosen from a Scholix controlled vocabulary * @@ -83,4 +97,13 @@ public class RelationshipType { this.subTypeSchema = subTypeSchema; return this; } + + public static RelationshipType fromScholixRelationship(ScholixRelationship inputRels) { + + return new RelationshipType() + .setName(relationMapping.getOrDefault(inputRels.getName(), "IsRelatedTo")) + .setSubType(inputRels.getName()) + .setSubTypeSchema(inputRels.getSchema()); + + } } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixCreatorType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixCreatorType.java index 61902ade..0f43c77f 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixCreatorType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixCreatorType.java @@ -1,11 +1,13 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixEntityId; import io.swagger.v3.oas.annotations.media.Schema; import javax.validation.constraints.NotBlank; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * The type Scholix creator type. @@ -63,4 +65,22 @@ public class ScholixCreatorType { this.name = name; return this; } + + public static ScholixCreatorType fromScholixEntityId(final ScholixEntityId inputCreator) { + if (inputCreator == null) + return null; + ScholixCreatorType instance = new ScholixCreatorType().setName(inputCreator.getName()); + + + if (inputCreator.getIdentifiers()!= null && inputCreator.getIdentifiers().size()>0) + instance.setIdentifier(inputCreator.getIdentifiers() + .stream() + .map(i ->new ScholixIdentifierType() + .setId(i.getIdentifier()) + .setIdScheme(i.getSchema())) + .collect(Collectors.toList())); + return instance; + + + } } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixIdentifierType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixIdentifierType.java index b023c772..662b9a4f 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixIdentifierType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixIdentifierType.java @@ -1,6 +1,7 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixIdentifier; import io.swagger.v3.oas.annotations.media.Schema; import javax.validation.constraints.NotBlank; @@ -88,5 +89,17 @@ public class ScholixIdentifierType { } + public static ScholixIdentifierType fromScholixIdentifier(ScholixIdentifier input) { + if (input== null) + return null; + final ScholixIdentifierType instance = new ScholixIdentifierType(); + + instance.setId(input.getIdentifier()); + instance.setIdScheme(input.getSchema()); + instance.setIdURL(input.getUrl()); + return instance; + } + + } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixItemType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixItemType.java index b137a9ac..c0fdcc24 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixItemType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixItemType.java @@ -1,9 +1,11 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixResource; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * The type Scholix item type. @@ -26,7 +28,7 @@ public class ScholixItemType { private String publicationDate; @JsonProperty("Publisher") - private List publisher = new ArrayList<>(); + private List publisher = new ArrayList<>(); /** @@ -134,7 +136,7 @@ public class ScholixItemType { * * @return the publisher */ - public List getPublisher() { + public List getPublisher() { return publisher; } @@ -144,8 +146,43 @@ public class ScholixItemType { * @param publisher the publisher * @return the publisher */ - public ScholixItemType setPublisher(List publisher) { + public ScholixItemType setPublisher(List publisher) { this.publisher = publisher; return this; } + + + + public static ScholixItemType fromScholixResource(final ScholixResource input) { + if (input == null) + return null; + final ScholixItemType instance = new ScholixItemType(); + instance.setType("publication".equalsIgnoreCase(input.getObjectType())?"literature": "dataset"); + instance.setTitle(input.getTitle()); + + if (input.getIdentifier()!= null) + instance.setIdentifier(input.getIdentifier() + .stream() + .map(ScholixIdentifierType::fromScholixIdentifier ) + .collect(Collectors.toList()) + ); + + + if (input.getPublisher()!= null) { + instance.setPublisher( + input.getPublisher().stream() + .map(ScholixLinkProviderType::fromScholixEntityId) + .collect(Collectors.toList()) + ); + } + + instance.setPublicationDate(input.getPublicationDate()); + if(input.getCreator()!=null) + instance.setCreator(input.getCreator() + .stream() + .map(ScholixCreatorType::fromScholixEntityId) + .collect(Collectors.toList())); + + return instance; + } } diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixLinkProviderType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixLinkProviderType.java index c14093d3..2fcab411 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixLinkProviderType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixLinkProviderType.java @@ -1,6 +1,7 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.ScholixEntityId; import io.swagger.v3.oas.annotations.media.Schema; import javax.validation.constraints.NotBlank; @@ -8,6 +9,7 @@ import javax.validation.constraints.Size; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * The type Scholix link provider type. @@ -64,4 +66,23 @@ public class ScholixLinkProviderType { this.name = name; return this; } + + + public static ScholixLinkProviderType fromScholixEntityId(final ScholixEntityId provider) { + if (provider == null) + return null; + ScholixLinkProviderType instance = new ScholixLinkProviderType().setName(provider.getName()); + + + if (provider.getIdentifiers()!= null && provider.getIdentifiers().size()>0) + instance.setIdentifier(provider.getIdentifiers() + .stream() + .map(i ->new ScholixIdentifierType() + .setId(i.getIdentifier()) + .setIdScheme(i.getSchema())) + .collect(Collectors.toList())); + return instance; + + + } } \ No newline at end of file diff --git a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixType.java b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixType.java index 3c994c96..9deaa803 100644 --- a/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixType.java +++ b/apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/model/v2/ScholixType.java @@ -1,11 +1,12 @@ package eu.dnetlib.scholix.api.model.v2; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.dnetlib.dhp.schema.sx.scholix.Scholix; import io.swagger.v3.oas.annotations.media.Schema; - import javax.validation.constraints.NotBlank; import java.io.Serializable; import java.util.List; +import java.util.stream.Collectors; /** * The type Scholix type. @@ -33,7 +34,7 @@ public class ScholixType implements Serializable { private String licenseURL ; @JsonProperty("LinkProvider") - private List linkProvider ; + private List linkProvider ; @JsonProperty("LinkPublicationDate") private String linkPublicationDate ; @@ -49,6 +50,11 @@ public class ScholixType implements Serializable { return relationshipType; } + /** + * Sets relationship type. + * + * @param relationshipType the relationship type + */ public void setRelationshipType(RelationshipType relationshipType) { this.relationshipType = relationshipType; } @@ -114,7 +120,6 @@ public class ScholixType implements Serializable { * * @return the license url */ - @Schema(description = "The URL of the license for the Scholix Link Information Package") public String getLicenseURL() { return licenseURL; @@ -135,7 +140,7 @@ public class ScholixType implements Serializable { * @return the link provider */ @Schema(description = "The source(s) of this Link Information Package") - public List getLinkProvider() { + public List getLinkProvider() { return linkProvider; } @@ -144,7 +149,7 @@ public class ScholixType implements Serializable { * * @param linkProvider the link provider */ - public void setLinkProvider(List linkProvider) { + public void setLinkProvider(List linkProvider) { this.linkProvider = linkProvider; } @@ -167,4 +172,25 @@ public class ScholixType implements Serializable { public void setLinkPublicationDate(String linkPublicationDate) { this.linkPublicationDate = linkPublicationDate; } + + + + public static ScholixType fromScholix(Scholix input) { + final ScholixType instance = new ScholixType(); + instance.setLinkPublicationDate(input.getPublicationDate()); + instance.setHarvestDate(input.getPublicationDate()); + instance.setRelationshipType(RelationshipType.fromScholixRelationship(input.getRelationship())); + + if(input.getLinkprovider()!= null && input.getLinkprovider().size()>0) + instance.setLinkProvider(input.getLinkprovider() + .stream() + .map(ScholixLinkProviderType::fromScholixEntityId) + .collect(Collectors.toList()) + ); + + + instance.setSource(ScholixItemType.fromScholixResource(input.getSource())); + instance.setTarget(ScholixItemType.fromScholixResource(input.getTarget())); + return instance; + } } diff --git a/apps/scholexplorer-api/src/main/resources/application.properties b/apps/scholexplorer-api/src/main/resources/application.properties index 6bcb8ef7..beb6bf51 100644 --- a/apps/scholexplorer-api/src/main/resources/application.properties +++ b/apps/scholexplorer-api/src/main/resources/application.properties @@ -20,4 +20,7 @@ management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, - +scholix.elastic.clusterNodes = localhost:9200 +scholix.elastic.indexName = dli_scholix +scholix.elastic.socketTimeout = 60000 +scholix.elastic.connectionTimeout= 60000 diff --git a/pom.xml b/pom.xml index 8f02c9d7..34ea20d2 100644 --- a/pom.xml +++ b/pom.xml @@ -233,12 +233,7 @@ 3.0.0 - - org.springframework.data - spring-data-elasticsearch - 4.3.1 - - + org.apache.hadoop