diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java index b7be9056..f60e93a5 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java @@ -31,6 +31,7 @@ public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchC @Override @Bean public RestHighLevelClient elasticsearchClient() { + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() .connectedTo(elasticSearchProperties.getClusterNodes().split(",")) .withConnectTimeout(elasticSearchProperties.getConnectionTimeout()) diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/ElasticSearchQueryUtils.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/ElasticSearchQueryUtils.java index d0b7ebd1..68cf83bf 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/ElasticSearchQueryUtils.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/ElasticSearchQueryUtils.java @@ -8,7 +8,7 @@ import org.apache.commons.lang3.math.NumberUtils; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery; +import org.elasticsearch.index.search.MatchQueryParser.ZeroTermsQuery; import eu.dnetlib.broker.common.utils.DateParser; diff --git a/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/LiteratureBrokerServiceApplicationTests.java b/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/LiteratureBrokerServiceApplicationTests.java index 427e7ca6..42d88dd0 100644 --- a/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/LiteratureBrokerServiceApplicationTests.java +++ b/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/LiteratureBrokerServiceApplicationTests.java @@ -1,15 +1,12 @@ package eu.dnetlib.broker; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -@Ignore -@RunWith(SpringRunner.class) +@Disabled @SpringBootTest -public class LiteratureBrokerServiceApplicationTests { +class LiteratureBrokerServiceApplicationTests { @Test public void contextLoads() {} diff --git a/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/integration/ScrollTest.java b/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/integration/ScrollTest.java index 32d7931c..00bca9ff 100644 --- a/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/integration/ScrollTest.java +++ b/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/integration/ScrollTest.java @@ -1,7 +1,7 @@ package eu.dnetlib.broker.integration; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.web.client.RestTemplate; import eu.dnetlib.broker.objects.OaBrokerEventPayload; @@ -16,7 +16,7 @@ public class ScrollTest { private static final String subscriptionId = "sub-c9767c84-3597-462b-803b-2d3e09de44c4"; @Test - @Ignore + @Disabled public void testScroll() { int total = 0; diff --git a/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/tests/OpenaireQueriesTest.java b/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/tests/OpenaireQueriesTest.java index 18982599..0f4dafb5 100644 --- a/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/tests/OpenaireQueriesTest.java +++ b/apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/tests/OpenaireQueriesTest.java @@ -6,11 +6,10 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; import org.springframework.data.domain.PageRequest; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; @@ -20,7 +19,6 @@ import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import eu.dnetlib.broker.common.elasticsearch.Event; import eu.dnetlib.broker.objects.OaBrokerEventPayload; @@ -29,8 +27,8 @@ import eu.dnetlib.broker.openaire.AdvQueryObject; import eu.dnetlib.broker.openaire.ElasticSearchQueryUtils; import eu.dnetlib.broker.openaire.Range; -@Ignore -@RunWith(SpringJUnit4ClassRunner.class) +@Disabled +@SpringBootTest @ContextConfiguration(locations = { "classpath:/applicationContext-test-queries.xml" }) @@ -41,13 +39,8 @@ public class OpenaireQueriesTest { @Autowired private ElasticsearchOperations esOperations; - @Before - public void setUp() throws Exception { - - } - @Test - @Ignore + @Disabled public void testAdvancedShowEvents() { final AdvQueryObject qObj = new AdvQueryObject(); diff --git a/apps/dhp-broker-public-application/pom.xml b/apps/dhp-broker-public-application/pom.xml index 7fd0b06e..257f05d7 100644 --- a/apps/dhp-broker-public-application/pom.xml +++ b/apps/dhp-broker-public-application/pom.xml @@ -25,23 +25,7 @@ org.apache.hadoop hadoop-client - 2.6.0-cdh5.9.2 - - - org.slf4j - slf4j-log4j12 - - - javax.servlet - servlet-api - - - com.google.guava - guava - - - diff --git a/apps/dhp-broker-public-application/src/test/java/eu/dnetlib/broker/oa/integration/ScrollTest.java b/apps/dhp-broker-public-application/src/test/java/eu/dnetlib/broker/oa/integration/ScrollTest.java index e8e60941..c0394fc1 100644 --- a/apps/dhp-broker-public-application/src/test/java/eu/dnetlib/broker/oa/integration/ScrollTest.java +++ b/apps/dhp-broker-public-application/src/test/java/eu/dnetlib/broker/oa/integration/ScrollTest.java @@ -1,7 +1,7 @@ package eu.dnetlib.broker.oa.integration; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.web.client.RestTemplate; import eu.dnetlib.broker.api.ShortEventMessage; @@ -18,7 +18,7 @@ public class ScrollTest { public class TestScrollPage extends ScrollPage {} @Test - @Ignore + @Disabled public void testScroll() { int total = 0; diff --git a/apps/dhp-mdstore-manager/pom.xml b/apps/dhp-mdstore-manager/pom.xml index ac1f0e67..a3d0ccbc 100644 --- a/apps/dhp-mdstore-manager/pom.xml +++ b/apps/dhp-mdstore-manager/pom.xml @@ -42,22 +42,8 @@ org.apache.hadoop hadoop-client - 2.6.0-cdh5.9.2 - - - org.slf4j - slf4j-log4j12 - - - javax.servlet - servlet-api - - - com.google.guava - guava - - + com.twitter parquet-hadoop diff --git a/libs/dnet-broker-apps-common/pom.xml b/libs/dnet-broker-apps-common/pom.xml index dca84130..c4f91eb3 100644 --- a/libs/dnet-broker-apps-common/pom.xml +++ b/libs/dnet-broker-apps-common/pom.xml @@ -131,6 +131,10 @@ test + + org.springframework.boot + spring-boot-configuration-processor + diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Event.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Event.java index f7d4ad5c..b233001d 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Event.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Event.java @@ -7,7 +7,7 @@ import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; -@Document(indexName = "#{elasticSearchProperties.eventsIndexName}", shards = 48, replicas = 1, refreshInterval = "600s") +@Document(indexName = "#{@elasticSearchProperties.getEventsIndexName()}") public class Event { @Id diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Notification.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Notification.java index fa99c941..200b5d5e 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Notification.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/Notification.java @@ -11,7 +11,7 @@ import org.springframework.data.elasticsearch.annotations.FieldType; import eu.dnetlib.broker.common.subscriptions.Subscription; -@Document(indexName = "#{elasticSearchProperties.notificationsIndexName}", shards = 48, replicas = 1, refreshInterval = "600s") +@Document(indexName = "#{@elasticSearchProperties.getNotificationsIndexName()}") public class Notification { @Id diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/properties/ElasticSearchProperties.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/properties/ElasticSearchProperties.java index dbc1f967..fe47fa42 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/properties/ElasticSearchProperties.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/properties/ElasticSearchProperties.java @@ -5,7 +5,7 @@ import javax.validation.constraints.NotNull; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -@Component +@Component("elasticSearchProperties") @ConfigurationProperties(prefix = "lbs.elastic") public class ElasticSearchProperties { diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/MapCondition.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/MapCondition.java index 08125cc0..18ec6fd1 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/MapCondition.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/MapCondition.java @@ -12,7 +12,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery; +import org.elasticsearch.index.search.MatchQueryParser.ZeroTermsQuery; import eu.dnetlib.broker.common.elasticsearch.Event; import eu.dnetlib.broker.common.utils.DateParser; @@ -87,8 +87,9 @@ public class MapCondition { case LIST_STRING: return ((List) val).stream().map(Object::toString).anyMatch(s -> cp.verify(s, operator)); case LIST_DATE: - return ((List) val).stream().map(o -> o instanceof Date ? (Date) o : DateParser.parse(o.toString())) - .anyMatch(s -> cp.verify(s, operator)); + return ((List) val).stream() + .map(o -> o instanceof Date ? (Date) o : DateParser.parse(o.toString())) + .anyMatch(s -> cp.verify(s, operator)); case LIST_INTEGER: return ((List) val).stream().map(Object::toString).map(NumberUtils::toInt).anyMatch(n -> cp.verify(n, operator)); case LIST_FLOAT: @@ -123,8 +124,9 @@ public class MapCondition { private QueryBuilder createSimpleOperator(final ConditionParams p) { if (StringUtils.isNotBlank(p.getValue())) { - return QueryBuilders.matchQuery("map." + field, convertToType(p.getValue())).operator(Operator.AND) - .zeroTermsQuery(ZeroTermsQuery.ALL); + return QueryBuilders.matchQuery("map." + field, convertToType(p.getValue())) + .operator(Operator.AND) + .zeroTermsQuery(ZeroTermsQuery.ALL); } else { return null; } @@ -133,8 +135,9 @@ public class MapCondition { private QueryBuilder createListOperator(final List list) { final BoolQueryBuilder query = QueryBuilders.boolQuery(); for (final ConditionParams p : list) { - query.should(QueryBuilders.matchQuery("map." + field, convertToType(p.getValue())).operator(Operator.AND) - .zeroTermsQuery(ZeroTermsQuery.ALL)); + query.should(QueryBuilders.matchQuery("map." + field, convertToType(p.getValue())) + .operator(Operator.AND) + .zeroTermsQuery(ZeroTermsQuery.ALL)); } return query; } @@ -142,8 +145,8 @@ public class MapCondition { private QueryBuilder createSimpleRangeOperator(final ConditionParams p) { if (StringUtils.isNotBlank(p.getValue()) || StringUtils.isNotBlank(p.getOtherValue())) { return QueryBuilders.rangeQuery("map." + field) - .from(convertToType(p.getValue())) - .to(convertToType(p.getOtherValue())); + .from(convertToType(p.getValue())) + .to(convertToType(p.getOtherValue())); } else { return null; } @@ -154,9 +157,9 @@ public class MapCondition { for (final ConditionParams p : list) { query.should(QueryBuilders - .rangeQuery("map." + field) - .from(convertToType(p.getValue())) - .to(convertToType(p.getOtherValue()))); + .rangeQuery("map." + field) + .from(convertToType(p.getValue())) + .to(convertToType(p.getOtherValue()))); } return query; } diff --git a/pom.xml b/pom.xml index 1771059f..7dc0f6a5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.springframework.boot spring-boot-starter-parent - 2.3.4.RELEASE + 2.5.2 @@ -228,6 +228,31 @@ springfox-swagger-ui 2.9.2 + + + + org.apache.hadoop + hadoop-client + 2.6.0-cdh5.9.2 + + + org.slf4j + slf4j-log4j12 + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + org.apache.hadoop + hadoop-aws + + +