code migration

This commit is contained in:
Michele Artini 2020-07-02 16:08:23 +02:00
parent d00b78dcec
commit 47c8139058
18 changed files with 67 additions and 10062 deletions

View File

@ -116,7 +116,7 @@ public class AdvQueryObject {
.collect(Collectors.toList());
if (StringUtils.isNotBlank(this.datasource)) {
list.add(new MapCondition("target_datasource_name", MapValueType.STRING, ConditionOperator.EXACT,
list.add(new MapCondition("targetDatasourceName", MapValueType.STRING, ConditionOperator.EXACT,
Arrays.asList(new ConditionParams(this.datasource, null))));
}
if (this.trust.isValid()) {
@ -124,16 +124,16 @@ public class AdvQueryObject {
Arrays.asList(new ConditionParams(this.trust.getMin(), this.trust.getMax()))));
}
if (!list_dates.isEmpty()) {
list.add(new MapCondition("target_dateofacceptance", MapValueType.DATE, ConditionOperator.RANGE, list_dates));
list.add(new MapCondition("targetDateofacceptance", MapValueType.DATE, ConditionOperator.RANGE, list_dates));
}
if (!list_titles.isEmpty()) {
list.add(new MapCondition("target_publication_title", MapValueType.LIST_STRING, ConditionOperator.MATCH_ALL, list_titles));
list.add(new MapCondition("targetResultTitle", MapValueType.LIST_STRING, ConditionOperator.MATCH_ALL, list_titles));
}
if (!list_authors.isEmpty()) {
list.add(new MapCondition("target_publication_author_list", MapValueType.LIST_STRING, ConditionOperator.MATCH_ALL, list_authors));
list.add(new MapCondition("targetAuthors", MapValueType.LIST_STRING, ConditionOperator.MATCH_ALL, list_authors));
}
if (!list_subjects.isEmpty()) {
list.add(new MapCondition("target_publication_subject_list", MapValueType.LIST_STRING, ConditionOperator.MATCH_ALL, list_subjects));
list.add(new MapCondition("targetSubjects", MapValueType.LIST_STRING, ConditionOperator.MATCH_ALL, list_subjects));
}
return list;

View File

@ -80,7 +80,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
.withSearchType(SearchType.DEFAULT)
.addAggregation(AggregationBuilders.nested("nested", "map")
// .path("map")
.subAggregation(AggregationBuilders.terms("by_map").field("map.target_datasource_name").size(1000).minDocCount(1)))
.subAggregation(AggregationBuilders.terms("by_map").field("map.targetDatasourceName").size(1000).minDocCount(1)))
.build();
final SearchHits<Event> hits = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(props.getEventsIndexName()));
@ -100,7 +100,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds) {
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.target_datasource_name", ds), ScoreMode.None))
.withQuery(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None))
.withSearchType(SearchType.DEFAULT)
.addAggregation(AggregationBuilders.terms("topic").field("topic").size(1000).minDocCount(1))
.build();
@ -122,7 +122,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", topic))
.must(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.target_datasource_name", ds), ScoreMode.None)))
.must(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None)))
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.withPageable(PageRequest.of(nPage, size))
@ -144,12 +144,12 @@ public class OpenaireBrokerController extends AbstractLbsController {
public EventsPage advancedShowEvents(@PathVariable final int nPage, @PathVariable final int size, @RequestBody final AdvQueryObject qObj) {
final BoolQueryBuilder mapQuery = QueryBuilders.boolQuery();
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_datasource_name", qObj.getDatasource());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_publication_title", qObj.getTitles());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_publication_author_list", qObj.getAuthors());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_publication_subject_list", qObj.getSubjects());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetDatasourceName", qObj.getDatasource());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetResultTitle", qObj.getTitles());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetAuthors", qObj.getAuthors());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetSubjects", qObj.getSubjects());
ElasticSearchQueryUtils.addMapConditionForTrust(mapQuery, "map.trust", qObj.getTrust());
ElasticSearchQueryUtils.addMapConditionForDates(mapQuery, "map.target_dateofacceptance", qObj.getDates());
ElasticSearchQueryUtils.addMapConditionForDates(mapQuery, "map.targetDateofacceptance", qObj.getDates());
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.boolQuery()
@ -297,7 +297,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
private String extractDatasource(final Subscription sub) {
return sub.getConditionsAsList()
.stream()
.filter(c -> c.getField().equals("target_datasource_name"))
.filter(c -> c.getField().equals("targetDatasourceName"))
.map(MapCondition::getListParams)
.filter(l -> !l.isEmpty())
.map(l -> l.get(0).getValue())

View File

@ -1,8 +1,11 @@
#!/bin/bash
INDEXBASEURL=http://localhost:9200
INDEXNAME=events_beta_michele
echo
echo "Fixing events index..."
curl -XPUT -H 'Content-Type: application/json' 'http://localhost:9200/events/_mapping' -d '
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$INDEXNAME/_mapping" -d '
{
"properties" : {
"map": {
@ -28,7 +31,7 @@ curl -XPUT -H 'Content-Type: application/json' 'http://localhost:9200/events/_ma
echo
echo
echo "Fixing notifications index..."
curl -XPUT -H 'Content-Type: application/json' 'http://localhost:9200/notifications/_mapping' -d '
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$INDEXNAME/_mapping" -d '
{
"properties" : {
"map": {
@ -54,15 +57,15 @@ curl -XPUT -H 'Content-Type: application/json' 'http://localhost:9200/notificati
echo
echo
echo "Registering topic type: ENRICH/<cond>/<field>..."
curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=target_datasource_name"
curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=targetDatasourceName"
echo
echo
echo "Registering topic type: ENRICH/<cond>/<field>/<subfield>..."
curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH_WITH_SUBFIELD&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E%2F%3Csubfield%3E&producerId=OpenAIRE&mapKeys=target_datasource_name"
curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH_WITH_SUBFIELD&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E%2F%3Csubfield%3E&producerId=OpenAIRE&mapKeys=targetDatasourceName"
echo
echo
echo "Registering topic type: ADD/<field>..."
curl "http://localhost:8080/api/topic-types/add" -d "name=ADD&expression=ADD%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=target_datasource_name"
curl "http://localhost:8080/api/topic-types/add" -d "name=ADD&expression=ADD%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=targetDatasourceName"
echo
echo
echo

View File

@ -25,13 +25,13 @@
<tr>
<th class="col-xs-4 col-md-3 col-lg-2">ID</th>
<td class="col-xs-8 col-md-9 col-lg-10">
{{val.publication.originalId}}
{{val.result.originalId}}
</td>
</tr>
<tr>
<th>Title(s)</th>
<td>
<span ng-repeat="t in val.publication.titles track by $index">
<span ng-repeat="t in val.result.titles track by $index">
<br ng-if="$index > 0"/>
{{t}}
</span>
@ -40,7 +40,7 @@
<tr>
<th>Author(s)</th>
<td>
<span ng-repeat="a in val.publication.creators track by $index">
<span ng-repeat="a in val.result.creators track by $index">
<br ng-if="$index > 0"/>
{{a}}
</span>
@ -49,7 +49,7 @@
<tr ng-class="{'success' : val.highlight.pids && val.highlight.pids.length > 0}">
<th>PID(s)</th>
<td>
<span ng-repeat="pid in val.publication.pids">
<span ng-repeat="pid in val.result.pids">
<br ng-if="$index > 0"/>
{{pid.value}} <b>({{pid.type}})</b>
@ -69,7 +69,7 @@
<tr ng-class="{'success' : val.highlight.abstracts && val.highlight.abstracts.length > 0}">
<th>Abstract(s)</th>
<td>
<span ng-repeat="d in val.publication.abstracts track by $index">
<span ng-repeat="d in val.result.abstracts track by $index">
<br ng-if="$index > 0"/>
{{d}}
</span>
@ -84,7 +84,7 @@
<tr>
<th>Subject(s)</th>
<td>
<span ng-repeat="s in val.publication.subjects track by $index">
<span ng-repeat="s in val.result.subjects track by $index">
<br ng-if="$index > 0"/>
{{s}}
</span>
@ -93,7 +93,7 @@
<tr ng-class="{'success' : val.highlight.publicationdate }">
<th>Publication date</th>
<td>
{{val.publication.publicationdate}}
{{val.result.publicationdate}}
<span class="pull-right" ng-if="val.highlight.publicationdate">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
@ -105,7 +105,7 @@
<tr ng-class="{'success' : val.highlight.instances && val.highlight.instances.length > 0}">
<th>Rights</th>
<td>
<span ng-repeat="i in val.publication.instances">
<span ng-repeat="i in val.result.instances">
<br ng-if="$index > 0"/>
{{i.license}} <b>({{i.hostedby}})</b>
<span ng-if="val.highlight.instances && isInstanceHighlighted(i, val.highlight.instances)">

View File

@ -27,13 +27,13 @@
<tr>
<th class="col-xs-4 col-md-3 col-lg-2">ID</th>
<td class="col-xs-8 col-md-9 col-lg-10">
{{val.publication.originalId}}
{{val.result.originalId}}
</td>
</tr>
<tr>
<th>Title(s)</th>
<td>
<span ng-repeat="t in val.publication.titles track by $index">
<span ng-repeat="t in val.result.titles track by $index">
<br ng-if="$index > 0"/>
{{t}}
</span>
@ -42,7 +42,7 @@
<tr>
<th>Author(s)</th>
<td>
<span ng-repeat="a in val.publication.creators track by $index">
<span ng-repeat="a in val.result.creators track by $index">
<br ng-if="$index > 0"/>
{{a}}
</span>
@ -51,7 +51,7 @@
<tr ng-class="{'success' : val.highlight.pids && val.highlight.pids.length > 0}">
<th>PID(s)</th>
<td>
<span ng-repeat="pid in val.publication.pids">
<span ng-repeat="pid in val.result.pids">
<br ng-if="$index > 0"/>
{{pid.value}} <b>({{pid.type}})</b>
@ -71,7 +71,7 @@
<tr ng-class="{'success' : val.highlight.abstracts && val.highlight.abstracts.length > 0}">
<th>Abstract(s)</th>
<td>
<span ng-repeat="d in val.publication.abstracts track by $index">
<span ng-repeat="d in val.result.abstracts track by $index">
<br ng-if="$index > 0"/>
{{d}}
</span>
@ -86,7 +86,7 @@
<tr ng-class="{'success' : val.highlight.subjects && val.highlight.subjects.length > 0}">
<th>Subject(s)</th>
<td>
<span ng-repeat="s in val.publication.subjects track by $index">
<span ng-repeat="s in val.result.subjects track by $index">
<br ng-if="$index > 0"/>
{{s}}
<span ng-if="val.highlight.subjects && isSubjectHighlighted(s, val.highlight.subjects)">
@ -104,7 +104,7 @@
<tr ng-class="{'success' : val.highlight.publicationdate }">
<th>Publication date</th>
<td>
{{val.publication.publicationdate}}
{{val.result.publicationdate}}
<span class="pull-right" ng-if="val.highlight.publicationdate">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
@ -116,7 +116,7 @@
<tr ng-class="{'success' : val.highlight.instances && val.highlight.instances.length > 0}">
<th>Rights</th>
<td>
<span ng-repeat="i in val.publication.instances">
<span ng-repeat="i in val.result.instances">
<br ng-if="$index > 0"/>
{{i.license}} <b>({{i.hostedby}})</b>
<span ng-if="val.highlight.instances && isInstanceHighlighted(i, val.highlight.instances)">
@ -136,7 +136,7 @@
<tr ng-class="{'success' : val.highlight.projects && val.highlight.projects.length > 0}">
<th>Project(s)</th>
<td>
<span ng-repeat="p in val.publication.projects">
<span ng-repeat="p in val.result.projects">
<br ng-if="$index > 0"/>
{{p.title}}
<span ng-if="val.highlight.projects && isProjectHighlighted(p, val.highlight.projects)">
@ -165,7 +165,7 @@
<tr ng-class="{'success' : val.highlight.datasets && val.highlight.datasets.length > 0}">
<th>Dataset(s)</th>
<td>
<span ng-repeat="d in val.publication.datasets">
<span ng-repeat="d in val.result.datasets">
<br ng-if="$index > 0"/>
{{d.titles[0]}}
<span ng-if="val.highlight.datasets && isDatasetHighlighted(d, val.highlight.datasets)">
@ -187,7 +187,7 @@
<tr ng-class="{'success' : val.highlight.publications && val.highlight.publications.length > 0}">
<th>Related publications(s)</th>
<td>
<span ng-repeat="p in val.publication.publications">
<span ng-repeat="p in val.result.publications">
<br ng-if="$index > 0"/>
{{p.titles[0]}}
<span ng-if="val.highlight.publications && isRelatedPublicationHighlighted(p, val.highlight.publications)">

View File

@ -74,7 +74,7 @@ public class IndexClientTest {
s.setFrequency(NotificationFrequency.daily);
s.setMode(NotificationMode.MOCK);
s.setConditions(
"[{\"field\":\"target_datasource_name\",\"fieldType\":\"STRING\",\"operator\":\"EXACT\",\"listParams\":[{\"value\":\"Research Papers in Economics\"}]},{\"field\":\"trust\",\"fieldType\":\"FLOAT\",\"operator\":\"RANGE\",\"listParams\":[{\"value\":\"0\",\"otherValue\":\"1\"}]}]");
"[{\"field\":\"targetDatasourceName\",\"fieldType\":\"STRING\",\"operator\":\"EXACT\",\"listParams\":[{\"value\":\"Research Papers in Economics\"}]},{\"field\":\"trust\",\"fieldType\":\"FLOAT\",\"operator\":\"RANGE\",\"listParams\":[{\"value\":\"0\",\"otherValue\":\"1\"}]}]");
final BoolQueryBuilder mapQuery = QueryBuilders.boolQuery();

View File

@ -4,7 +4,7 @@ import org.junit.Ignore;
import org.junit.Test;
import org.springframework.web.client.RestTemplate;
import eu.dnetlib.broker.objects.OpenAireEventPayload;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.lbs.openaire.ScrollPage;
public class ScrollTest {
@ -27,7 +27,7 @@ public class ScrollTest {
while (!page.isCompleted()) {
page = getPage(baseUrl + "/api/openaireBroker/scroll/notifications/ " + page.getId());
total += page.getValues().size();
for (final OpenAireEventPayload p : page.getValues()) {
for (final OaBrokerEventPayload p : page.getValues()) {
// DO SOMETHING
}
}

View File

@ -22,8 +22,8 @@ import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilde
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import eu.dnetlib.broker.objects.OpenAireEventPayload;
import eu.dnetlib.broker.objects.Publication;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.broker.objects.OaBrokerMainEntity;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.openaire.AdvQueryObject;
import eu.dnetlib.lbs.openaire.ElasticSearchQueryUtils;
@ -31,7 +31,9 @@ import eu.dnetlib.lbs.openaire.Range;
@Ignore
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = { "classpath:/applicationContext-test-queries.xml" })
@ContextConfiguration(locations = {
"classpath:/applicationContext-test-queries.xml"
})
public class OpenaireQueriesTest {
private static final String indexName = Event.class.getAnnotation(Document.class).indexName();
@ -57,31 +59,31 @@ public class OpenaireQueriesTest {
qObj.setTrust(new Range("0", "1"));
final BoolQueryBuilder mapQuery = QueryBuilders.boolQuery();
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_datasource_name", qObj.getDatasource());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_publication_title", qObj.getTitles());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_publication_author_list", qObj.getAuthors());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.target_publication_subject_list", qObj.getSubjects());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetDatasourceName", qObj.getDatasource());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetResultTitle", qObj.getTitles());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetAuthors", qObj.getAuthors());
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetSubjects", qObj.getSubjects());
ElasticSearchQueryUtils.addMapConditionForTrust(mapQuery, "map.trust", qObj.getTrust());
ElasticSearchQueryUtils.addMapConditionForDates(mapQuery, "map.target_dateofacceptance", qObj.getDates());
ElasticSearchQueryUtils.addMapConditionForDates(mapQuery, "map.targetDateofacceptance", qObj.getDates());
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", qObj.getTopic()))
.must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None)))
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.withPageable(PageRequest.of(0, 10))
.build();
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", qObj.getTopic()))
.must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None)))
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.withPageable(PageRequest.of(0, 10))
.build();
final SearchHits<Event> page = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(indexName));
page.stream()
.map(SearchHit::getContent)
.map(Event::getPayload)
.map(OpenAireEventPayload::fromJSON)
.map(OpenAireEventPayload::getPublication)
.map(Publication::getTitles)
.forEach(System.out::println);
.map(SearchHit::getContent)
.map(Event::getPayload)
.map(OaBrokerEventPayload::fromJSON)
.map(OaBrokerEventPayload::getResult)
.map(OaBrokerMainEntity::getTitles)
.forEach(System.out::println);
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long