339 lines
14 KiB
Java
339 lines
14 KiB
Java
package eu.dnetlib.broker.openaire;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Optional;
|
|
import java.util.stream.Collectors;
|
|
import java.util.stream.StreamSupport;
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.lucene.search.join.ScoreMode;
|
|
import org.elasticsearch.action.search.SearchType;
|
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
|
import org.elasticsearch.index.query.NestedQueryBuilder;
|
|
import org.elasticsearch.index.query.Operator;
|
|
import org.elasticsearch.index.query.QueryBuilder;
|
|
import org.elasticsearch.index.query.QueryBuilders;
|
|
import org.elasticsearch.search.aggregations.Aggregation;
|
|
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
|
import org.elasticsearch.search.aggregations.Aggregations;
|
|
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
|
|
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.context.annotation.Profile;
|
|
import org.springframework.data.domain.PageRequest;
|
|
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
|
|
import org.springframework.data.elasticsearch.core.SearchHit;
|
|
import org.springframework.data.elasticsearch.core.SearchHits;
|
|
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
|
|
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
|
|
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
|
import org.springframework.jdbc.core.RowMapper;
|
|
import org.springframework.web.bind.annotation.GetMapping;
|
|
import org.springframework.web.bind.annotation.PathVariable;
|
|
import org.springframework.web.bind.annotation.PostMapping;
|
|
import org.springframework.web.bind.annotation.RequestBody;
|
|
import org.springframework.web.bind.annotation.RequestMapping;
|
|
import org.springframework.web.bind.annotation.RequestParam;
|
|
import org.springframework.web.bind.annotation.RestController;
|
|
|
|
import eu.dnetlib.broker.LiteratureBrokerServiceApplication;
|
|
import eu.dnetlib.broker.common.elasticsearch.Event;
|
|
import eu.dnetlib.broker.common.elasticsearch.Notification;
|
|
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
|
|
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
|
|
import eu.dnetlib.broker.common.subscriptions.MapCondition;
|
|
import eu.dnetlib.broker.common.subscriptions.Subscription;
|
|
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
|
|
import eu.dnetlib.broker.events.output.DispatcherManager;
|
|
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
|
|
import eu.dnetlib.common.controller.AbstractDnetController;
|
|
import io.swagger.v3.oas.annotations.Operation;
|
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
|
|
|
@Profile("openaire")
|
|
@RestController
|
|
@RequestMapping("/api/openaireBroker")
|
|
@Tag(name = LiteratureBrokerServiceApplication.TAG_OPENAIRE)
|
|
public class OpenaireBrokerController extends AbstractDnetController {
|
|
|
|
@Autowired
|
|
private ElasticsearchOperations esOperations;
|
|
|
|
@Autowired
|
|
private NotificationRepository notificationRepository;
|
|
|
|
@Autowired
|
|
private SubscriptionRepository subscriptionRepo;
|
|
|
|
@Autowired
|
|
private ElasticSearchProperties props;
|
|
|
|
@Autowired
|
|
private DispatcherManager dispatcher;
|
|
|
|
@Autowired
|
|
private JdbcTemplate jdbcTemplate;
|
|
|
|
private static final Log log = LogFactory.getLog(OpenaireBrokerController.class);
|
|
|
|
@Operation(summary = "Return the datasources having events")
|
|
@GetMapping("/datasources")
|
|
public List<BrowseEntry> findDatasourcesWithEvents(@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
|
|
return useIndex ? findDatasourcesWithEventsUsingIndex() : findDatasourcesWithEventsUsingDb();
|
|
}
|
|
|
|
private List<BrowseEntry> findDatasourcesWithEventsUsingIndex() {
|
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
|
.withQuery(QueryBuilders.matchAllQuery())
|
|
.withSearchType(SearchType.DEFAULT)
|
|
.addAggregation(AggregationBuilders.nested("nested", "map")
|
|
// .path("map")
|
|
.subAggregation(AggregationBuilders.terms("by_map").field("map.targetDatasourceName").size(1000).minDocCount(1)))
|
|
.build();
|
|
|
|
final SearchHits<Event> hits = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(props.getEventsIndexName()));
|
|
|
|
final Aggregations aggregations = hits.getAggregations();
|
|
|
|
final Aggregation aggByMap = ((ParsedNested) aggregations.asMap().get("nested")).getAggregations().asMap().get("by_map");
|
|
|
|
return ((ParsedStringTerms) aggByMap).getBuckets()
|
|
.stream()
|
|
.map(b -> new BrowseEntry(b.getKeyAsString(), b.getDocCount()))
|
|
.collect(Collectors.toList());
|
|
}
|
|
|
|
private List<BrowseEntry> findDatasourcesWithEventsUsingDb() {
|
|
try {
|
|
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopics.sql"));
|
|
final RowMapper<BrowseEntry> mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("name"), rs.getLong("size"));
|
|
return jdbcTemplate.query(sql, mapper);
|
|
} catch (final Exception e) {
|
|
log.error("Error executing query", e);
|
|
return new ArrayList<>();
|
|
}
|
|
}
|
|
|
|
@Operation(summary = "Return the topics of the events of a datasource")
|
|
@GetMapping("/topicsForDatasource")
|
|
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds,
|
|
@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
|
|
return useIndex ? findTopicsForDatasourceUsingIndex(ds) : findTopicsForDatasourceUsingDb(ds);
|
|
}
|
|
|
|
private List<BrowseEntry> findTopicsForDatasourceUsingIndex(final String ds) {
|
|
final String term = "topic.keyword";
|
|
|
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
|
.withQuery(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None))
|
|
.withSearchType(SearchType.DEFAULT)
|
|
.addAggregation(AggregationBuilders.terms(term).field(term).size(1000).minDocCount(1))
|
|
.build();
|
|
|
|
final SearchHits<Event> hits = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(props.getEventsIndexName()));
|
|
|
|
final Aggregations aggregations = hits.getAggregations();
|
|
|
|
return ((ParsedStringTerms) aggregations.asMap().get(term)).getBuckets()
|
|
.stream()
|
|
.map(b -> new BrowseEntry(b.getKeyAsString(), b.getDocCount()))
|
|
.collect(Collectors.toList());
|
|
}
|
|
|
|
private List<BrowseEntry> findTopicsForDatasourceUsingDb(final String ds) {
|
|
|
|
try {
|
|
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopicsDetailed.sql"));
|
|
final RowMapper<BrowseEntry> mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("topic"), rs.getLong("size"));
|
|
return jdbcTemplate.query(sql, new Object[] {
|
|
ds
|
|
}, mapper);
|
|
} catch (final Exception e) {
|
|
log.error("Error executing query", e);
|
|
return new ArrayList<>();
|
|
}
|
|
}
|
|
|
|
@Operation(summary = "Return a page of events of a datasource (by topic)")
|
|
@GetMapping("/events/{nPage}/{size}")
|
|
public EventsPage showEvents(@RequestParam final String ds, @RequestParam final String topic, @PathVariable final int nPage, @PathVariable final int size) {
|
|
|
|
final NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None);
|
|
|
|
final QueryBuilder q = StringUtils.isNotBlank(topic) && !topic.equals("*") ? QueryBuilders.boolQuery()
|
|
.must(QueryBuilders.matchQuery("topic", topic).operator(Operator.AND))
|
|
.must(nestedQuery) : nestedQuery;
|
|
|
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
|
.withQuery(q)
|
|
.withSearchType(SearchType.DEFAULT)
|
|
.withFields("payload")
|
|
.withPageable(PageRequest.of(nPage, size))
|
|
.build();
|
|
|
|
final SearchHits<Event> page = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(props.getEventsIndexName()));
|
|
|
|
final List<OaBrokerEventPayload> list = page.stream()
|
|
.map(SearchHit::getContent)
|
|
.map(Event::getPayload)
|
|
.map(OaBrokerEventPayload::fromJSON)
|
|
.collect(Collectors.toList());
|
|
|
|
return new EventsPage(ds, topic, nPage, overrideGetTotalPage(page, size), page.getTotalHits(), list);
|
|
}
|
|
|
|
@Operation(summary = "Return a page of events of a datasource (by query)")
|
|
@PostMapping("/events/{nPage}/{size}")
|
|
public EventsPage advancedShowEvents(@PathVariable final int nPage, @PathVariable final int size, @RequestBody final AdvQueryObject qObj) {
|
|
|
|
final BoolQueryBuilder mapQuery = QueryBuilders.boolQuery();
|
|
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetDatasourceName", qObj.getDatasource());
|
|
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetResultTitle", qObj.getTitles());
|
|
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetAuthors", qObj.getAuthors());
|
|
ElasticSearchQueryUtils.addMapCondition(mapQuery, "map.targetSubjects", qObj.getSubjects());
|
|
ElasticSearchQueryUtils.addMapConditionForTrust(mapQuery, "map.trust", qObj.getTrust());
|
|
ElasticSearchQueryUtils.addMapConditionForDates(mapQuery, "map.targetDateofacceptance", qObj.getDates());
|
|
|
|
final NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None);
|
|
|
|
final QueryBuilder q = StringUtils.isNotBlank(qObj.getTopic()) && !qObj.getTopic().equals("*") ? QueryBuilders.boolQuery()
|
|
.must(QueryBuilders.matchQuery("topic", qObj.getTopic()).operator(Operator.AND))
|
|
.must(nestedQuery) : nestedQuery;
|
|
|
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
|
.withQuery(q)
|
|
.withSearchType(SearchType.DEFAULT)
|
|
.withFields("payload")
|
|
.withPageable(PageRequest.of(nPage, size))
|
|
.build();
|
|
|
|
final SearchHits<Event> page = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(props.getEventsIndexName()));
|
|
|
|
final List<OaBrokerEventPayload> list = page.stream()
|
|
.map(SearchHit::getContent)
|
|
.map(Event::getPayload)
|
|
.map(OaBrokerEventPayload::fromJSON)
|
|
.collect(Collectors.toList());
|
|
|
|
return new EventsPage(qObj.getDatasource(), qObj.getTopic(), nPage, overrideGetTotalPage(page, size), page.getTotalHits(), list);
|
|
}
|
|
|
|
@Operation(summary = "Perform a subscription")
|
|
@PostMapping("/subscribe")
|
|
public Subscription registerSubscription(@RequestBody final OpenaireSubscription oSub) {
|
|
final Subscription sub = oSub.asSubscription();
|
|
|
|
subscriptionRepo.save(sub);
|
|
|
|
return sub;
|
|
}
|
|
|
|
@Operation(summary = "Return the subscriptions of an user (by email and datasource (optional))")
|
|
@GetMapping("/subscriptions")
|
|
public Map<String, List<SimpleSubscriptionDesc>> subscriptions(@RequestParam final String email, @RequestParam(required = false) final String ds) {
|
|
final Iterable<Subscription> iter = subscriptionRepo.findBySubscriber(email);
|
|
return StreamSupport.stream(iter.spliterator(), false)
|
|
.map(this::subscriptionDesc)
|
|
.filter(s -> StringUtils.isBlank(ds) || StringUtils.equalsIgnoreCase(s.getDatasource(), ds))
|
|
.collect(Collectors.groupingBy(SimpleSubscriptionDesc::getDatasource));
|
|
}
|
|
|
|
@Operation(summary = "Return a page of notifications")
|
|
@GetMapping("/notifications/{subscrId}/{nPage}/{size}")
|
|
public EventsPage notifications(@PathVariable final String subscrId, @PathVariable final int nPage, @PathVariable final int size) {
|
|
|
|
final Optional<Subscription> optSub = subscriptionRepo.findById(subscrId);
|
|
|
|
if (optSub.isPresent()) {
|
|
final Subscription sub = optSub.get();
|
|
|
|
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
|
|
.withQuery(QueryBuilders.termQuery("subscriptionId.keyword", subscrId))
|
|
.withSearchType(SearchType.DEFAULT)
|
|
.withFields("payload")
|
|
.withPageable(PageRequest.of(nPage, size))
|
|
.build();
|
|
|
|
final SearchHits<Notification> page = esOperations.search(searchQuery, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
|
|
|
|
final List<OaBrokerEventPayload> list = page.stream()
|
|
.map(SearchHit::getContent)
|
|
.map(Notification::getPayload)
|
|
.map(OaBrokerEventPayload::fromJSON)
|
|
.collect(Collectors.toList());
|
|
|
|
return new EventsPage(extractDatasource(sub), sub.getTopic(), nPage, overrideGetTotalPage(page, size), page.getTotalHits(), list);
|
|
} else {
|
|
log.warn("Invalid subscription: " + subscrId);
|
|
return new EventsPage("", "", nPage, 0, 0, new ArrayList<>());
|
|
}
|
|
|
|
}
|
|
|
|
@Operation(summary = "Send notifications")
|
|
@GetMapping("/notifications/send/{date}")
|
|
private List<String> sendMailForNotifications(@PathVariable final long date) {
|
|
new Thread(() -> innerSendMailForNotifications(date)).start();
|
|
return Arrays.asList("Sending ...");
|
|
}
|
|
|
|
@Operation(summary = "Update stats")
|
|
@GetMapping("/stats/update")
|
|
private List<String> updateStats() {
|
|
new Thread(() -> {
|
|
try {
|
|
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/updateStats.sql")));
|
|
} catch (final Exception e) {
|
|
log.error("Error updating stats", e);
|
|
}
|
|
}).start();
|
|
return Arrays.asList("Sending ...");
|
|
}
|
|
|
|
private void innerSendMailForNotifications(final long date) {
|
|
for (final Subscription s : subscriptionRepo.findAll()) {
|
|
|
|
final long count = notificationRepository.countBySubscriptionIdAndDateAfter(s.getSubscriptionId(), date);
|
|
if (count > 0) {
|
|
final Map<String, Object> params = new HashMap<>();
|
|
params.put("oa_notifications_total", count);
|
|
params.put("oa_datasource", extractDatasource(s));
|
|
dispatcher.sendNotification(s, params);
|
|
}
|
|
|
|
s.setLastNotificationDate(new Date());
|
|
subscriptionRepo.save(s);
|
|
}
|
|
}
|
|
|
|
private SimpleSubscriptionDesc subscriptionDesc(final Subscription s) {
|
|
return new SimpleSubscriptionDesc(s.getSubscriptionId(), extractDatasource(s), s.getTopic(), s.getCreationDate(), s.getLastNotificationDate(),
|
|
OpenaireBrokerController.this.notificationRepository.countBySubscriptionId(s.getSubscriptionId()));
|
|
}
|
|
|
|
private String extractDatasource(final Subscription sub) {
|
|
return sub.getConditionsAsList()
|
|
.stream()
|
|
.filter(c -> c.getField().equals("targetDatasourceName"))
|
|
.map(MapCondition::getListParams)
|
|
.filter(l -> !l.isEmpty())
|
|
.map(l -> l.get(0).getValue())
|
|
.findFirst()
|
|
.orElse("");
|
|
}
|
|
|
|
private long overrideGetTotalPage(final SearchHits<?> page, final int size) {
|
|
return (page.getTotalHits() + size - 1) / size;
|
|
}
|
|
|
|
}
|