From a945eeed6a071b39ebac58569c40b2645396f0c2 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 17 Sep 2020 15:50:51 +0200 Subject: [PATCH] total topic counts in a sql table --- .../openaire/OpenaireBrokerController.java | 55 +++++++++++++- .../src/main/resources/config/schema.sql | 9 +++ .../main/resources/sql/datasourceTopics.sql | 1 + .../sql/datasourceTopicsDetailed.sql | 7 ++ .../src/main/resources/sql/updateStats.sql | 5 ++ .../broker/common/stats/OpenaireDsStat.java | 74 +++++++++++++++++++ .../broker/common/stats/OpenaireDsStatPK.java | 50 +++++++++++++ .../stats/OpenaireDsStatRepository.java | 7 ++ 8 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 apps/dhp-broker-application/src/main/resources/sql/datasourceTopics.sql create mode 100644 apps/dhp-broker-application/src/main/resources/sql/datasourceTopicsDetailed.sql create mode 100644 apps/dhp-broker-application/src/main/resources/sql/updateStats.sql create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStat.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatPK.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java index 23c5907b..38e22108 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java @@ -10,6 +10,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.search.join.ScoreMode; @@ -31,6 +32,8 @@ import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -74,12 +77,18 @@ public class OpenaireBrokerController extends AbstractLbsController { @Autowired private DispatcherManager dispatcher; + @Autowired + private JdbcTemplate jdbcTemplate; + private static final Log log = LogFactory.getLog(OpenaireBrokerController.class); @ApiOperation("Return the datasources having events") @GetMapping("/datasources") - public List findDatasourcesWithEvents() { + public List findDatasourcesWithEvents(@RequestParam(defaultValue = "false", required = false) final boolean useIndex) { + return useIndex ? findDatasourcesWithEventsUsingIndex() : findDatasourcesWithEventsUsingDb(); + } + private List findDatasourcesWithEventsUsingIndex() { final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() .withQuery(QueryBuilders.matchAllQuery()) .withSearchType(SearchType.DEFAULT) @@ -100,10 +109,25 @@ public class OpenaireBrokerController extends AbstractLbsController { .collect(Collectors.toList()); } + private List findDatasourcesWithEventsUsingDb() { + try { + final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopics.sql")); + final RowMapper mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("name"), rs.getLong("size")); + return jdbcTemplate.query(sql, mapper); + } catch (final Exception e) { + log.error("Error executing query", e); + return new ArrayList<>(); + } + } + @ApiOperation("Return the topics of the events of a datasource") @GetMapping("/topicsForDatasource") - public List findTopicsForDatasource(@RequestParam final String ds) { + public List findTopicsForDatasource(@RequestParam final String ds, + @RequestParam(defaultValue = "false", required = false) final boolean useIndex) { + return useIndex ? findTopicsForDatasourceUsingIndex(ds) : findTopicsForDatasourceUsingDb(ds); + } + private List findTopicsForDatasourceUsingIndex(final String ds) { final String term = "topic.keyword"; final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() @@ -122,6 +146,20 @@ public class OpenaireBrokerController extends AbstractLbsController { .collect(Collectors.toList()); } + private List findTopicsForDatasourceUsingDb(final String ds) { + + try { + final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopicsDetailed.sql")); + final RowMapper mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("topic"), rs.getLong("size")); + return jdbcTemplate.query(sql, new Object[] { + ds + }, mapper); + } catch (final Exception e) { + log.error("Error executing query", e); + return new ArrayList<>(); + } + } + @ApiOperation("Return a page of events of a datasource (by topic)") @GetMapping("/events/{nPage}/{size}") public EventsPage showEvents(@RequestParam final String ds, @RequestParam final String topic, @PathVariable final int nPage, @PathVariable final int size) { @@ -236,6 +274,19 @@ public class OpenaireBrokerController extends AbstractLbsController { return Arrays.asList("Sending ..."); } + @ApiOperation("Update stats") + @GetMapping("/stats/update") + private List updateStats() { + new Thread(() -> { + try { + jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/updateStats.sql"))); + } catch (final Exception e) { + log.error("Error updating stats", e); + } + }).start(); + return Arrays.asList("Sending ..."); + } + private void innerSendMailForNotifications(final long date) { for (final Subscription s : subscriptionRepo.findAll()) { diff --git a/apps/dhp-broker-application/src/main/resources/config/schema.sql b/apps/dhp-broker-application/src/main/resources/config/schema.sql index a356c351..7472374b 100644 --- a/apps/dhp-broker-application/src/main/resources/config/schema.sql +++ b/apps/dhp-broker-application/src/main/resources/config/schema.sql @@ -18,6 +18,15 @@ CREATE TABLE topic_types ( regex text UNIQUE NOT NULL ); +CREATE TABLE oa_datasource_stats ( + id text NOT NULL, + name text NOT NULL, + type text NOT NULL, + topic text NOT NULL, + size bigint NOT NULL, + PRIMARY KEY (id, topic) +); + -- curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=targetDatasourceName" INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-a739fa2b-fde0-4eb2-bcee-6e7f277347db', 'ENRICH', 'ENRICH//', 'targetDatasourceName', 'OpenAIRE', '^ENRICH\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$'); diff --git a/apps/dhp-broker-application/src/main/resources/sql/datasourceTopics.sql b/apps/dhp-broker-application/src/main/resources/sql/datasourceTopics.sql new file mode 100644 index 00000000..d795e9f0 --- /dev/null +++ b/apps/dhp-broker-application/src/main/resources/sql/datasourceTopics.sql @@ -0,0 +1 @@ +select (array_agg(name))[1] as name, sum(size) as size from oa_datasource_stats group by lower(name) order by sum(size) desc; diff --git a/apps/dhp-broker-application/src/main/resources/sql/datasourceTopicsDetailed.sql b/apps/dhp-broker-application/src/main/resources/sql/datasourceTopicsDetailed.sql new file mode 100644 index 00000000..5aa98aa5 --- /dev/null +++ b/apps/dhp-broker-application/src/main/resources/sql/datasourceTopicsDetailed.sql @@ -0,0 +1,7 @@ +select + topic as topic, + sum(size) as size +from oa_datasource_stats +where lower(name) = lower(?) +group by topic +order by sum(size) desc; diff --git a/apps/dhp-broker-application/src/main/resources/sql/updateStats.sql b/apps/dhp-broker-application/src/main/resources/sql/updateStats.sql new file mode 100644 index 00000000..1f6934de --- /dev/null +++ b/apps/dhp-broker-application/src/main/resources/sql/updateStats.sql @@ -0,0 +1,5 @@ +begin; +delete from oa_datasource_stats; +insert into oa_datasource_stats (id, name, type, topic, size) select id, name, type, topic, size from oa_datasource_stats_temp; +drop table oa_datasource_stats_temp; +commit; diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStat.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStat.java new file mode 100644 index 00000000..101b4dfa --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStat.java @@ -0,0 +1,74 @@ +package eu.dnetlib.broker.common.stats; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.Table; + +@Entity +@Table(name = "oa_datasource_stats") +@IdClass(OpenaireDsStatPK.class) +public class OpenaireDsStat implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 6498718759303687338L; + + @Id + @Column(name = "id", nullable = false) + private String id; + + @Id + @Column(name = "topic", nullable = false) + private String topic; + + @Column(name = "name", nullable = false) + private String name; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "size", nullable = false) + private final Long size = 0l; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getTopic() { + return topic; + } + + public void setTopic(final String topic) { + this.topic = topic; + } + + public Long getSize() { + return size; + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatPK.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatPK.java new file mode 100644 index 00000000..5a25fb45 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatPK.java @@ -0,0 +1,50 @@ +package eu.dnetlib.broker.common.stats; + +import java.io.Serializable; +import java.util.Objects; + +public class OpenaireDsStatPK implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 3061558721019854932L; + + private String id; + + private String topic; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getTopic() { + return topic; + } + + public void setTopic(final String topic) { + this.topic = topic; + } + + @Override + public int hashCode() { + return Objects.hash(id, topic); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { return true; } + if (!(obj instanceof OpenaireDsStatPK)) { return false; } + final OpenaireDsStatPK other = (OpenaireDsStatPK) obj; + return Objects.equals(id, other.id) && Objects.equals(topic, other.topic); + } + + @Override + public String toString() { + return String.format("OpenaireDsStatPK [id=%s, topic=%s]", id, topic); + } +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java new file mode 100644 index 00000000..95c7ce86 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java @@ -0,0 +1,7 @@ +package eu.dnetlib.broker.common.stats; + +import org.springframework.data.repository.CrudRepository; + +public interface OpenaireDsStatRepository extends CrudRepository { + +}