D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

total topic counts in a sql table

This commit is contained in:
Michele Artini 2020-09-17 15:50:51 +02:00
parent 7a074c6b87
commit a945eeed6a
8 changed files with 206 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java
View File

@ -10,6 +10,7 @@ import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.search.join.ScoreMode;
@ -31,6 +32,8 @@ import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@ -74,12 +77,18 @@ public class OpenaireBrokerController extends AbstractLbsController {
@Autowired
private DispatcherManager dispatcher;
@Autowired
private JdbcTemplate jdbcTemplate;
private static final Log log = LogFactory.getLog(OpenaireBrokerController.class);
@ApiOperation("Return the datasources having events")
@GetMapping("/datasources")
public List<BrowseEntry> findDatasourcesWithEvents() {
public List<BrowseEntry> findDatasourcesWithEvents(@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
return useIndex ? findDatasourcesWithEventsUsingIndex() : findDatasourcesWithEventsUsingDb();
}
private List<BrowseEntry> findDatasourcesWithEventsUsingIndex() {
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.matchAllQuery())
.withSearchType(SearchType.DEFAULT)
@ -100,10 +109,25 @@ public class OpenaireBrokerController extends AbstractLbsController {
.collect(Collectors.toList());
}
private List<BrowseEntry> findDatasourcesWithEventsUsingDb() {
try {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopics.sql"));
final RowMapper<BrowseEntry> mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("name"), rs.getLong("size"));
return jdbcTemplate.query(sql, mapper);
} catch (final Exception e) {
log.error("Error executing query", e);
return new ArrayList<>();
}
}
@ApiOperation("Return the topics of the events of a datasource")
@GetMapping("/topicsForDatasource")
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds) {
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds,
@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
return useIndex ? findTopicsForDatasourceUsingIndex(ds) : findTopicsForDatasourceUsingDb(ds);
}
private List<BrowseEntry> findTopicsForDatasourceUsingIndex(final String ds) {
final String term = "topic.keyword";
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
@ -122,6 +146,20 @@ public class OpenaireBrokerController extends AbstractLbsController {
.collect(Collectors.toList());
}
private List<BrowseEntry> findTopicsForDatasourceUsingDb(final String ds) {
try {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopicsDetailed.sql"));
final RowMapper<BrowseEntry> mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("topic"), rs.getLong("size"));
return jdbcTemplate.query(sql, new Object[] {
ds
}, mapper);
} catch (final Exception e) {
log.error("Error executing query", e);
return new ArrayList<>();
}
}
@ApiOperation("Return a page of events of a datasource (by topic)")
@GetMapping("/events/{nPage}/{size}")
public EventsPage showEvents(@RequestParam final String ds, @RequestParam final String topic, @PathVariable final int nPage, @PathVariable final int size) {
@ -236,6 +274,19 @@ public class OpenaireBrokerController extends AbstractLbsController {
return Arrays.asList("Sending ...");
}
@ApiOperation("Update stats")
@GetMapping("/stats/update")
private List<String> updateStats() {
new Thread(() -> {
try {
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/updateStats.sql")));
} catch (final Exception e) {
log.error("Error updating stats", e);
}
}).start();
return Arrays.asList("Sending ...");
}
private void innerSendMailForNotifications(final long date) {
for (final Subscription s : subscriptionRepo.findAll()) {

9
apps/dhp-broker-application/src/main/resources/config/schema.sql
View File

@ -18,6 +18,15 @@ CREATE TABLE topic_types (
regex text UNIQUE NOT NULL
);
CREATE TABLE oa_datasource_stats (
id text NOT NULL,
name text NOT NULL,
type text NOT NULL,
topic text NOT NULL,
size bigint NOT NULL,
PRIMARY KEY (id, topic)
);
-- curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=targetDatasourceName"
INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-a739fa2b-fde0-4eb2-bcee-6e7f277347db', 'ENRICH', 'ENRICH/<cond>/<field>', 'targetDatasourceName', 'OpenAIRE', '^ENRICH\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$');

1
apps/dhp-broker-application/src/main/resources/sql/datasourceTopics.sql Normal file
View File

@ -0,0 +1 @@
select (array_agg(name))[1] as name, sum(size) as size from oa_datasource_stats group by lower(name) order by sum(size) desc;

7
apps/dhp-broker-application/src/main/resources/sql/datasourceTopicsDetailed.sql Normal file
View File

@ -0,0 +1,7 @@
select
topic as topic,
sum(size) as size
from oa_datasource_stats
where lower(name) = lower(?)
group by topic
order by sum(size) desc;

5
apps/dhp-broker-application/src/main/resources/sql/updateStats.sql Normal file
View File

@ -0,0 +1,5 @@
begin;
delete from oa_datasource_stats;
insert into oa_datasource_stats (id, name, type, topic, size) select id, name, type, topic, size from oa_datasource_stats_temp;
drop table oa_datasource_stats_temp;
commit;

74
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStat.java Normal file
View File

@ -0,0 +1,74 @@
package eu.dnetlib.broker.common.stats;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.Table;
@Entity
@Table(name = "oa_datasource_stats")
@IdClass(OpenaireDsStatPK.class)
public class OpenaireDsStat implements Serializable {
/**
*
*/
private static final long serialVersionUID = 6498718759303687338L;
@Id
@Column(name = "id", nullable = false)
private String id;
@Id
@Column(name = "topic", nullable = false)
private String topic;
@Column(name = "name", nullable = false)
private String name;
@Column(name = "type", nullable = false)
private String type;
@Column(name = "size", nullable = false)
private final Long size = 0l;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getTopic() {
return topic;
}
public void setTopic(final String topic) {
this.topic = topic;
}
public Long getSize() {
return size;
}
}

50
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatPK.java Normal file
View File

@ -0,0 +1,50 @@
package eu.dnetlib.broker.common.stats;
import java.io.Serializable;
import java.util.Objects;
public class OpenaireDsStatPK implements Serializable {
/**
*
*/
private static final long serialVersionUID = 3061558721019854932L;
private String id;
private String topic;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getTopic() {
return topic;
}
public void setTopic(final String topic) {
this.topic = topic;
}
@Override
public int hashCode() {
return Objects.hash(id, topic);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof OpenaireDsStatPK)) { return false; }
final OpenaireDsStatPK other = (OpenaireDsStatPK) obj;
return Objects.equals(id, other.id) && Objects.equals(topic, other.topic);
}
@Override
public String toString() {
return String.format("OpenaireDsStatPK [id=%s, topic=%s]", id, topic);
}
}

7
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java Normal file
View File

@ -0,0 +1,7 @@
package eu.dnetlib.broker.common.stats;
import org.springframework.data.repository.CrudRepository;
public interface OpenaireDsStatRepository extends CrudRepository<OpenaireDsStat, OpenaireDsStatPK> {
}