D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

broker email, /health and /metrics controllers

This commit is contained in:
Michele Artini 2020-12-11 13:50:35 +01:00
parent 836bb179f8
commit 9c79faa83c
19 changed files with 304 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/dhp-broker-application/pom.xml
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
@ -36,6 +38,20 @@
</dependencies>
<build>
<resources>
<resource>
<directory>.</directory>
<includes>
<include>pom.xml</include>
</includes>
</resource>
<resource>
<directory>src/main/java</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>

34
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java
View File

@ -1,7 +1,15 @@
package eu.dnetlib.broker;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.annotation.PostConstruct;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -9,6 +17,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.web.servlet.ServletRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
@ -16,9 +25,13 @@ import org.springframework.data.elasticsearch.repository.config.EnableElasticsea
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import com.google.common.collect.Lists;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import io.micrometer.core.instrument.ImmutableTag;
import io.micrometer.core.instrument.Metrics;
import io.prometheus.client.exporter.MetricsServlet;
import io.prometheus.client.hotspot.DefaultExports;
import springfox.documentation.builders.RequestHandlerSelectors;
@ -42,6 +55,9 @@ public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchC
@Autowired
private ElasticSearchProperties elasticSearchProperties;
@Value("pom.xml")
private ClassPathResource pom;
public static final String TAG_EVENTS = "Events";
public static final String TAG_SUBSCRIPTIONS = "Subscriptions";
public static final String TAG_NOTIFICATIONS = "Notifications";
@ -49,6 +65,24 @@ public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchC
public static final String TAG_OPENAIRE = "OpenAIRE";
public static final String TAG_MATCHING = "Subscription-Event Matching";
@PostConstruct
public void init() {
final MavenXpp3Reader reader = new MavenXpp3Reader();
try {
final Model model = reader.read(new InputStreamReader(pom.getInputStream()));
log.info(String.format("registering metric for %s", model.getArtifactId()));
final ImmutableTag tag1 = new ImmutableTag("component", model.getGroupId() + ":" + model.getArtifactId());
// final ImmutableTag tag2 = new ImmutableTag("version", model.getVersion());
// final ImmutableTag tag3 = new ImmutableTag("scmtag", model.getScm().getTag());
Metrics.gauge("micrometer_info", Lists.newArrayList(tag1), 1);
} catch (IOException | XmlPullParserException e) {
log.error("Error registering metric", e);
}
}
@Override
@Bean
public RestHighLevelClient elasticsearchClient() {

46
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/output/EmailDispatcher.java
View File

@ -44,6 +44,15 @@ public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
@Value("${lbs.mail.message.template}")
private Resource emailTemplate;
@Value("${lbs.mail.message.template.dashboard.baseUrl}")
private String dashboardBaseUrl;
@Value("${lbs.mail.message.template.dashboard.helpdesk}")
private String helpdesk;
@Value("${lbs.mail.message.template.publicApi}")
private String publicApi;
public static final int MAX_NUMBER_OF_EVENTS = 20;
private static final Log log = LogFactory.getLog(EmailDispatcher.class);
@ -75,12 +84,15 @@ public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
return prepareMimeMessage(subscription, content);
}
private String generateMailContent(final Subscription subscription, final Map<String, Object> params, final Event... events) throws IOException {
protected String generateMailContent(final Subscription subscription, final Map<String, Object> params, final Event... events) throws IOException {
final StringTemplate st = new StringTemplate(IOUtils.toString(emailTemplate.getInputStream(), StandardCharsets.UTF_8));
st.setAttribute("sub", subscription);
st.setAttribute("total", events.length);
st.setAttribute("max", MAX_NUMBER_OF_EVENTS);
st.setAttribute("dashboardBaseUrl", dashboardBaseUrl.trim());
st.setAttribute("publicApi", publicApi.trim());
st.setAttribute("helpdesk", helpdesk.trim());
if (events.length > MAX_NUMBER_OF_EVENTS) {
st.setAttribute("events", Arrays.copyOfRange(events, 0, MAX_NUMBER_OF_EVENTS));
@ -154,4 +166,36 @@ public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
return NotificationMode.EMAIL;
}
protected Resource getEmailTemplate() {
return emailTemplate;
}
protected void setEmailTemplate(final Resource emailTemplate) {
this.emailTemplate = emailTemplate;
}
protected String getDashboardBaseUrl() {
return dashboardBaseUrl;
}
protected void setDashboardBaseUrl(final String dashboardBaseUrl) {
this.dashboardBaseUrl = dashboardBaseUrl;
}
protected String getHelpdesk() {
return helpdesk;
}
protected void setHelpdesk(final String helpdesk) {
this.helpdesk = helpdesk;
}
protected String getPublicApi() {
return publicApi;
}
protected void setPublicApi(final String publicApi) {
this.publicApi = publicApi;
}
}

3
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java
View File

@ -306,6 +306,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
if (count > 0) {
final Map<String, Object> params = new HashMap<>();
params.put("oa_notifications_total", count);
params.put("oa_datasource", extractDatasource(s));
dispatcher.sendNotification(s, params);
}
@ -327,7 +328,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
.filter(l -> !l.isEmpty())
.map(l -> l.get(0).getValue())
.findFirst()
.get();
.orElse("");
}
private long overrideGetTotalPage(final SearchHits<?> page, final int size) {

13
apps/dhp-broker-application/src/main/resources/application.properties
View File

@ -2,6 +2,11 @@ spring.profiles.active = dev,openaire
#logging.level.root=DEBUG
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
spring.datasource.url=jdbc:postgresql://localhost:5432/oa_broker
spring.datasource.username=oa_broker
spring.datasource.password=
@ -33,13 +38,17 @@ lbs.elastic.eventsIndexName = oa_events_beta
lbs.elastic.notificationsIndexName = oa_notifications_beta
lbs.mail.from = notifications-noreply@openaire.eu
lbs.mail.fromName = Literature Broker Service
lbs.mail.cc = michele.artini@isti.cnr.it
lbs.mail.fromName = OpenAIRE Broker Service
lbs.mail.cc =
lbs.mail.smtpHost = smtp.isti.cnr.it
lbs.mail.smtpPort = 587
lbs.mail.smtpUser = smtp-dnet
lbs.mail.smtpPassword = hhr*7932
lbs.mail.message.template = classpath:/templates/dhp_openaire_mail.st
lbs.mail.message.template.dashboard.baseUrl = https://provide.openaire.eu/content/notifications/
lbs.mail.message.template.dashboard.helpdesk = https://www.openaire.eu/support/helpdesk
lbs.mail.message.template.publicApi = https://api.openaire.eu/broker
lbs.queues.maxReturnedValues = 1000

11
apps/dhp-broker-application/src/main/resources/templates/dhp_openaire_mail.st
View File

@ -1,7 +1,7 @@
You have received this mail because you are owner of the following subscription:
<ul>
<li><b>ID: </b>$sub.subscriptionId$</li>
<li><b>Datasource Name: </b>$oa_datasource$</li>
<li><b>Topic: </b>$sub.topic$</li>
<li><b>Number of events: </b>$oa_notifications_total$</li>
</ul>
@ -9,9 +9,12 @@ You have received this mail because you are owner of the following subscription:
<hr />
<p>
You can access the notified events at this address: .... <br />
or using the Public Broker Service API.
You can access the notified events at this address:
<a href="$dashboardBaseUrl$$sub.subscriptionId$">$dashboardBaseUrl$$sub.subscriptionId$</a><br />
or using the <a href="$publicApi$"">Public Broker Service API</a> (the ID of the current subscription is <b>>$sub.subscriptionId$</b>).
</p>
<hr />
This email message was auto-generated. Please do not respond.
This email message was auto-generated. Please do not respond. If you need support, please contact the
<a href="$helpdesk$">OpenAIRE helpdesk</a>.

35
apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/events/output/EmailDispatcherTest.java Normal file
View File

@ -0,0 +1,35 @@
package eu.dnetlib.broker.events.output;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.subscriptions.Subscription;
public class EmailDispatcherTest {
@Test
void testGenerateMailContent() throws Exception {
final EmailDispatcher dispatcher = new EmailDispatcher();
dispatcher.setEmailTemplate(new ClassPathResource("/templates/dhp_openaire_mail.st"));
dispatcher.setDashboardBaseUrl("https://provide.openaire.eu/content/notifications/");
dispatcher.setHelpdesk("https://www.openaire.eu/support/helpdesk");
dispatcher.setPublicApi("https://api.openaire.eu/broker");
final Subscription s = new Subscription();
s.setSubscriptionId("sub-00001");
s.setTopic("ENRICH/MISSING/ABSTRACT");
final Event events = new Event();
final Map<String, Object> params = new HashMap<>();
params.put("oa_notifications_total", 1023);
params.put("oa_datasource", "ISTI OpenPortal");
System.out.println(dispatcher.generateMailContent(s, params, events));
}
}

0
apps/dhp-broker-application/ssh_tunnel_postgres.sh → apps/dhp-broker-application/ssh_tunnel_postgres_beta.sh
View File

3
apps/dhp-broker-application/ssh_tunnel_postgres_prod.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
ssh -vNL 5432:10.19.65.39:5432 michele.artini@iis-cdh5-test-gw.ocean.icm.edu.pl

18
apps/dhp-broker-public-application/pom.xml
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
@ -58,6 +60,20 @@
</repositories>
<build>
<resources>
<resource>
<directory>.</directory>
<includes>
<include>pom.xml</include>
</includes>
</resource>
<resource>
<directory>src/main/java</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>

34
apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/BrokerConfiguration.java
View File

@ -1,7 +1,15 @@
package eu.dnetlib.broker;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.annotation.PostConstruct;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -9,6 +17,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.web.servlet.ServletRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
@ -16,9 +25,13 @@ import org.springframework.data.elasticsearch.repository.config.EnableElasticsea
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import com.google.common.collect.Lists;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import io.micrometer.core.instrument.ImmutableTag;
import io.micrometer.core.instrument.Metrics;
import io.prometheus.client.exporter.MetricsServlet;
import io.prometheus.client.hotspot.DefaultExports;
import springfox.documentation.builders.RequestHandlerSelectors;
@ -42,8 +55,29 @@ public class BrokerConfiguration extends AbstractElasticsearchConfiguration {
@Autowired
private ElasticSearchProperties elasticSearchProperties;
@Value("pom.xml")
private ClassPathResource pom;
public static final String OA_PUBLIC_APIS = "Openaire Broker Public API";
@PostConstruct
public void init() {
final MavenXpp3Reader reader = new MavenXpp3Reader();
try {
final Model model = reader.read(new InputStreamReader(pom.getInputStream()));
log.info(String.format("registering metric for %s", model.getArtifactId()));
final ImmutableTag tag1 = new ImmutableTag("component", model.getGroupId() + ":" + model.getArtifactId());
// final ImmutableTag tag2 = new ImmutableTag("version", model.getVersion());
// final ImmutableTag tag3 = new ImmutableTag("scmtag", model.getScm().getTag());
Metrics.gauge("micrometer_info", Lists.newArrayList(tag1), 1);
} catch (IOException | XmlPullParserException e) {
log.error("Error registering metric", e);
}
}
@Override
@Bean
public RestHighLevelClient elasticsearchClient() {

5
apps/dhp-broker-public-application/src/main/resources/application.properties
View File

@ -3,6 +3,11 @@ spring.profiles.active = dev,openaire
#logging.level.root=DEBUG
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
spring.datasource.url=jdbc:postgresql://localhost:5432/oa_broker
spring.datasource.username=oa_broker
spring.datasource.password=

18
apps/dnet-orgs-database-application/pom.xml
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
@ -84,6 +86,20 @@
</dependencies>
<build>
<resources>
<resource>
<directory>.</directory>
<includes>
<include>pom.xml</include>
</includes>
</resource>
<resource>
<directory>src/main/java</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>

47
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MainConfiguration.java Normal file
View File

@ -0,0 +1,47 @@
package eu.dnetlib.organizations;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.annotation.PostConstruct;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import com.google.common.collect.Lists;
import io.micrometer.core.instrument.ImmutableTag;
import io.micrometer.core.instrument.Metrics;
@Configuration
public class MainConfiguration {
private static final Logger log = LoggerFactory.getLogger(MainConfiguration.class);
@Value("pom.xml")
private ClassPathResource pom;
@PostConstruct
public void init() {
final MavenXpp3Reader reader = new MavenXpp3Reader();
try {
final Model model = reader.read(new InputStreamReader(pom.getInputStream()));
log.info(String.format("registering metric for %s", model.getArtifactId()));
final ImmutableTag tag1 = new ImmutableTag("component", model.getGroupId() + ":" + model.getArtifactId());
// final ImmutableTag tag2 = new ImmutableTag("version", model.getVersion());
// final ImmutableTag tag3 = new ImmutableTag("scmtag", model.getScm().getTag());
Metrics.gauge("micrometer_info", Lists.newArrayList(tag1), 1);
} catch (IOException | XmlPullParserException e) {
log.error("Error registering metric", e);
}
}
}

2
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MockSecurityConfig.java
View File

@ -48,7 +48,7 @@ public class MockSecurityConfig extends WebSecurityConfigurerAdapter {
.hasAnyRole(OpenOrgsConstants.VALID_ROLES)
.antMatchers("/registration_api/**")
.hasRole(OpenOrgsConstants.NOT_AUTORIZED_ROLE)
.antMatchers("/resources/**", "/webjars/**")
.antMatchers("/resources/**", "/webjars/**", "/metrics", "/health")
.permitAll()
.antMatchers("/oa_api/**")
.hasIpAddress(openaireApiValidSubnet)

2
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/OAuth2WebSecurityConfig.java
View File

@ -60,7 +60,7 @@ public class OAuth2WebSecurityConfig extends WebSecurityConfigurerAdapter {
.hasAnyRole(OpenOrgsConstants.VALID_ROLES)
.antMatchers("/registration_api/**")
.hasRole(OpenOrgsConstants.NOT_AUTORIZED_ROLE)
.antMatchers("/", "/resources/**", "/webjars/**")
.antMatchers("/", "/resources/**", "/webjars/**", "/metrics", "/health")
.permitAll()
.antMatchers("/oa_api/**")
.hasIpAddress(openaireApiValidSubnet)

7
apps/dnet-orgs-database-application/src/main/resources/application.properties
View File

@ -7,6 +7,12 @@ spring.main.banner-mode = off
logging.level.root = INFO
#logging.level.org.springframework = DEBUG
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
spring.datasource.url=jdbc:postgresql://localhost:5432/oa_organizations
spring.datasource.username=oa_organizations
spring.datasource.password=
@ -38,3 +44,4 @@ spring.security.oauth2.client.provider.oidc.token-uri = http://localhost
spring.security.oauth2.client.provider.oidc.user-info-uri = http://localhost:8080/openid-connect-server-webapp/userinfo
openaire.override.logout.url =

15
apps/pom.xml
View File

@ -35,6 +35,21 @@
<scope>test</scope>
</dependency>
<!-- for /metrics and /health controllers -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
</dependency>
</dependencies>
<build>

6
pom.xml
View File

@ -157,6 +157,12 @@
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
<version>3.2.3</version>
</dependency>
<!-- JAXB API, java.xml.bind module -->
<dependency>
<groupId>jakarta.xml.bind</groupId>