metrics explore service and prometheus formatter
Katerina
3 years ago
commit
969d7f31b9
@ -0,0 +1,60 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.5.4</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
</parent>
|
||||
<groupId>eu.dnetlib</groupId>
|
||||
<artifactId>dnet-explore-metrics</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<packaging>war</packaging>
|
||||
<name>dnet-explore-metrics</name>
|
||||
<description>Project for exposing Explore metrics</description>
|
||||
<properties>
|
||||
<java.version>1.8</java.version>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-tomcat</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20210307</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,13 @@
|
||||
package eu.dnetlib.dnetexploremetrics;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class DnetExploreMetricsApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(DnetExploreMetricsApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package eu.dnetlib.dnetexploremetrics;
|
||||
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
|
||||
|
||||
public class ServletInitializer extends SpringBootServletInitializer {
|
||||
|
||||
@Override
|
||||
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
|
||||
return application.sources(DnetExploreMetricsApplication.class);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package eu.dnetlib.dnetexploremetrics.dao;
|
||||
|
||||
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface MetricsDAO {
|
||||
|
||||
public void save(Metrics metrics);
|
||||
|
||||
public List<Metrics> getMetrics();
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
package eu.dnetlib.dnetexploremetrics.dao;
|
||||
|
||||
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
public class MetricsFileDAO implements MetricsDAO {
|
||||
|
||||
private final Logger logger = Logger.getLogger(this.getClass());
|
||||
private String fileName;
|
||||
|
||||
@Override
|
||||
public void save(Metrics metrics) {
|
||||
try {
|
||||
Files.write(Paths.get("fileName"), PrometheusMetricsFormatter.formatMetrics(metrics).getBytes());
|
||||
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error writing metrics to file " + fileName + ".", ioe);
|
||||
System.out.println(ioe);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Metrics> getMetrics() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
MetricsFileDAO mfd = new MetricsFileDAO();
|
||||
mfd.fileName="/home/katerina/Desktop/test.txt";
|
||||
Metrics m = new Metrics();
|
||||
m.publications = 1+"";
|
||||
mfd.save(m);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package eu.dnetlib.dnetexploremetrics.model;
|
||||
|
||||
public class Metrics {
|
||||
|
||||
//Records (Number of metadata collections registered/harvested by the OpenAIRE Research Graph)
|
||||
public String records;
|
||||
|
||||
//Publications (Number of total publications available through OpenAIRE)
|
||||
public String publications;
|
||||
|
||||
//Datasets (Number of software available through EXPLORE)
|
||||
public String datasets;
|
||||
|
||||
//Software (Number of software available through EXPLORE)
|
||||
public String software;
|
||||
|
||||
//OtherReseachProducts (Number of other research products through EXPLORE)
|
||||
public String orp;
|
||||
|
||||
//Funders (This number of funders on EXPLORE)
|
||||
public String funders;
|
||||
|
||||
//Projects (Funded projects)
|
||||
public String projects;
|
||||
|
||||
//Content Providers TODO: define criteria
|
||||
public String contentProviders;
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
package eu.dnetlib.dnetexploremetrics.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.util.JSONPObject;
|
||||
import eu.dnetlib.dnetexploremetrics.dao.MetricsFileDAO;
|
||||
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
||||
import org.json.JSONObject;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
public class MetricsService {
|
||||
|
||||
@Value("${search.service.url}")
|
||||
private String searchServiceUrl;
|
||||
|
||||
private final String recordsRequest = "results?size=0&format=json";
|
||||
private final String publicationsRequest = "publications?size=0&format=json";
|
||||
private final String datasetsRequest = "datasets?size=0&format=json";
|
||||
private final String softwareRequest = "software?size=0&format=json";
|
||||
private final String orpRequest = "other?size=0&format=json";
|
||||
private final String fundersRequest = "resources2?format=json&refine=true&fields=relfunder&type=results&page=0&size=0";
|
||||
private final String projectsRequest = "projects?size=0&format=json";
|
||||
private final String contentProvidersRequest = "datasources?size=0&format=json";
|
||||
|
||||
|
||||
private void getMetrics(Metrics metrics) {
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
ResponseEntity<String> response
|
||||
= restTemplate.getForEntity("https://services.openaire.eu/search/v2/api/" + recordsRequest, String.class);
|
||||
|
||||
JSONObject jsonObject = new JSONObject(response.getBody());
|
||||
metrics.records= jsonObject.getJSONObject("meta").get("total").toString();
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + publicationsRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.publications = jsonObject.getJSONObject("meta").get("total").toString();
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + datasetsRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.datasets = jsonObject.getJSONObject("meta").get("total").toString();
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + softwareRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.software = jsonObject.getJSONObject("meta").get("total").toString();
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + orpRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.orp = jsonObject.getJSONObject("meta").get("total").toString();
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + fundersRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.funders = jsonObject.getJSONObject("refineResults").getJSONArray("relfunder").length()+"";
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + projectsRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.projects = jsonObject.getJSONObject("meta").get("total").toString();
|
||||
|
||||
response = restTemplate.
|
||||
getForEntity("https://services.openaire.eu/search/v2/api/" + contentProvidersRequest, String.class);
|
||||
jsonObject = new JSONObject(response.getBody());
|
||||
metrics.contentProviders = jsonObject.getJSONObject("meta").get("total").toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
MetricsService ms = new MetricsService();
|
||||
Metrics m = new Metrics();
|
||||
ms.getMetrics(m);
|
||||
PrometheusMetricsFormatter.formatMetrics(m);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package eu.dnetlib.dnetexploremetrics.utils;
|
||||
|
||||
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||
import org.apache.logging.log4j.message.StringFormattedMessage;
|
||||
|
||||
public class PrometheusMetricsFormatter {
|
||||
|
||||
private enum PrometheusMetrics {
|
||||
RECORDS("# TYPE explore_total_records gauge\nexplore_total_records %s"),
|
||||
PUBLICATIONS("# TYPE explore_total_publications gauge\nexplore_total_publications %s"),
|
||||
DATASETS("# TYPE explore_total_datasets gauge\nexplore_total_datasets %s"),
|
||||
SOFTWARE("# TYPE explore_total_software gauge\nexplore_total_software %s"),
|
||||
ORP("# TYPE explore_total_otherresearchproducts gauge\nexplore_total_otherresearchproducts %s"),
|
||||
FUNDERS("# TYPE explore_total_funders gauge\nexplore_total_funders %s"),
|
||||
PROJECTS("#TYPE explore_total_projects gauge\nexplore_total_projects %s"),
|
||||
CONTENT_PROVIDERS("#TYPE explore_total_contentproviders gauge\nexplore_total_contentproviders %s");
|
||||
|
||||
private String value;
|
||||
PrometheusMetrics(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
private String getValue() {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
public static String formatMetrics(Metrics metrics) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(String.format(String.valueOf(PrometheusMetrics.RECORDS.getValue()), metrics.records)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.PUBLICATIONS.getValue()), metrics.publications)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.DATASETS.getValue()), metrics.datasets)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.SOFTWARE.getValue()), metrics.software)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.ORP.getValue()), metrics.orp)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.FUNDERS.getValue()), metrics.funders)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.PROJECTS.getValue()), metrics.projects)).append("\n").
|
||||
append(String.format(String.valueOf(PrometheusMetrics.CONTENT_PROVIDERS.getValue()), metrics.contentProviders));
|
||||
System.out.println(builder.toString());
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
search.service.url = http://services.openaire.eu/search/
|
||||
@ -0,0 +1,13 @@
|
||||
package eu.dnetlib.dnetexploremetrics;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
||||
@SpringBootTest
|
||||
class DnetExploreMetricsApplicationTests {
|
||||
|
||||
@Test
|
||||
void contextLoads() {
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue