metrics explore service and prometheus formatter
This commit is contained in:
commit
969d7f31b9
|
@ -0,0 +1,60 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-parent</artifactId>
|
||||||
|
<version>2.5.4</version>
|
||||||
|
<relativePath/> <!-- lookup parent from repository -->
|
||||||
|
</parent>
|
||||||
|
<groupId>eu.dnetlib</groupId>
|
||||||
|
<artifactId>dnet-explore-metrics</artifactId>
|
||||||
|
<version>0.0.1-SNAPSHOT</version>
|
||||||
|
<packaging>war</packaging>
|
||||||
|
<name>dnet-explore-metrics</name>
|
||||||
|
<description>Project for exposing Explore metrics</description>
|
||||||
|
<properties>
|
||||||
|
<java.version>1.8</java.version>
|
||||||
|
</properties>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-web</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-tomcat</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-test</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>log4j</groupId>
|
||||||
|
<artifactId>log4j</artifactId>
|
||||||
|
<version>1.2.17</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.json</groupId>
|
||||||
|
<artifactId>json</artifactId>
|
||||||
|
<version>20210307</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</project>
|
|
@ -0,0 +1,13 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics;
|
||||||
|
|
||||||
|
import org.springframework.boot.SpringApplication;
|
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
|
||||||
|
@SpringBootApplication
|
||||||
|
public class DnetExploreMetricsApplication {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
SpringApplication.run(DnetExploreMetricsApplication.class, args);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,13 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics;
|
||||||
|
|
||||||
|
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||||
|
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
|
||||||
|
|
||||||
|
public class ServletInitializer extends SpringBootServletInitializer {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
|
||||||
|
return application.sources(DnetExploreMetricsApplication.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,12 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics.dao;
|
||||||
|
|
||||||
|
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface MetricsDAO {
|
||||||
|
|
||||||
|
public void save(Metrics metrics);
|
||||||
|
|
||||||
|
public List<Metrics> getMetrics();
|
||||||
|
}
|
|
@ -0,0 +1,43 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics.dao;
|
||||||
|
|
||||||
|
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||||
|
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.List;
|
||||||
|
public class MetricsFileDAO implements MetricsDAO {
|
||||||
|
|
||||||
|
private final Logger logger = Logger.getLogger(this.getClass());
|
||||||
|
private String fileName;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void save(Metrics metrics) {
|
||||||
|
try {
|
||||||
|
Files.write(Paths.get("fileName"), PrometheusMetricsFormatter.formatMetrics(metrics).getBytes());
|
||||||
|
|
||||||
|
} catch (IOException ioe) {
|
||||||
|
logger.error("Error writing metrics to file " + fileName + ".", ioe);
|
||||||
|
System.out.println(ioe);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Metrics> getMetrics() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
|
||||||
|
MetricsFileDAO mfd = new MetricsFileDAO();
|
||||||
|
mfd.fileName="/home/katerina/Desktop/test.txt";
|
||||||
|
Metrics m = new Metrics();
|
||||||
|
m.publications = 1+"";
|
||||||
|
mfd.save(m);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,28 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics.model;
|
||||||
|
|
||||||
|
public class Metrics {
|
||||||
|
|
||||||
|
//Records (Number of metadata collections registered/harvested by the OpenAIRE Research Graph)
|
||||||
|
public String records;
|
||||||
|
|
||||||
|
//Publications (Number of total publications available through OpenAIRE)
|
||||||
|
public String publications;
|
||||||
|
|
||||||
|
//Datasets (Number of software available through EXPLORE)
|
||||||
|
public String datasets;
|
||||||
|
|
||||||
|
//Software (Number of software available through EXPLORE)
|
||||||
|
public String software;
|
||||||
|
|
||||||
|
//OtherReseachProducts (Number of other research products through EXPLORE)
|
||||||
|
public String orp;
|
||||||
|
|
||||||
|
//Funders (This number of funders on EXPLORE)
|
||||||
|
public String funders;
|
||||||
|
|
||||||
|
//Projects (Funded projects)
|
||||||
|
public String projects;
|
||||||
|
|
||||||
|
//Content Providers TODO: define criteria
|
||||||
|
public String contentProviders;
|
||||||
|
}
|
|
@ -0,0 +1,78 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.util.JSONPObject;
|
||||||
|
import eu.dnetlib.dnetexploremetrics.dao.MetricsFileDAO;
|
||||||
|
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||||
|
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
||||||
|
import org.json.JSONObject;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
public class MetricsService {
|
||||||
|
|
||||||
|
@Value("${search.service.url}")
|
||||||
|
private String searchServiceUrl;
|
||||||
|
|
||||||
|
private final String recordsRequest = "results?size=0&format=json";
|
||||||
|
private final String publicationsRequest = "publications?size=0&format=json";
|
||||||
|
private final String datasetsRequest = "datasets?size=0&format=json";
|
||||||
|
private final String softwareRequest = "software?size=0&format=json";
|
||||||
|
private final String orpRequest = "other?size=0&format=json";
|
||||||
|
private final String fundersRequest = "resources2?format=json&refine=true&fields=relfunder&type=results&page=0&size=0";
|
||||||
|
private final String projectsRequest = "projects?size=0&format=json";
|
||||||
|
private final String contentProvidersRequest = "datasources?size=0&format=json";
|
||||||
|
|
||||||
|
|
||||||
|
private void getMetrics(Metrics metrics) {
|
||||||
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
|
ResponseEntity<String> response
|
||||||
|
= restTemplate.getForEntity("https://services.openaire.eu/search/v2/api/" + recordsRequest, String.class);
|
||||||
|
|
||||||
|
JSONObject jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.records= jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + publicationsRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.publications = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + datasetsRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.datasets = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + softwareRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.software = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + orpRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.orp = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + fundersRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.funders = jsonObject.getJSONObject("refineResults").getJSONArray("relfunder").length()+"";
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + projectsRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.projects = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + contentProvidersRequest, String.class);
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.contentProviders = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MetricsService ms = new MetricsService();
|
||||||
|
Metrics m = new Metrics();
|
||||||
|
ms.getMetrics(m);
|
||||||
|
PrometheusMetricsFormatter.formatMetrics(m);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,40 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics.utils;
|
||||||
|
|
||||||
|
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||||
|
import org.apache.logging.log4j.message.StringFormattedMessage;
|
||||||
|
|
||||||
|
public class PrometheusMetricsFormatter {
|
||||||
|
|
||||||
|
private enum PrometheusMetrics {
|
||||||
|
RECORDS("# TYPE explore_total_records gauge\nexplore_total_records %s"),
|
||||||
|
PUBLICATIONS("# TYPE explore_total_publications gauge\nexplore_total_publications %s"),
|
||||||
|
DATASETS("# TYPE explore_total_datasets gauge\nexplore_total_datasets %s"),
|
||||||
|
SOFTWARE("# TYPE explore_total_software gauge\nexplore_total_software %s"),
|
||||||
|
ORP("# TYPE explore_total_otherresearchproducts gauge\nexplore_total_otherresearchproducts %s"),
|
||||||
|
FUNDERS("# TYPE explore_total_funders gauge\nexplore_total_funders %s"),
|
||||||
|
PROJECTS("#TYPE explore_total_projects gauge\nexplore_total_projects %s"),
|
||||||
|
CONTENT_PROVIDERS("#TYPE explore_total_contentproviders gauge\nexplore_total_contentproviders %s");
|
||||||
|
|
||||||
|
private String value;
|
||||||
|
PrometheusMetrics(String value) {
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
private String getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String formatMetrics(Metrics metrics) {
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
builder.append(String.format(String.valueOf(PrometheusMetrics.RECORDS.getValue()), metrics.records)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.PUBLICATIONS.getValue()), metrics.publications)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.DATASETS.getValue()), metrics.datasets)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.SOFTWARE.getValue()), metrics.software)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.ORP.getValue()), metrics.orp)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.FUNDERS.getValue()), metrics.funders)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.PROJECTS.getValue()), metrics.projects)).append("\n").
|
||||||
|
append(String.format(String.valueOf(PrometheusMetrics.CONTENT_PROVIDERS.getValue()), metrics.contentProviders));
|
||||||
|
System.out.println(builder.toString());
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
search.service.url = http://services.openaire.eu/search/
|
|
@ -0,0 +1,13 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
|
|
||||||
|
@SpringBootTest
|
||||||
|
class DnetExploreMetricsApplicationTests {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void contextLoads() {
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue