commit 969d7f31b9e4302857e3e046c5fcd44a98cca9f6 Author: Katerina Date: Fri Sep 24 14:25:23 2021 +0300 metrics explore service and prometheus formatter diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..39cf934 --- /dev/null +++ b/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.5.4 + + + eu.dnetlib + dnet-explore-metrics + 0.0.1-SNAPSHOT + war + dnet-explore-metrics + Project for exposing Explore metrics + + 1.8 + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-tomcat + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + log4j + log4j + 1.2.17 + + + org.json + json + 20210307 + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/DnetExploreMetricsApplication.java b/src/main/java/eu/dnetlib/dnetexploremetrics/DnetExploreMetricsApplication.java new file mode 100644 index 0000000..1a6acfe --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/DnetExploreMetricsApplication.java @@ -0,0 +1,13 @@ +package eu.dnetlib.dnetexploremetrics; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DnetExploreMetricsApplication { + + public static void main(String[] args) { + SpringApplication.run(DnetExploreMetricsApplication.class, args); + } + +} diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/ServletInitializer.java b/src/main/java/eu/dnetlib/dnetexploremetrics/ServletInitializer.java new file mode 100644 index 0000000..9f9645a --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/ServletInitializer.java @@ -0,0 +1,13 @@ +package eu.dnetlib.dnetexploremetrics; + +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; + +public class ServletInitializer extends SpringBootServletInitializer { + + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(DnetExploreMetricsApplication.class); + } + +} diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/dao/MetricsDAO.java b/src/main/java/eu/dnetlib/dnetexploremetrics/dao/MetricsDAO.java new file mode 100644 index 0000000..678ecd7 --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/dao/MetricsDAO.java @@ -0,0 +1,12 @@ +package eu.dnetlib.dnetexploremetrics.dao; + +import eu.dnetlib.dnetexploremetrics.model.Metrics; + +import java.util.List; + +public interface MetricsDAO { + + public void save(Metrics metrics); + + public List getMetrics(); +} diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/dao/MetricsFileDAO.java b/src/main/java/eu/dnetlib/dnetexploremetrics/dao/MetricsFileDAO.java new file mode 100644 index 0000000..c4ccbed --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/dao/MetricsFileDAO.java @@ -0,0 +1,43 @@ +package eu.dnetlib.dnetexploremetrics.dao; + +import eu.dnetlib.dnetexploremetrics.model.Metrics; +import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +public class MetricsFileDAO implements MetricsDAO { + + private final Logger logger = Logger.getLogger(this.getClass()); + private String fileName; + + @Override + public void save(Metrics metrics) { + try { + Files.write(Paths.get("fileName"), PrometheusMetricsFormatter.formatMetrics(metrics).getBytes()); + + } catch (IOException ioe) { + logger.error("Error writing metrics to file " + fileName + ".", ioe); + System.out.println(ioe); + } + + } + + @Override + public List getMetrics() { + return null; + } + + public static void main(String[] args) { + + MetricsFileDAO mfd = new MetricsFileDAO(); + mfd.fileName="/home/katerina/Desktop/test.txt"; + Metrics m = new Metrics(); + m.publications = 1+""; + mfd.save(m); + } +} diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/model/Metrics.java b/src/main/java/eu/dnetlib/dnetexploremetrics/model/Metrics.java new file mode 100644 index 0000000..ff0a58d --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/model/Metrics.java @@ -0,0 +1,28 @@ +package eu.dnetlib.dnetexploremetrics.model; + +public class Metrics { + + //Records (Number of metadata collections registered/harvested by the OpenAIRE Research Graph) + public String records; + + //Publications (Number of total publications available through OpenAIRE) + public String publications; + + //Datasets (Number of software available through EXPLORE) + public String datasets; + + //Software (Number of software available through EXPLORE) + public String software; + + //OtherReseachProducts (Number of other research products through EXPLORE) + public String orp; + + //Funders (This number of funders on EXPLORE) + public String funders; + + //Projects (Funded projects) + public String projects; + + //Content Providers TODO: define criteria + public String contentProviders; +} diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/service/MetricsService.java b/src/main/java/eu/dnetlib/dnetexploremetrics/service/MetricsService.java new file mode 100644 index 0000000..f06fc52 --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/service/MetricsService.java @@ -0,0 +1,78 @@ +package eu.dnetlib.dnetexploremetrics.service; + +import com.fasterxml.jackson.databind.util.JSONPObject; +import eu.dnetlib.dnetexploremetrics.dao.MetricsFileDAO; +import eu.dnetlib.dnetexploremetrics.model.Metrics; +import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +public class MetricsService { + + @Value("${search.service.url}") + private String searchServiceUrl; + + private final String recordsRequest = "results?size=0&format=json"; + private final String publicationsRequest = "publications?size=0&format=json"; + private final String datasetsRequest = "datasets?size=0&format=json"; + private final String softwareRequest = "software?size=0&format=json"; + private final String orpRequest = "other?size=0&format=json"; + private final String fundersRequest = "resources2?format=json&refine=true&fields=relfunder&type=results&page=0&size=0"; + private final String projectsRequest = "projects?size=0&format=json"; + private final String contentProvidersRequest = "datasources?size=0&format=json"; + + + private void getMetrics(Metrics metrics) { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity response + = restTemplate.getForEntity("https://services.openaire.eu/search/v2/api/" + recordsRequest, String.class); + + JSONObject jsonObject = new JSONObject(response.getBody()); + metrics.records= jsonObject.getJSONObject("meta").get("total").toString(); + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + publicationsRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.publications = jsonObject.getJSONObject("meta").get("total").toString(); + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + datasetsRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.datasets = jsonObject.getJSONObject("meta").get("total").toString(); + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + softwareRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.software = jsonObject.getJSONObject("meta").get("total").toString(); + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + orpRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.orp = jsonObject.getJSONObject("meta").get("total").toString(); + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + fundersRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.funders = jsonObject.getJSONObject("refineResults").getJSONArray("relfunder").length()+""; + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + projectsRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.projects = jsonObject.getJSONObject("meta").get("total").toString(); + + response = restTemplate. + getForEntity("https://services.openaire.eu/search/v2/api/" + contentProvidersRequest, String.class); + jsonObject = new JSONObject(response.getBody()); + metrics.contentProviders = jsonObject.getJSONObject("meta").get("total").toString(); + } + + public static void main(String[] args) { + MetricsService ms = new MetricsService(); + Metrics m = new Metrics(); + ms.getMetrics(m); + PrometheusMetricsFormatter.formatMetrics(m); + } + +} diff --git a/src/main/java/eu/dnetlib/dnetexploremetrics/utils/PrometheusMetricsFormatter.java b/src/main/java/eu/dnetlib/dnetexploremetrics/utils/PrometheusMetricsFormatter.java new file mode 100644 index 0000000..f2655ec --- /dev/null +++ b/src/main/java/eu/dnetlib/dnetexploremetrics/utils/PrometheusMetricsFormatter.java @@ -0,0 +1,40 @@ +package eu.dnetlib.dnetexploremetrics.utils; + +import eu.dnetlib.dnetexploremetrics.model.Metrics; +import org.apache.logging.log4j.message.StringFormattedMessage; + +public class PrometheusMetricsFormatter { + + private enum PrometheusMetrics { + RECORDS("# TYPE explore_total_records gauge\nexplore_total_records %s"), + PUBLICATIONS("# TYPE explore_total_publications gauge\nexplore_total_publications %s"), + DATASETS("# TYPE explore_total_datasets gauge\nexplore_total_datasets %s"), + SOFTWARE("# TYPE explore_total_software gauge\nexplore_total_software %s"), + ORP("# TYPE explore_total_otherresearchproducts gauge\nexplore_total_otherresearchproducts %s"), + FUNDERS("# TYPE explore_total_funders gauge\nexplore_total_funders %s"), + PROJECTS("#TYPE explore_total_projects gauge\nexplore_total_projects %s"), + CONTENT_PROVIDERS("#TYPE explore_total_contentproviders gauge\nexplore_total_contentproviders %s"); + + private String value; + PrometheusMetrics(String value) { + this.value = value; + } + private String getValue() { + return value; + } + } + + public static String formatMetrics(Metrics metrics) { + StringBuilder builder = new StringBuilder(); + builder.append(String.format(String.valueOf(PrometheusMetrics.RECORDS.getValue()), metrics.records)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.PUBLICATIONS.getValue()), metrics.publications)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.DATASETS.getValue()), metrics.datasets)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.SOFTWARE.getValue()), metrics.software)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.ORP.getValue()), metrics.orp)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.FUNDERS.getValue()), metrics.funders)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.PROJECTS.getValue()), metrics.projects)).append("\n"). + append(String.format(String.valueOf(PrometheusMetrics.CONTENT_PROVIDERS.getValue()), metrics.contentProviders)); + System.out.println(builder.toString()); + return builder.toString(); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..a6f09c4 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1 @@ +search.service.url = http://services.openaire.eu/search/ diff --git a/src/test/java/eu/dnetlib/dnetexploremetrics/DnetExploreMetricsApplicationTests.java b/src/test/java/eu/dnetlib/dnetexploremetrics/DnetExploreMetricsApplicationTests.java new file mode 100644 index 0000000..0ae318b --- /dev/null +++ b/src/test/java/eu/dnetlib/dnetexploremetrics/DnetExploreMetricsApplicationTests.java @@ -0,0 +1,13 @@ +package eu.dnetlib.dnetexploremetrics; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DnetExploreMetricsApplicationTests { + + @Test + void contextLoads() { + } + +}