finalizing service and adding organization line
This commit is contained in:
parent
969d7f31b9
commit
2613c1dd8a
6
pom.xml
6
pom.xml
|
@ -43,6 +43,12 @@
|
||||||
<artifactId>json</artifactId>
|
<artifactId>json</artifactId>
|
||||||
<version>20210307</version>
|
<version>20210307</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>commons-io</groupId>
|
||||||
|
<artifactId>commons-io</artifactId>
|
||||||
|
<version>2.11.0</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
package eu.dnetlib.dnetexploremetrics;
|
package eu.dnetlib.dnetexploremetrics;
|
||||||
|
|
||||||
|
import eu.dnetlib.dnetexploremetrics.dao.MetricsDAO;
|
||||||
|
import eu.dnetlib.dnetexploremetrics.dao.MetricsFileDAO;
|
||||||
import org.springframework.boot.SpringApplication;
|
import org.springframework.boot.SpringApplication;
|
||||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
|
||||||
@SpringBootApplication
|
@SpringBootApplication
|
||||||
public class DnetExploreMetricsApplication {
|
public class DnetExploreMetricsApplication {
|
||||||
|
@ -10,4 +13,9 @@ public class DnetExploreMetricsApplication {
|
||||||
SpringApplication.run(DnetExploreMetricsApplication.class, args);
|
SpringApplication.run(DnetExploreMetricsApplication.class, args);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public MetricsDAO metricsDAO() {
|
||||||
|
return new MetricsFileDAO();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
package eu.dnetlib.dnetexploremetrics.controller;
|
||||||
|
|
||||||
|
import eu.dnetlib.dnetexploremetrics.service.MetricsService;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMethod;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
public class MetricsController {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
MetricsService metricsService;
|
||||||
|
|
||||||
|
@RequestMapping(value = "/metrics", method = RequestMethod.GET, produces={"text/plain"})
|
||||||
|
public String getMetrics() {
|
||||||
|
return metricsService.getMetrics();
|
||||||
|
}
|
||||||
|
}
|
|
@ -8,5 +8,5 @@ public interface MetricsDAO {
|
||||||
|
|
||||||
public void save(Metrics metrics);
|
public void save(Metrics metrics);
|
||||||
|
|
||||||
public List<Metrics> getMetrics();
|
public String getMetrics();
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,14 +2,13 @@ package eu.dnetlib.dnetexploremetrics.dao;
|
||||||
|
|
||||||
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||||
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.log4j.Logger;
|
import org.apache.log4j.Logger;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.*;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
import java.util.List;
|
|
||||||
public class MetricsFileDAO implements MetricsDAO {
|
public class MetricsFileDAO implements MetricsDAO {
|
||||||
|
|
||||||
private final Logger logger = Logger.getLogger(this.getClass());
|
private final Logger logger = Logger.getLogger(this.getClass());
|
||||||
|
@ -20,24 +19,32 @@ public class MetricsFileDAO implements MetricsDAO {
|
||||||
try {
|
try {
|
||||||
Files.write(Paths.get("fileName"), PrometheusMetricsFormatter.formatMetrics(metrics).getBytes());
|
Files.write(Paths.get("fileName"), PrometheusMetricsFormatter.formatMetrics(metrics).getBytes());
|
||||||
|
|
||||||
|
ClassLoader classLoader = getClass().getClassLoader();
|
||||||
|
File file = new File(classLoader.getResource(".").getFile() +"/metrics.txt");
|
||||||
|
|
||||||
|
FileWriter fileWriter = new FileWriter(file);
|
||||||
|
PrintWriter printWriter = new PrintWriter(fileWriter);
|
||||||
|
printWriter.print(PrometheusMetricsFormatter.formatMetrics(metrics));
|
||||||
|
printWriter.close();
|
||||||
|
|
||||||
} catch (IOException ioe) {
|
} catch (IOException ioe) {
|
||||||
logger.error("Error writing metrics to file " + fileName + ".", ioe);
|
logger.error("Error writing metrics to file " + fileName + ".", ioe);
|
||||||
System.out.println(ioe);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Metrics> getMetrics() {
|
public String getMetrics() {
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
String metrics="";
|
||||||
|
try {
|
||||||
|
FileInputStream inputStream = new FileInputStream(getClass().getClassLoader().getResource(".").getFile() +"/metrics.txt");
|
||||||
|
metrics = IOUtils.toString(inputStream, "UTF-8");
|
||||||
|
// do something with everything string
|
||||||
|
} catch (IOException ioe) {
|
||||||
|
logger.error("Error reading metrics to file " + fileName + ".", ioe);
|
||||||
|
}
|
||||||
|
|
||||||
MetricsFileDAO mfd = new MetricsFileDAO();
|
return metrics;
|
||||||
mfd.fileName="/home/katerina/Desktop/test.txt";
|
|
||||||
Metrics m = new Metrics();
|
|
||||||
m.publications = 1+"";
|
|
||||||
mfd.save(m);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,9 @@ public class Metrics {
|
||||||
//Projects (Funded projects)
|
//Projects (Funded projects)
|
||||||
public String projects;
|
public String projects;
|
||||||
|
|
||||||
//Content Providers TODO: define criteria
|
//Content Providers
|
||||||
public String contentProviders;
|
public String contentProviders;
|
||||||
|
|
||||||
|
//Organizations
|
||||||
|
public String organizations;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,18 +1,24 @@
|
||||||
package eu.dnetlib.dnetexploremetrics.service;
|
package eu.dnetlib.dnetexploremetrics.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.util.JSONPObject;
|
import com.fasterxml.jackson.databind.util.JSONPObject;
|
||||||
|
import eu.dnetlib.dnetexploremetrics.dao.MetricsDAO;
|
||||||
import eu.dnetlib.dnetexploremetrics.dao.MetricsFileDAO;
|
import eu.dnetlib.dnetexploremetrics.dao.MetricsFileDAO;
|
||||||
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
import eu.dnetlib.dnetexploremetrics.model.Metrics;
|
||||||
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
import eu.dnetlib.dnetexploremetrics.utils.PrometheusMetricsFormatter;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.http.ResponseEntity;
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.web.client.RestTemplate;
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
@EnableScheduling
|
||||||
|
@Service
|
||||||
public class MetricsService {
|
public class MetricsService {
|
||||||
|
|
||||||
@Value("${search.service.url}")
|
@Autowired
|
||||||
private String searchServiceUrl;
|
private MetricsDAO metricsDAO;
|
||||||
|
|
||||||
private final String recordsRequest = "results?size=0&format=json";
|
private final String recordsRequest = "results?size=0&format=json";
|
||||||
private final String publicationsRequest = "publications?size=0&format=json";
|
private final String publicationsRequest = "publications?size=0&format=json";
|
||||||
|
@ -22,9 +28,17 @@ public class MetricsService {
|
||||||
private final String fundersRequest = "resources2?format=json&refine=true&fields=relfunder&type=results&page=0&size=0";
|
private final String fundersRequest = "resources2?format=json&refine=true&fields=relfunder&type=results&page=0&size=0";
|
||||||
private final String projectsRequest = "projects?size=0&format=json";
|
private final String projectsRequest = "projects?size=0&format=json";
|
||||||
private final String contentProvidersRequest = "datasources?size=0&format=json";
|
private final String contentProvidersRequest = "datasources?size=0&format=json";
|
||||||
|
private final String organizationsRequest = "resources2?query=((reldatasourcecompatibilityid exact driver or " +
|
||||||
|
"reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or " +
|
||||||
|
"reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or " +
|
||||||
|
"reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data" +
|
||||||
|
" or reldatasourcecompatibilityid exact hostedBy or relproject=*))&type=organizations&size=0&format=json";
|
||||||
|
|
||||||
|
|
||||||
private void getMetrics(Metrics metrics) {
|
@Scheduled(cron = "0 0 0 * * ?")
|
||||||
|
private void calculateMetrics() {
|
||||||
|
System.out.println("Calculating metrics...");
|
||||||
|
Metrics metrics = new Metrics();
|
||||||
RestTemplate restTemplate = new RestTemplate();
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
ResponseEntity<String> response
|
ResponseEntity<String> response
|
||||||
= restTemplate.getForEntity("https://services.openaire.eu/search/v2/api/" + recordsRequest, String.class);
|
= restTemplate.getForEntity("https://services.openaire.eu/search/v2/api/" + recordsRequest, String.class);
|
||||||
|
@ -66,13 +80,19 @@ public class MetricsService {
|
||||||
getForEntity("https://services.openaire.eu/search/v2/api/" + contentProvidersRequest, String.class);
|
getForEntity("https://services.openaire.eu/search/v2/api/" + contentProvidersRequest, String.class);
|
||||||
jsonObject = new JSONObject(response.getBody());
|
jsonObject = new JSONObject(response.getBody());
|
||||||
metrics.contentProviders = jsonObject.getJSONObject("meta").get("total").toString();
|
metrics.contentProviders = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
System.out.println("https://services.openaire.eu/search/v2/api/" + organizationsRequest);
|
||||||
|
response = restTemplate.
|
||||||
|
getForEntity("https://services.openaire.eu/search/v2/api/" + organizationsRequest, String.class);
|
||||||
|
|
||||||
|
jsonObject = new JSONObject(response.getBody());
|
||||||
|
metrics.organizations = jsonObject.getJSONObject("meta").get("total").toString();
|
||||||
|
|
||||||
|
metricsDAO.save(metrics);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public String getMetrics() {
|
||||||
MetricsService ms = new MetricsService();
|
MetricsDAO metricsDAO = new MetricsFileDAO();
|
||||||
Metrics m = new Metrics();
|
return metricsDAO.getMetrics();
|
||||||
ms.getMetrics(m);
|
|
||||||
PrometheusMetricsFormatter.formatMetrics(m);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,7 +13,8 @@ public class PrometheusMetricsFormatter {
|
||||||
ORP("# TYPE explore_total_otherresearchproducts gauge\nexplore_total_otherresearchproducts %s"),
|
ORP("# TYPE explore_total_otherresearchproducts gauge\nexplore_total_otherresearchproducts %s"),
|
||||||
FUNDERS("# TYPE explore_total_funders gauge\nexplore_total_funders %s"),
|
FUNDERS("# TYPE explore_total_funders gauge\nexplore_total_funders %s"),
|
||||||
PROJECTS("#TYPE explore_total_projects gauge\nexplore_total_projects %s"),
|
PROJECTS("#TYPE explore_total_projects gauge\nexplore_total_projects %s"),
|
||||||
CONTENT_PROVIDERS("#TYPE explore_total_contentproviders gauge\nexplore_total_contentproviders %s");
|
CONTENT_PROVIDERS("#TYPE explore_total_contentproviders gauge\nexplore_total_contentproviders %s"),
|
||||||
|
ORGANIZATIONS("#TYPE explore_total_organizations gauge \nexplore_total_organizations %s");
|
||||||
|
|
||||||
private String value;
|
private String value;
|
||||||
PrometheusMetrics(String value) {
|
PrometheusMetrics(String value) {
|
||||||
|
@ -33,8 +34,8 @@ public class PrometheusMetricsFormatter {
|
||||||
append(String.format(String.valueOf(PrometheusMetrics.ORP.getValue()), metrics.orp)).append("\n").
|
append(String.format(String.valueOf(PrometheusMetrics.ORP.getValue()), metrics.orp)).append("\n").
|
||||||
append(String.format(String.valueOf(PrometheusMetrics.FUNDERS.getValue()), metrics.funders)).append("\n").
|
append(String.format(String.valueOf(PrometheusMetrics.FUNDERS.getValue()), metrics.funders)).append("\n").
|
||||||
append(String.format(String.valueOf(PrometheusMetrics.PROJECTS.getValue()), metrics.projects)).append("\n").
|
append(String.format(String.valueOf(PrometheusMetrics.PROJECTS.getValue()), metrics.projects)).append("\n").
|
||||||
append(String.format(String.valueOf(PrometheusMetrics.CONTENT_PROVIDERS.getValue()), metrics.contentProviders));
|
append(String.format(String.valueOf(PrometheusMetrics.CONTENT_PROVIDERS.getValue()), metrics.contentProviders)).append("\n").
|
||||||
System.out.println(builder.toString());
|
append(String.format(String.valueOf(PrometheusMetrics.ORGANIZATIONS.getValue()),metrics.organizations));
|
||||||
return builder.toString();
|
return builder.toString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue