Changes 15092022
This commit is contained in:
parent
f02fc236f2
commit
82632de1df
|
@ -1 +1,3 @@
|
||||||
/target/
|
/target/
|
||||||
|
/.idea/
|
||||||
|
*.iml
|
16
pom.xml
16
pom.xml
|
@ -112,17 +112,11 @@
|
||||||
<artifactId>commons-dbutils</artifactId>
|
<artifactId>commons-dbutils</artifactId>
|
||||||
<version>1.7</version>
|
<version>1.7</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>com.cloudera</groupId>
|
|
||||||
<artifactId>ImpalaJDBC41</artifactId>
|
|
||||||
<version>2.6.3</version>
|
|
||||||
<type>jar</type>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.cloudera.impala</groupId>
|
<groupId>com.cloudera.impala</groupId>
|
||||||
<artifactId>jdbc</artifactId>
|
<artifactId>jdbc</artifactId>
|
||||||
<version>2.5.31</version>
|
<version>2.5.31</version>
|
||||||
<scope>runtime</scope>
|
<scope>runtime</scope>
|
||||||
<exclusions>
|
<exclusions>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
|
@ -154,7 +148,7 @@
|
||||||
</exclusion>
|
</exclusion>
|
||||||
</exclusions>
|
</exclusions>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>javax.servlet</groupId>
|
<groupId>javax.servlet</groupId>
|
||||||
<artifactId>javax.servlet-api</artifactId>
|
<artifactId>javax.servlet-api</artifactId>
|
||||||
<version>3.1.0</version>
|
<version>3.1.0</version>
|
||||||
|
|
0
src/main/java/eu/dnetlib/usagestats/config/DataSourceConfiguration.java
Normal file → Executable file
0
src/main/java/eu/dnetlib/usagestats/config/DataSourceConfiguration.java
Normal file → Executable file
0
src/main/java/eu/dnetlib/usagestats/config/SpringRedisConfiguration.java
Normal file → Executable file
0
src/main/java/eu/dnetlib/usagestats/config/SpringRedisConfiguration.java
Normal file → Executable file
|
@ -2,11 +2,14 @@ package eu.dnetlib.usagestats.controllers;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import eu.dnetlib.usagestats.services.SushiLiteService;
|
import eu.dnetlib.usagestats.services.SushiLiteService;
|
||||||
|
import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Error_Model;
|
||||||
import java.io.BufferedInputStream;
|
import java.io.BufferedInputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.time.ZonedDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
@ -92,8 +95,11 @@ class SushiLiteController {
|
||||||
if (report.indexOf(".zip") < 0) {
|
if (report.indexOf(".zip") < 0) {
|
||||||
return new ResponseEntity<>(report, HttpStatus.OK);
|
return new ResponseEntity<>(report, HttpStatus.OK);
|
||||||
} else {
|
} else {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
String compressedOutput = "<pre> {\"Report\":\"IR\", \"Description\":\"Compressed Report Due to large number of records\", \"URL To Download Report from: \":\"" + report + "\"} </pre>";
|
//String compressedOutput = "<pre> {\"Report\":\"IR\", \"Description\":\"Compressed Report Due to large number of records\", \"URL To Download Report from: \":\"" + report + "\"} </pre>";
|
||||||
|
SUSHI_Error_Model errorModel = new SUSHI_Error_Model("100000", "Notice", "Compressed IR report due to a large number of records",
|
||||||
|
null, report);
|
||||||
|
String compressedOutput = "<pre>" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorModel) + "</pre>";
|
||||||
return new ResponseEntity<>(compressedOutput, HttpStatus.OK);
|
return new ResponseEntity<>(compressedOutput, HttpStatus.OK);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
12
src/main/java/eu/dnetlib/usagestats/controllers/UsageStatsController.java
Normal file → Executable file
12
src/main/java/eu/dnetlib/usagestats/controllers/UsageStatsController.java
Normal file → Executable file
|
@ -11,11 +11,7 @@ import eu.dnetlib.usagestats.portal.UsageStats;
|
||||||
import eu.dnetlib.usagestats.services.UsageStatsService;
|
import eu.dnetlib.usagestats.services.UsageStatsService;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import org.apache.log4j.Logger;
|
import org.apache.log4j.Logger;
|
||||||
import org.springframework.web.bind.annotation.CrossOrigin;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.springframework.web.bind.annotation.PathVariable;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMethod;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
|
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
|
@ -93,6 +89,12 @@ public class UsageStatsController {
|
||||||
return usageStatsService.getResultClicks(resultId);
|
return usageStatsService.getResultClicks(resultId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = "/resultspid", method = RequestMethod.GET)
|
||||||
|
public UsageStats getResultClicksByPID(@RequestParam(value = "doi", defaultValue = "") String doi) {
|
||||||
|
log.info("stats request for pid: " + doi);
|
||||||
|
return usageStatsService.getResultClicksByPID(doi);
|
||||||
|
}
|
||||||
|
|
||||||
@RequestMapping(value = "/allmetrics")
|
@RequestMapping(value = "/allmetrics")
|
||||||
public TotalStatsReposViewsDownloads getTotalStatsReposViewsDownloads() {
|
public TotalStatsReposViewsDownloads getTotalStatsReposViewsDownloads() {
|
||||||
log.info("total stats request");
|
log.info("total stats request");
|
||||||
|
|
0
src/main/java/eu/dnetlib/usagestats/portal/CountryUsageStatsAll.java
Normal file → Executable file
0
src/main/java/eu/dnetlib/usagestats/portal/CountryUsageStatsAll.java
Normal file → Executable file
0
src/main/java/eu/dnetlib/usagestats/portal/TotalStatsReposViewsDownloads.java
Normal file → Executable file
0
src/main/java/eu/dnetlib/usagestats/portal/TotalStatsReposViewsDownloads.java
Normal file → Executable file
File diff suppressed because it is too large
Load Diff
|
@ -316,7 +316,7 @@ public class SushiLiteServiceImpl implements SushiLiteService {
|
||||||
SUSHI_Report_List r4 = new SUSHI_Report_List("Platform Master Report PR", "PR", "5", "A customizable report summarizing activity across a provider’s platforms that allows the user to apply filters and select other configuration options for the report. ", "/PR");
|
SUSHI_Report_List r4 = new SUSHI_Report_List("Platform Master Report PR", "PR", "5", "A customizable report summarizing activity across a provider’s platforms that allows the user to apply filters and select other configuration options for the report. ", "/PR");
|
||||||
SUSHI_Report_List r5 = new SUSHI_Report_List("Platform Usage Report", "PR_P1", "5", "Standard View of the Package Master Report that presents usage for the overall Platform broken down by Metric_Type.", "/PR_1");
|
SUSHI_Report_List r5 = new SUSHI_Report_List("Platform Usage Report", "PR_P1", "5", "Standard View of the Package Master Report that presents usage for the overall Platform broken down by Metric_Type.", "/PR_1");
|
||||||
SUSHI_Report_List r6 = new SUSHI_Report_List("Platform Item Report", "IR", "5", "COUNTER Item Master Report", "/IR");
|
SUSHI_Report_List r6 = new SUSHI_Report_List("Platform Item Report", "IR", "5", "COUNTER Item Master Report", "/IR");
|
||||||
SUSHI_Report_List r7 = new SUSHI_Report_List("Datasets Report", "IR", "5", "COUNTER Datasets Report", "/DSR");
|
SUSHI_Report_List r7 = new SUSHI_Report_List("Datasets Report", "DSR", "5", "COUNTER Datasets Report", "/DSR");
|
||||||
|
|
||||||
reportSupportedList.add(r1);
|
reportSupportedList.add(r1);
|
||||||
reportSupportedList.add(r2);
|
reportSupportedList.add(r2);
|
||||||
|
@ -596,6 +596,28 @@ public class SushiLiteServiceImpl implements SushiLiteService {
|
||||||
reportExceptions.add(new SUSHI_Error_Model("3021", "Warning", "Unspecified Date Arguments", "usagecounts.openaire.eu", "End Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed)));
|
reportExceptions.add(new SUSHI_Error_Model("3021", "Warning", "Unspecified Date Arguments", "usagecounts.openaire.eu", "End Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Calucalte time difference
|
||||||
|
// in milliseconds
|
||||||
|
beginDateParsed = tryParse(beginDate);
|
||||||
|
Calendar temp = Calendar.getInstance();
|
||||||
|
temp.setTime(beginDateParsed);
|
||||||
|
temp.set(Calendar.DAY_OF_MONTH, temp.getActualMaximum(Calendar.DAY_OF_MONTH));
|
||||||
|
beginDateParsed = temp.getTime();
|
||||||
|
|
||||||
|
|
||||||
|
endDateParsed = tryParse(endDate);
|
||||||
|
Calendar temp1 = Calendar.getInstance();
|
||||||
|
temp1.setTime(endDateParsed);
|
||||||
|
temp1.set(Calendar.DAY_OF_MONTH, temp1.getActualMaximum(Calendar.DAY_OF_MONTH));
|
||||||
|
endDateParsed = temp1.getTime();
|
||||||
|
|
||||||
|
long difference_In_Time
|
||||||
|
= endDateParsed.getTime() - beginDateParsed.getTime();
|
||||||
|
long difference_In_Years = (difference_In_Time/ (1000 * 60 * 60 * 24));
|
||||||
|
|
||||||
|
if(difference_In_Years>365)
|
||||||
|
reportExceptions.add(new SUSHI_Error_Model("4000", "Notice", "Requested Period for more than a year not allowed", "usagecounts.openaire.eu", "Contact usagecounts@openaire.eu for longer period"));
|
||||||
// if (beginDateParsed == null) {
|
// if (beginDateParsed == null) {
|
||||||
// reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "Begin Date: " + beginDate + " is not a valid date"));
|
// reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "Begin Date: " + beginDate + " is not a valid date"));
|
||||||
// }
|
// }
|
||||||
|
@ -809,7 +831,10 @@ public class SushiLiteServiceImpl implements SushiLiteService {
|
||||||
//reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data"));
|
//reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||||||
//}
|
//}
|
||||||
} else if (!repoid.equals("")) {
|
} else if (!repoid.equals("")) {
|
||||||
usageStatsRepository.executeBatchItemsDSR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, granularity);
|
if(usageStatsRepository.checkIfDatacite(repoid))
|
||||||
|
usageStatsRepository.executeBatchItemsDSRDatacite(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, granularity);
|
||||||
|
else
|
||||||
|
usageStatsRepository.executeBatchItemsDSR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, granularity);
|
||||||
if (reportItems.isEmpty()) {
|
if (reportItems.isEmpty()) {
|
||||||
reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data"));
|
reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ public interface UsageStatsService {
|
||||||
UsageStats getDatasourceClicks(String id);
|
UsageStats getDatasourceClicks(String id);
|
||||||
UsageStats getProjectClicks(String id);
|
UsageStats getProjectClicks(String id);
|
||||||
UsageStats getResultClicks(String id);
|
UsageStats getResultClicks(String id);
|
||||||
|
UsageStats getResultClicksByPID(String resultPID);
|
||||||
//UsageStats getOrganizationClicks(String id);
|
//UsageStats getOrganizationClicks(String id);
|
||||||
TotalStats getTotalStats();
|
TotalStats getTotalStats();
|
||||||
List<MonthlyUsageStats> getMonthlyUsageStats();
|
List<MonthlyUsageStats> getMonthlyUsageStats();
|
||||||
|
|
18
src/main/java/eu/dnetlib/usagestats/services/UsageStatsServiceImpl.java
Normal file → Executable file
18
src/main/java/eu/dnetlib/usagestats/services/UsageStatsServiceImpl.java
Normal file → Executable file
|
@ -86,6 +86,24 @@ public class UsageStatsServiceImpl implements UsageStatsService {
|
||||||
return usageStatsRepository.executeUsageStats(query, values, "result");
|
return usageStatsRepository.executeUsageStats(query, values, "result");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public UsageStats getResultClicksByPID(String resultPID) {
|
||||||
|
String query = "SELECT 'views', s.repository_id, CASE WHEN s.source='OpenAIRE' THEN d.name ELSE concat(d.name,' - ',s.source) END, sum(count), sum(openaire) "
|
||||||
|
+ "FROM (select distinct * from "+usagestatsImpalaDB+".views_stats) s, "+statsDB+".datasource d WHERE s.repository_id=d.id "
|
||||||
|
+ "AND s.result_id in (select distinct id from "+ statsDB+".result_oids WHERE oid=?) GROUP BY s.source, s.repository_id, d.name "
|
||||||
|
+ "UNION ALL SELECT 'downloads', s.repository_id, CASE WHEN s.source='OpenAIRE' THEN d.name ELSE concat(d.name,' - ',s.source) END, sum(count), sum(s.openaire) "
|
||||||
|
+ "FROM (select distinct * from "+usagestatsImpalaDB+".downloads_stats) s, "+statsDB+".datasource d WHERE s.repository_id=d.id "
|
||||||
|
+ "AND s.result_id in (select distinct id from "+ statsDB+".result_oids WHERE oid=?) GROUP BY s.source, s.repository_id, d.name "
|
||||||
|
+ "UNION ALL SELECT 'pageviews', 'OpenAIRE id', 'OpenAIRE', sum(count), 0 FROM "+usagestatsImpalaDB+".pageviews_stats s "
|
||||||
|
+ "WHERE result_id in (select distinct id from "+ statsDB+".result_oids WHERE oid=?);";
|
||||||
|
|
||||||
|
List<String> values = new ArrayList<>();
|
||||||
|
values.add(resultPID);
|
||||||
|
values.add(resultPID);
|
||||||
|
values.add(resultPID);
|
||||||
|
|
||||||
|
return usageStatsRepository.executeUsageStats(query, values, "result");
|
||||||
|
}
|
||||||
@Override
|
@Override
|
||||||
public TotalStats getTotalStats() {
|
public TotalStats getTotalStats() {
|
||||||
return usageStatsRepository.executeTotalStats();
|
return usageStatsRepository.executeTotalStats();
|
||||||
|
|
|
@ -10,9 +10,9 @@ log4j.additivity.org.springframework = false
|
||||||
log4j.appender.R=org.apache.log4j.RollingFileAppender
|
log4j.appender.R=org.apache.log4j.RollingFileAppender
|
||||||
#log4j.appender.R.File=/var/log/dnet/usageStatsAPI/usageStatsAPI_R5.log
|
#log4j.appender.R.File=/var/log/dnet/usageStatsAPI/usageStatsAPI_R5.log
|
||||||
#log4j.appender.R.File=/home/dimitris.pierrakos/Sushilite_R5/usageStatsAPI_R5.log
|
#log4j.appender.R.File=/home/dimitris.pierrakos/Sushilite_R5/usageStatsAPI_R5.log
|
||||||
#log4j.appender.R.File=/Users/dpie/Desktop/usageStatsAPI_R5.log
|
log4j.appender.R.File=/Users/dpie/OneDrive/Research/Projects/OpenAIRE Usage Statistics Service/Code/code-repo.d4science.org/Sushilite-R5/usageStatsAPI_R5.log
|
||||||
#log4j.appender.R.File=/srv/springboot/9880/usageStatsAPI_R5.log
|
#log4j.appender.R.File=/srv/springboot/9880/usageStatsAPI_R5.log
|
||||||
log4j.appender.R.File=/home/dpie/Desktop/usageStatsAPI_R5.log
|
#log4j.appender.R.File=/home/dpie/Desktop/usageStatsAPI_R5.log
|
||||||
loglog4j.appender.R.MaxFileSize=10MB
|
loglog4j.appender.R.MaxFileSize=10MB
|
||||||
log4j.appender.R.MaxBackupIndex=10
|
log4j.appender.R.MaxBackupIndex=10
|
||||||
log4j.appender.R.layout=org.apache.log4j.PatternLayout
|
log4j.appender.R.layout=org.apache.log4j.PatternLayout
|
||||||
|
@ -20,10 +20,10 @@ log4j.appender.R.layout.ConversionPattern= %d %p %t [%c] - %m%n
|
||||||
log4j.appender.S=org.apache.log4j.RollingFileAppender
|
log4j.appender.S=org.apache.log4j.RollingFileAppender
|
||||||
#log4j.appender.S.File=/var/log/dnet/usageStatsAPI/usageStatsAPI_R5-spring.log
|
#log4j.appender.S.File=/var/log/dnet/usageStatsAPI/usageStatsAPI_R5-spring.log
|
||||||
#log4j.appender.S.File=/home/dimitris.pierrakos/Sushilite_R5/usageStatsAPI_R5-spring.log
|
#log4j.appender.S.File=/home/dimitris.pierrakos/Sushilite_R5/usageStatsAPI_R5-spring.log
|
||||||
#log4j.appender.S.File=/Users/dpie/Desktop/usageStatsAPI_R5-spring.log
|
log4j.appender.S.File=/Users/dpie/OneDrive/Research/Projects/OpenAIRE Usage Statistics Service/Code/code-repo.d4science.org/Sushilite-R5/usageStatsAPI_R5-spring.log
|
||||||
#log4j.appender.S.File=/Volumes/Zeus/dpie/Desktop/usageStatsAPI_R5-spring.log
|
#log4j.appender.S.File=/Volumes/Zeus/dpie/Desktop/usageStatsAPI_R5-spring.log
|
||||||
#log4j.appender.S.File=/srv/springboot/9880/usageStatsAPI_R5-spring.log
|
#log4j.appender.S.File=/srv/springboot/9880/usageStatsAPI_R5-spring.log
|
||||||
log4j.appender.S.File=/home/dpie/Desktop/usageStatsAPI_R5-spring.log
|
#log4j.appender.S.File=/home/dpie/Desktop/usageStatsAPI_R5-spring.log
|
||||||
log4j.appender.S.MaxFileSize=10MB
|
log4j.appender.S.MaxFileSize=10MB
|
||||||
log4j.appender.S.MaxBackupIndex=10
|
log4j.appender.S.MaxBackupIndex=10
|
||||||
log4j.appender.S.layout=org.apache.log4j.PatternLayout
|
log4j.appender.S.layout=org.apache.log4j.PatternLayout
|
||||||
|
|
|
@ -12,7 +12,7 @@ spring.jackson.serialization.INDENT_OUTPUT=true
|
||||||
prod.statsdb=openaire_prod_stats
|
prod.statsdb=openaire_prod_stats
|
||||||
prod.usagestatsImpalaDB=openaire_prod_usage_stats
|
prod.usagestatsImpalaDB=openaire_prod_usage_stats
|
||||||
compression.max_number_of_records=100
|
compression.max_number_of_records=100
|
||||||
download.folder=/Users/dpie/Desktop/DownloadSushiReports
|
download.folder=/Users/dpie/Desktop/DownloadSushiReports/
|
||||||
sushi-lite.server=http:://localhost/usagestats_r5
|
sushi-lite.server=http:://localhost/usagestats_r5
|
||||||
server.servlet.context-path=/usagestats_r5
|
server.servlet.context-path=/usagestats_r5
|
||||||
#server.port=8080
|
#server.port=8080
|
||||||
|
|
Loading…
Reference in New Issue