358 lines
20 KiB
Java
358 lines
20 KiB
Java
|
package eu.dnetlib.usagestats.services;
|
||
|
|
|
||
|
|
import com.fasterxml.jackson.core.JsonGenerator;
|
||
|
|
import com.fasterxml.jackson.core.JsonParser;
|
||
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||
|
|
import com.fasterxml.jackson.databind.ObjectWriter;
|
||
|
|
import eu.dnetlib.usagestats.repositories.UsageStatsRepository;
|
||
|
|
import eu.dnetlib.usagestats.sushilite.domain.ReportItem;
|
||
|
|
import eu.dnetlib.usagestats.sushilite.domain.ReportException;
|
||
|
|
import eu.dnetlib.usagestats.sushilite.domain.ReportResponse;
|
||
|
|
import eu.dnetlib.usagestats.sushilite.domain.ReportResponseWrapper;
|
||
|
|
import java.io.File;
|
||
|
|
import java.io.FileInputStream;
|
||
|
|
import java.io.FileOutputStream;
|
||
|
|
import org.springframework.stereotype.Service;
|
||
|
|
|
||
|
|
import java.text.SimpleDateFormat;
|
||
|
|
import java.util.ArrayList;
|
||
|
|
import java.util.Calendar;
|
||
|
|
import java.util.Date;
|
||
|
|
import java.util.List;
|
||
|
|
import java.util.zip.ZipEntry;
|
||
|
|
import java.util.zip.ZipOutputStream;
|
||
|
|
import org.apache.log4j.Logger;
|
||
|
|
import org.springframework.beans.factory.annotation.Value;
|
||
|
|
|
||
|
|
@Service
|
||
|
|
public class SushiLiteServiceImpl implements SushiLiteService {
|
||
|
|
|
||
|
|
private final UsageStatsRepository usageStatsRepository;
|
||
|
|
|
||
|
|
private final Logger log = Logger.getLogger(this.getClass());
|
||
|
|
|
||
|
|
@Value("${usagestats.redis.hostname}")
|
||
|
|
private String hostname;
|
||
|
|
|
||
|
|
@Value("${usagestats.redis.port}")
|
||
|
|
private int port;
|
||
|
|
|
||
|
|
@Value("${compression.max_number_of_records}")
|
||
|
|
private int compression_max_number_of_records;
|
||
|
|
@Value("${download.folder}")
|
||
|
|
private String download_folder;
|
||
|
|
@Value("${sushi-lite.server}")
|
||
|
|
private String sushi_lite_server;
|
||
|
|
|
||
|
|
String finalcompressedString = "";
|
||
|
|
boolean reportForCompression = false;
|
||
|
|
private int noOfItems = 0;
|
||
|
|
String userAgent;
|
||
|
|
|
||
|
|
public SushiLiteServiceImpl(UsageStatsRepository usageStatsRepository) {
|
||
|
|
this.usageStatsRepository = usageStatsRepository;
|
||
|
|
}
|
||
|
|
|
||
|
|
@Override
|
||
|
|
public ReportResponseWrapper buildReport(String reportName, String release, String requestorId, String beginDate,
|
||
|
|
String endDate, String repositoryIdentifier, String itemIdentifier,
|
||
|
|
String itemDataType, String hasDoi, String granularity, String callback) {
|
||
|
|
|
||
|
|
List<ReportItem> reportItems = new ArrayList<>();
|
||
|
|
List<ReportException> reportExceptions = new ArrayList<>();
|
||
|
|
|
||
|
|
if (!granularity.equalsIgnoreCase("totals") && !granularity.equalsIgnoreCase("monthly")) {
|
||
|
|
reportExceptions.add(new ReportException("3062", "Warning", "Invalid ReportAttribute Value", "Granularity: \'" + granularity + "\' unknown. Defaulting to Monthly"));
|
||
|
|
granularity = "Monthly";
|
||
|
|
}
|
||
|
|
|
||
|
|
Date beginDateParsed;
|
||
|
|
if (!beginDate.equals("")) {
|
||
|
|
beginDateParsed = tryParse(beginDate);
|
||
|
|
if (beginDateParsed != null && (granularity.toLowerCase().equals("monthly") || beginDate.length() == 7)) {
|
||
|
|
Calendar temp = Calendar.getInstance();
|
||
|
|
temp.setTime(beginDateParsed);
|
||
|
|
temp.set(Calendar.DAY_OF_MONTH, temp.getActualMinimum(Calendar.DAY_OF_MONTH));
|
||
|
|
beginDateParsed = temp.getTime();
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
Calendar temp = Calendar.getInstance();
|
||
|
|
temp.add(Calendar.MONTH, -1);
|
||
|
|
temp.set(Calendar.DAY_OF_MONTH, temp.getActualMinimum(Calendar.DAY_OF_MONTH));
|
||
|
|
beginDateParsed = temp.getTime();
|
||
|
|
reportExceptions.add(new ReportException("3021", "Warning", "Unspecified Date Arguments", "Begin Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(beginDateParsed)));
|
||
|
|
}
|
||
|
|
|
||
|
|
Date endDateParsed;
|
||
|
|
if (!endDate.equals("")) {
|
||
|
|
endDateParsed = tryParse(endDate);
|
||
|
|
if (endDateParsed != null && (granularity.toLowerCase().equals("monthly") || endDate.length() == 7)) {
|
||
|
|
Calendar temp = Calendar.getInstance();
|
||
|
|
temp.setTime(endDateParsed);
|
||
|
|
temp.set(Calendar.DAY_OF_MONTH, temp.getActualMaximum(Calendar.DAY_OF_MONTH));
|
||
|
|
endDateParsed = temp.getTime();
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
Calendar temp = Calendar.getInstance();
|
||
|
|
temp.add(Calendar.MONTH, -1);
|
||
|
|
temp.set(Calendar.DAY_OF_MONTH, temp.getActualMaximum(Calendar.DAY_OF_MONTH));
|
||
|
|
endDateParsed = temp.getTime();
|
||
|
|
reportExceptions.add(new ReportException("3021", "Warning", "Unspecified Date Arguments", "End Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed)));
|
||
|
|
}
|
||
|
|
//log.error("dates: " + beginDateParsed.toString() + " - " + endDateParsed.toString());
|
||
|
|
|
||
|
|
if (beginDateParsed == null) {
|
||
|
|
reportExceptions.add(new ReportException("3020", "Error", "Invalid Date Arguments", "Begin Date: " + beginDate + " is not a valid date"));
|
||
|
|
}
|
||
|
|
if (endDateParsed == null) {
|
||
|
|
reportExceptions.add(new ReportException("3020", "Error", "Invalid Date Arguments", "End Date: " + endDate + " is not a valid date"));
|
||
|
|
}
|
||
|
|
if (beginDateParsed != null && endDateParsed != null && !beginDateParsed.before(endDateParsed)) {
|
||
|
|
reportExceptions.add(new ReportException("3020", "Error", "Invalid Date Arguments", "BeginDate \'" + new SimpleDateFormat("yyyy-MM-dd").format(beginDateParsed) + "\' is greater than EndDate \'" + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed) + "\'"));
|
||
|
|
}
|
||
|
|
|
||
|
|
String repoid = "";
|
||
|
|
if (!repositoryIdentifier.equals("")) {
|
||
|
|
repoid = usageStatsRepository.executeRepoId(repositoryIdentifier, reportName.toLowerCase());
|
||
|
|
if (repoid.equals("-1")) {
|
||
|
|
reportExceptions.add(new ReportException("3060", "Error", "Invalid Filter Value", "RepositoryIdentifier: " + repositoryIdentifier + " is not valid"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
String itemid = "";
|
||
|
|
if (!itemIdentifier.equals("")) {
|
||
|
|
String[] split = itemIdentifier.split(":");
|
||
|
|
switch (split[0].toLowerCase()) {
|
||
|
|
case "oid":
|
||
|
|
itemid = itemIdentifier;
|
||
|
|
break;
|
||
|
|
case "doi":
|
||
|
|
itemid = itemIdentifier;
|
||
|
|
break;
|
||
|
|
case "openaire":
|
||
|
|
itemid = itemIdentifier;
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
reportExceptions.add(new ReportException("3060", "Error", "Invalid Filter Value", "ItemIdentifier: " + itemIdentifier + " is not valid"));
|
||
|
|
itemid = "-1";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (itemid.equals("") && repoid.equals("") && !reportName.equalsIgnoreCase("rr1") && !reportName.equalsIgnoreCase("jr1")) {
|
||
|
|
reportExceptions.add(new ReportException("3070", "Error", "Required Filter Missing", "ItemIdentifier or RepositoryIdentifier must be supplied"));
|
||
|
|
}
|
||
|
|
if (reportName.equalsIgnoreCase("ar1")) {
|
||
|
|
if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) {
|
||
|
|
if (!itemid.equals("")) {
|
||
|
|
if (itemDataType.equalsIgnoreCase("") || itemDataType.equalsIgnoreCase("article")) {
|
||
|
|
usageStatsRepository.executeItem(reportItems, itemIdentifier, repoid, "Article", beginDateParsed, endDateParsed, granularity);
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else if (!repoid.equals("")) {
|
||
|
|
usageStatsRepository.executeBatchItems(reportItems, repoid, "Article", beginDateParsed, endDateParsed, granularity);
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else if (reportName.equalsIgnoreCase("br1")) {
|
||
|
|
if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) {
|
||
|
|
if (!itemid.equals("")) {
|
||
|
|
if (itemDataType.equalsIgnoreCase("") || itemDataType.equalsIgnoreCase("book")) {
|
||
|
|
usageStatsRepository.executeItem(reportItems, itemIdentifier, repoid, "Book", beginDateParsed, endDateParsed, granularity);
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else if (!repoid.equals("")) {
|
||
|
|
usageStatsRepository.executeBatchItems(reportItems, repoid, "Book", beginDateParsed, endDateParsed, granularity);
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else if (reportName.equalsIgnoreCase("br2")) {
|
||
|
|
if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) {
|
||
|
|
if (!itemid.equals("")) {
|
||
|
|
if (itemDataType.equalsIgnoreCase("") || itemDataType.equalsIgnoreCase("part of book or chapter of book")) {
|
||
|
|
usageStatsRepository.executeItem(reportItems, itemIdentifier, repoid, "Part of book or chapter of book", beginDateParsed, endDateParsed, granularity);
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else if (!repoid.equals("")) {
|
||
|
|
usageStatsRepository.executeBatchItems(reportItems, repoid, "Part of book or chapter of book", beginDateParsed, endDateParsed, granularity);
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else if (reportName.equalsIgnoreCase("ir1")) {
|
||
|
|
if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) {
|
||
|
|
if (!itemid.equals("")) {
|
||
|
|
usageStatsRepository.executeItem(reportItems, itemIdentifier, repoid, itemDataType, beginDateParsed, endDateParsed, granularity);
|
||
|
|
} else if (!repoid.equals("")) {
|
||
|
|
usageStatsRepository.executeBatchItems(reportItems, repoid, itemDataType, beginDateParsed, endDateParsed, granularity);
|
||
|
|
}
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else if (reportName.equalsIgnoreCase("rr1")) {
|
||
|
|
if (!repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) {
|
||
|
|
usageStatsRepository.executeRepo(reportItems, repoid, itemDataType, beginDateParsed, endDateParsed, granularity);
|
||
|
|
}
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else if (reportName.equalsIgnoreCase("jr1")) {
|
||
|
|
if (!repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) {
|
||
|
|
usageStatsRepository.executeJournal(reportItems, repoid, itemDataType, beginDateParsed, endDateParsed, granularity);
|
||
|
|
}
|
||
|
|
if (reportItems.isEmpty()) {
|
||
|
|
reportExceptions.add(new ReportException("3030", "Error", "No Usage Available for Requested Dates", "Service did not find any data"));
|
||
|
|
}
|
||
|
|
} else if (reportName.equals("")) {
|
||
|
|
reportExceptions.add(new ReportException("3050", "Error", "Report argument is missing", "You must supply a Report argument"));
|
||
|
|
} else {
|
||
|
|
reportExceptions.add(new ReportException("3000", "Error", "Report " + reportName + " not supported", "Supported reports: AR1, IR1, RR1, BR1, BR2"));
|
||
|
|
}
|
||
|
|
|
||
|
|
ReportResponse reportResponse = new ReportResponse(reportName, release, requestorId, beginDate, endDate,
|
||
|
|
repositoryIdentifier, itemIdentifier, itemDataType, hasDoi, granularity, callback, reportItems, reportExceptions);
|
||
|
|
if (reportItems.size() > compression_max_number_of_records) {
|
||
|
|
log.info("Compression due to "+reportItems.size());
|
||
|
|
reportForCompression = true;
|
||
|
|
}
|
||
|
|
|
||
|
|
noOfItems = reportItems.size();
|
||
|
|
return new ReportResponseWrapper(reportResponse);
|
||
|
|
}
|
||
|
|
|
||
|
|
@Override
|
||
|
|
public String displayReport(String reportName, String release, String requestorId, String beginDate, String endDate, String repositoryIdentifier, String itemIdentifier, String itemDataType, String hasDoi, String granularity, String callback, String pretty, String userAgent) {
|
||
|
|
ObjectMapper objectMapper = new ObjectMapper();
|
||
|
|
objectMapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, false);
|
||
|
|
objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
|
||
|
|
java.sql.Timestamp timestamp = new java.sql.Timestamp(System.currentTimeMillis());
|
||
|
|
//System.out.println("Display report start " + timestamp);
|
||
|
|
log.info("Starting process...." + timestamp);
|
||
|
|
this.userAgent = userAgent;
|
||
|
|
try {
|
||
|
|
ReportResponseWrapper reportResponseWrapper = buildReport(reportName, release, requestorId, beginDate, endDate, repositoryIdentifier, itemIdentifier, itemDataType, hasDoi, granularity, callback);
|
||
|
|
if (pretty.equalsIgnoreCase("pretty")) {
|
||
|
|
if (reportForCompression == false) {
|
||
|
|
if (userAgent.contains("Mozilla")) {
|
||
|
|
return "<pre>" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(reportResponseWrapper) + "</pre>";
|
||
|
|
} else {
|
||
|
|
return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(reportResponseWrapper);
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
if (userAgent.contains("Mozilla")) {
|
||
|
|
byte[] sourceReport = objectMapper.writeValueAsBytes(reportResponseWrapper);
|
||
|
|
ObjectWriter writer = objectMapper.writer();
|
||
|
|
java.sql.Timestamp timestamp1 = new java.sql.Timestamp(System.currentTimeMillis());
|
||
|
|
//System.out.println("String start " + timestamp1);
|
||
|
|
log.info("Start building report..." + timestamp1);
|
||
|
|
String outputname = reportName + "_" + repositoryIdentifier.replace("_", "").replace(":", "") + beginDate.replace("-", "") + "_" + endDate.replace("-", "");
|
||
|
|
String directory = new File(download_folder).getAbsolutePath();
|
||
|
|
writer.writeValue(new File(directory + "/" + outputname + ".json"), new String(sourceReport));
|
||
|
|
|
||
|
|
FileOutputStream fos = new FileOutputStream(directory + "/" + outputname + ".zip");
|
||
|
|
ZipOutputStream zipOut = new ZipOutputStream(fos);
|
||
|
|
File fileToZip = new File(directory + "/" + outputname + ".json");
|
||
|
|
FileInputStream fis = new FileInputStream(fileToZip);
|
||
|
|
ZipEntry zipEntry = new ZipEntry(fileToZip.getName());
|
||
|
|
zipOut.putNextEntry(zipEntry);
|
||
|
|
byte[] bytes = new byte[1024];
|
||
|
|
int length;
|
||
|
|
while ((length = fis.read(bytes)) >= 0) {
|
||
|
|
zipOut.write(bytes, 0, length);
|
||
|
|
}
|
||
|
|
zipOut.close();
|
||
|
|
fis.close();
|
||
|
|
fos.close();
|
||
|
|
fileToZip.delete();
|
||
|
|
|
||
|
|
java.sql.Timestamp timestamp2 = new java.sql.Timestamp(System.currentTimeMillis());
|
||
|
|
//System.out.println("String end " + timestamp2);
|
||
|
|
log.info("Report created..." + timestamp2);
|
||
|
|
|
||
|
|
return new String(sushi_lite_server + "/download/" + outputname + ".zip");
|
||
|
|
//return "report is available. Download it from localhost:8080/download/"+outputname+".zip";
|
||
|
|
} else {
|
||
|
|
byte[] sourceReport = objectMapper.writeValueAsBytes(reportResponseWrapper);
|
||
|
|
return new String(sourceReport);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (reportForCompression == false) {
|
||
|
|
byte[] sourceReport = objectMapper.writeValueAsBytes(reportResponseWrapper);
|
||
|
|
return new String(sourceReport);
|
||
|
|
} else if (userAgent.contains("Mozilla")) {
|
||
|
|
|
||
|
|
java.sql.Timestamp timestamp3 = new java.sql.Timestamp(System.currentTimeMillis());
|
||
|
|
//System.out.println("String start " + timestamp3);
|
||
|
|
log.info("Start building report..." + timestamp3);
|
||
|
|
|
||
|
|
byte[] sourceReport = objectMapper.writeValueAsBytes(reportResponseWrapper);
|
||
|
|
ObjectWriter writer = objectMapper.writer();
|
||
|
|
|
||
|
|
String outputname = reportName + "_" + repositoryIdentifier.replace("_", "").replace(":", "") + beginDate.replace("-", "") + "_" + endDate.replace("-", "");
|
||
|
|
|
||
|
|
String directory = new File(download_folder).getAbsolutePath();
|
||
|
|
writer.writeValue(new File(directory + "/" + outputname + ".json"), new String(sourceReport).replaceAll("\\\"", ""));
|
||
|
|
|
||
|
|
FileOutputStream fos = new FileOutputStream(directory + "/" + outputname + ".zip");
|
||
|
|
ZipOutputStream zipOut = new ZipOutputStream(fos);
|
||
|
|
File fileToZip = new File(directory + "/" + outputname + ".json");
|
||
|
|
FileInputStream fis = new FileInputStream(fileToZip);
|
||
|
|
ZipEntry zipEntry = new ZipEntry(fileToZip.getName());
|
||
|
|
zipOut.putNextEntry(zipEntry);
|
||
|
|
byte[] bytes = new byte[1024];
|
||
|
|
int length;
|
||
|
|
while ((length = fis.read(bytes)) >= 0) {
|
||
|
|
zipOut.write(bytes, 0, length);
|
||
|
|
}
|
||
|
|
zipOut.close();
|
||
|
|
fis.close();
|
||
|
|
fos.close();
|
||
|
|
fileToZip.delete();
|
||
|
|
|
||
|
|
java.sql.Timestamp timestamp4 = new java.sql.Timestamp(System.currentTimeMillis());
|
||
|
|
//System.out.println("String end " + timestamp4);
|
||
|
|
log.info("Report created..." + timestamp4);
|
||
|
|
|
||
|
|
return new String(sushi_lite_server + "/download/" + outputname + ".zip");
|
||
|
|
|
||
|
|
} else {
|
||
|
|
byte[] sourceReport = objectMapper.writeValueAsBytes(reportResponseWrapper);
|
||
|
|
return new String(sourceReport);
|
||
|
|
}
|
||
|
|
} catch (Exception e) {
|
||
|
|
e.printStackTrace();
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private Date tryParse(String dateString) {
|
||
|
|
try {
|
||
|
|
if (dateString.length() == 7) {
|
||
|
|
return new SimpleDateFormat("yyyy-MM").parse(dateString);
|
||
|
|
} else if (dateString.length() == 10) {
|
||
|
|
return new SimpleDateFormat("yyyy-MM-dd").parse(dateString);
|
||
|
|
}
|
||
|
|
} catch (Exception e) {
|
||
|
|
log.error("ParseError: ", e);
|
||
|
|
}
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|