diff --git a/runme.sh b/runme.sh new file mode 100755 index 0000000..fa80ebf --- /dev/null +++ b/runme.sh @@ -0,0 +1 @@ +java -jar target/usagestats_r5.jar --spring.config.location=file:./usageStatsAPI.properties & \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/usagestats/UsageStatsApi.java b/src/main/java/eu/dnetlib/usagestats/UsageStatsApi.java index cfc01eb..3ef05a6 100644 --- a/src/main/java/eu/dnetlib/usagestats/UsageStatsApi.java +++ b/src/main/java/eu/dnetlib/usagestats/UsageStatsApi.java @@ -6,10 +6,10 @@ import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySources; import org.springframework.retry.annotation.EnableRetry; -@PropertySources({ - @PropertySource("classpath:usageStatsAPI.properties"), //@PropertySource("classpath:dnet-override.properties") -} -) +//@PropertySources({ +// @PropertySource("classpath:usageStatsAPI.properties"), //@PropertySource("classpath:dnet-override.properties") +//} +//) @EnableRetry @SpringBootApplication public class UsageStatsApi { diff --git a/src/main/java/eu/dnetlib/usagestats/config/DataSourceConfiguration.java b/src/main/java/eu/dnetlib/usagestats/config/DataSourceConfiguration.java index a85aa70..44132f6 100644 --- a/src/main/java/eu/dnetlib/usagestats/config/DataSourceConfiguration.java +++ b/src/main/java/eu/dnetlib/usagestats/config/DataSourceConfiguration.java @@ -13,7 +13,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; /** - * Created by dpie on 25/01/2021. + * Created by D.Pierrakos. */ @Configuration public class DataSourceConfiguration { @@ -46,6 +46,12 @@ public class DataSourceConfiguration { //log.info("driverClassName " + driverClassName); // // hikariConfig.setMaximumPoolSize(5); +// hikariConfig.setMaximumPoolSize(100); +// hikariConfig.setIdleTimeout(300); +// hikariConfig.setMaximumPoolSize(20); +// hikariConfig.setConnectionTimeout(300000); +// hikariConfig.setConnectionTimeout(120000); +// hikariConfig.setLeakDetectionThreshold(300000); hikariConfig.setConnectionTestQuery("SELECT 1"); hikariConfig.setPoolName("UsageStats_HikariCP"); diff --git a/src/main/java/eu/dnetlib/usagestats/config/WebMvcConfiguration.java b/src/main/java/eu/dnetlib/usagestats/config/WebMvcConfiguration.java index d58172e..ec7c4d8 100644 --- a/src/main/java/eu/dnetlib/usagestats/config/WebMvcConfiguration.java +++ b/src/main/java/eu/dnetlib/usagestats/config/WebMvcConfiguration.java @@ -35,6 +35,15 @@ public class WebMvcConfiguration extends WebMvcConfigurerAdapter { registry.addViewController("/sushilite/BR2").setViewName("redirect:/sushilite/BR2/"); registry.addViewController("/sushilite/BR2/").setViewName("forward:/sushilite/BR2/index.html"); + registry.addViewController("/sushilite/PR").setViewName("redirect:/sushilite/PR/"); + registry.addViewController("/sushilite/PR/").setViewName("forward:/sushilite/PR/index.html"); + + registry.addViewController("/sushilite/PR_P1").setViewName("redirect:/sushilite/PR_P1/"); + registry.addViewController("/sushilite/PR_P1/").setViewName("forward:/sushilite/PR_P1/index.html"); + + registry.addViewController("/sushilite/IR").setViewName("redirect:/sushilite/IR/"); + registry.addViewController("/sushilite/IR/").setViewName("forward:/sushilite/IR/index.html"); + super.addViewControllers(registry); } } diff --git a/src/main/java/eu/dnetlib/usagestats/controllers/SushiLiteController.java b/src/main/java/eu/dnetlib/usagestats/controllers/SushiLiteController.java index fef8292..3d746cc 100755 --- a/src/main/java/eu/dnetlib/usagestats/controllers/SushiLiteController.java +++ b/src/main/java/eu/dnetlib/usagestats/controllers/SushiLiteController.java @@ -1,16 +1,29 @@ package eu.dnetlib.usagestats.controllers; +import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.usagestats.services.SushiLiteService; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.List; +import javax.servlet.http.HttpServletResponse; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.util.FileCopyUtils; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** - * Created by dpie on 30/12/2020. + * Created by D.Pierrakos */ @RestController class SushiLiteController { @@ -18,6 +31,8 @@ class SushiLiteController { private final Logger log = Logger.getLogger(this.getClass()); private final SushiLiteService sushiLiteService; + @Value("${download.folder}") + private String download_folder; public SushiLiteController(SushiLiteService sushiLiteService) { this.sushiLiteService = sushiLiteService; @@ -42,7 +57,6 @@ class SushiLiteController { // // return sushiLiteService.displayReport(reportP, release, requestorId, beginDate, endDate, repositoryIdentifier, itemIdentifier, itemDataType, hasDoi, granularity, callback, pretty); // } - @RequestMapping(value = "/sushilite/r5/status", method = RequestMethod.GET) public String getReportStatus() { log.info("COUNTER Report status request "); @@ -55,17 +69,10 @@ class SushiLiteController { return sushiLiteService.displayReportsSupported(); } - @RequestMapping(value = "/sushilite/r5/reports/pr", method = RequestMethod.GET) - public String getReportPR( - @RequestParam(value = "RepositoryIdentifier", defaultValue = "") String repositoryIdentifier, - @RequestParam(value = "RequestorID", defaultValue = "anonymous") String requestorId, - @RequestParam(value = "BeginDate", defaultValue = "") String beginDate, - @RequestParam(value = "EndDate", defaultValue = "") String endDate, - @RequestParam(value = "MetricType", defaultValue = "") String metricType, - @RequestParam(value = "DataType", defaultValue = "") String dataType, - @RequestParam(value = "Granularity", defaultValue = "Monthly") String granularity) { - log.info("COUNTER PR Report request for repository " + repositoryIdentifier); - return sushiLiteService.displayReportPR(requestorId,repositoryIdentifier, beginDate, endDate, metricType, dataType, granularity); + @RequestMapping(value = "/sushilite/r5/members", method = RequestMethod.GET) + public String getMembers() { + log.info("COUNTER Members request "); + return sushiLiteService.displayConsortiumMemberList(); } @RequestMapping(value = "/sushilite/r5/reports/pr_p1", method = RequestMethod.GET) @@ -77,8 +84,9 @@ class SushiLiteController { log.info("COUNTER PR_P1 Report request for repository " + repositoryIdentifier); return sushiLiteService.displayReportPR_P1(requestorId, repositoryIdentifier, beginDate, endDate); } + @RequestMapping(value = "/sushilite/r5/reports/ir", method = RequestMethod.GET) - public String getReportTR( + public ResponseEntity getReportΙR( @RequestParam(value = "RepositoryIdentifier", defaultValue = "") String repositoryIdentifier, @RequestParam(value = "ItemIdentifier", defaultValue = "") String itemIdentifier, @RequestParam(value = "RequestorID", defaultValue = "anonymous") String requestorId, @@ -87,7 +95,65 @@ class SushiLiteController { @RequestParam(value = "MetricType", defaultValue = "") List metricType, @RequestParam(value = "DataType", defaultValue = "") String dataType, @RequestParam(value = "Granularity", defaultValue = "Monthly") String granularity) { - log.info("COUNTER TR Report request for repository " + repositoryIdentifier +" and for item "+ itemIdentifier); - return sushiLiteService.displayReportIR(requestorId, repositoryIdentifier, itemIdentifier, beginDate, endDate, metricType, dataType, granularity); + log.info("COUNTER ΙR Report request for repository " + repositoryIdentifier + " and for item " + itemIdentifier); + String report = sushiLiteService.displayReportIR(requestorId, repositoryIdentifier, itemIdentifier, beginDate, endDate, metricType, dataType, granularity); + + if (report.indexOf(".zip") < 0) { + return new ResponseEntity<>(report, HttpStatus.OK); + } else { + + String compressedOutput = "
 {\"Report\":\"IR\", \"Description\":\"Compressed Report Due to large number of records\", \"URL To Download Report from: \":\"" + report + "\"}
"; + return new ResponseEntity<>(compressedOutput, HttpStatus.OK); + } + } + + @RequestMapping(value = "/sushilite/r5/reports/dsr", method = RequestMethod.GET) + public String getReportDSR( + @RequestParam(value = "RepositoryIdentifier", defaultValue = "") String repositoryIdentifier, + @RequestParam(value = "ItemIdentifier", defaultValue = "") String itemIdentifier, + @RequestParam(value = "RequestorID", defaultValue = "anonymous") String requestorId, + @RequestParam(value = "BeginDate", defaultValue = "") String beginDate, + @RequestParam(value = "EndDate", defaultValue = "") String endDate, + @RequestParam(value = "MetricType", defaultValue = "") List metricType, + @RequestParam(value = "DataType", defaultValue = "") String dataType, + @RequestParam(value = "Granularity", defaultValue = "Monthly") String granularity) { + log.info("COUNTER ΙR Report request for repository " + repositoryIdentifier + " and for item " + itemIdentifier); + return sushiLiteService.displayReportDSR(requestorId, repositoryIdentifier, itemIdentifier, beginDate, endDate, metricType, dataType, granularity); + } + + @RequestMapping(value = "/download/{file_name}", method = RequestMethod.GET) + public void downloadFile(HttpServletResponse response, @PathVariable("file_name") String filetoDownload) throws IOException { + File file = new File(download_folder + "/" + filetoDownload); + log.info("File downloaded at " + file.getAbsolutePath()); + String mimeType = "application/octet-stream"; + + response.setContentType(mimeType); + + /* "Content-Disposition : attachment" will be directly download, may provide save as popup, based on your browser setting*/ + response.setHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", file.getName())); + response.setContentLength((int) file.length()); + InputStream inputStream = new BufferedInputStream(new FileInputStream(file)); + FileCopyUtils.copy(inputStream, response.getOutputStream()); + } + + @RequestMapping(value = "/sushilite/r5/reports/pr", method = RequestMethod.GET) + public ResponseEntity getReportPR(@RequestParam(value = "RepositoryIdentifier", defaultValue = "") String repositoryIdentifier, + @RequestParam(value = "RequestorID", defaultValue = "anonymous") String requestorId, + @RequestParam(value = "BeginDate", defaultValue = "") String beginDate, + @RequestParam(value = "EndDate", defaultValue = "") String endDate, + @RequestParam(value = "MetricType", defaultValue = "") String metricType, + @RequestParam(value = "DataType", defaultValue = "") String dataType, + @RequestParam(value = "Granularity", defaultValue = "Monthly") String granularity) throws InterruptedException, Exception { + + String report = sushiLiteService.displayReportPR(requestorId, repositoryIdentifier, beginDate, endDate, metricType, dataType, granularity); + + if (report.indexOf(".zip") < 0) { + return new ResponseEntity<>(report, HttpStatus.OK); + } else { + + String compreessedOutput = "
 {\"Report\":\"PR\", \"Description\":\"Compressed Report Due to large number of records\", \"URL To Download Report from: \":\"" + report + "\"}
"; + + return new ResponseEntity<>(compreessedOutput, HttpStatus.OK); + } } } diff --git a/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java b/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java index 6c05319..0c02f88 100755 --- a/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java +++ b/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java @@ -17,10 +17,15 @@ import eu.dnetlib.usagestats.portal.TotalStats; import eu.dnetlib.usagestats.portal.TotalStatsReposViewsDownloads; import eu.dnetlib.usagestats.portal.UsageStats; import eu.dnetlib.usagestats.portal.YearlyStats; +import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Identifiers; +import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Performance; +import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Usage; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Identifiers; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Performance; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Usage; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Platform_Usage; +import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Consortium_Member_List; +import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Org_Identifiers; import org.apache.commons.dbutils.DbUtils; @@ -39,6 +44,9 @@ import java.util.HashMap; import java.util.List; import org.springframework.beans.factory.annotation.Value; +/** + * Created by D.Pierrakos + */ @Repository public class UsageStatsRepository { @@ -675,6 +683,25 @@ public class UsageStatsRepository { default: } } + + public void executeItemDSR(List reportItems, String repositoryIdentifier, String itemIdentifier, Date beginDate, + Date endDate, List metricType, String itemDataType, String granularity) { + + String[] split = itemIdentifier.split(":"); + switch (split[0].toLowerCase()) { + case "oid": + executeDSROid(reportItems, repositoryIdentifier, itemIdentifier.replaceFirst(split[0] + ":", ""), beginDate, endDate, metricType, itemDataType, granularity); + break; + case "doi": + executeDSRDoi(reportItems, repositoryIdentifier, itemIdentifier.replaceFirst(split[0] + ":", ""), beginDate, endDate, metricType, itemDataType, granularity); + break; +// case "openaire": +// executeOpenaire(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); +// break; + default: + } + } + private void executeOid(List reportItems, String repositoryIdentifier, String oid, Date beginDate, Date endDate, List metricType, String itemDataType, String granularity) { Connection connection = null; @@ -690,7 +717,11 @@ public class UsageStatsRepository { rs = st.executeQuery(); while (rs.next()) { - executeBatchItemsIR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + if (repositoryIdentifier != null) { + executeBatchItemsIR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } else { + executeItemsAllRepoIR(reportItems, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } } connection.close(); } catch (Exception e) { @@ -718,7 +749,76 @@ public class UsageStatsRepository { rs = st.executeQuery(); while (rs.next()) { - executeBatchItemsIR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + if (repositoryIdentifier != null) { + executeBatchItemsIR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } else { + executeItemsAllRepoIR(reportItems, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } + } + } catch (Exception e) { + log.error("Doi to OpenAIRE id failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } + + private void executeDSROid(List reportItems, String repositoryIdentifier, + String oid, Date beginDate, Date endDate, List metricType, String itemDataType, String granularity) { + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + try { + connection = usageStatsDB.getConnection(); + //st = connection.prepareStatement("SELECT DISTINCT roid.id FROM public.result_oids roid, public.downloads_stats s WHERE s.result_id=roid.id AND roid.orid=? UNION SELECT DISTINCT roid.id FROM public.result_oids roid, public.views_stats s WHERE s.result_id=roid.id AND roid.orid=?"); + st = connection.prepareStatement("SELECT DISTINCT roid.id FROM " + statsDB + ".result_oids roid, " + usagestatsImpalaDB + ".usage_stats us, " + + statsDB + ".result_classifications rc " + + "WHERE us.result_id=roid.id AND rc.id=us.result_id AND rc.type='Dataset' AND roid.oid=?"); + st.setString(1, oid); + //st.setString(2, oid); + + rs = st.executeQuery(); + + while (rs.next()) { + if (repositoryIdentifier != null) { + executeBatchItemsDSR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } else { + executeItemsAllRepoDSR(reportItems, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } + } + } catch (Exception e) { + log.error("Oid to OpenAIRE id failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } + + private void executeDSRDoi(List reportItems, String repositoryIdentifier, + String doi, Date beginDate, Date endDate, List metricType, String itemDataType, String granularity) { + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + + try { + connection = usageStatsDB.getConnection(); + //st = connection.prepareStatement("SELECT DISTINCT poid.id FROM public.result_pids poid, public.downloads_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=? UNION SELECT DISTINCT poid.id FROM public.result_pids poid, public.views_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=?"); + st = connection.prepareStatement("SELECT DISTINCT poid.id FROM " + statsDB + ".result_pids poid, " + usagestatsImpalaDB + ".usage_stats us, " + + statsDB + ".result_classifications rc " + + "WHERE us.result_id=poid.id AND poid.type='Digital Object Identifier' AND rc.id=us.result_id AND rc.type='Dataset' AND poid.pid=?"); + st.setString(1, doi); + //st.setString(2, doi); + + rs = st.executeQuery(); + + while (rs.next()) { + if (repositoryIdentifier != null) { + executeBatchItemsDSR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } else { + executeItemsAllRepoDSR(reportItems, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } } } catch (Exception e) { log.error("Doi to OpenAIRE id failed: ", e); @@ -1511,12 +1611,15 @@ public class UsageStatsRepository { ResultSet rs = null; String platform = getInstitutionName(repositoryIdentifier); log.info("Platform " + platform); + log.info("Data Type "+dataType); try { connection = usageStatsDB.getConnection(); if (granularity.equalsIgnoreCase("totals")) { - if (dataType.equals("")) { - st = connection.prepareStatement("SELECT rc_id, rc.type, sum(us.downloads) as downloads, sum(us.views) as views FROM " + usagestatsImpalaDB + ".usage_stats us, " + if (dataType.equals("") || dataType.equals("All")) { + st = connection.prepareStatement("SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id GROUP BY rc.type order by rc.type ASC;"); st.setString(1, beginDateStr); @@ -1525,32 +1628,30 @@ public class UsageStatsRepository { rs = st.executeQuery(); COUNTER_Platform_Usage reportItem = null; - - switch (metricType) { - case (""): - while (rs.next()) { - reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); - reportItems.add(reportItem); - } - break; - case ("Total_Items_Investigations"): - while (rs.next()) { - reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); - reportItems.add(reportItem); - } - break; - case ("Total_Items_Requests"): - while (rs.next()) { - reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); - reportItems.add(reportItem); - } - break; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + while (rs.next()) { + reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(4) : null, metricTypeValue[3] == true ? rs.getString(5) : null)); + reportItems.add(reportItem); + } + } else { - st = connection.prepareStatement("SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views FROM " + usagestatsImpalaDB + ".usage_stats us, " + st = connection.prepareStatement("SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id GROUP BY rc.type order by rc.type ASC;"); st.setString(1, beginDateStr); @@ -1560,34 +1661,34 @@ public class UsageStatsRepository { rs = st.executeQuery(); COUNTER_Platform_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; - switch (metricType) { - case (""): - while (rs.next()) { - reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); - reportItems.add(reportItem); - } - break; - case ("Total_Items_Investigations"): - while (rs.next()) { - reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); - reportItems.add(reportItem); - } - break; - case ("Total_Items_Requests"): - while (rs.next()) { - reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); - reportItems.add(reportItem); - } - break; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Platform_Usage(platform, rs.getString(1), "Regular"); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(4) : null, metricTypeValue[3] == true ? rs.getString(5) : null)); + reportItems.add(reportItem); + } + } } else if (granularity.equalsIgnoreCase("monthly")) { - if (dataType.equals("")) { - st = connection.prepareStatement("SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views FROM " + usagestatsImpalaDB + ".usage_stats us, " + if (dataType.equals("") || dataType.equals("All")) { + st = connection.prepareStatement("SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id GROUP BY rc.type, us.`date` order by rc.type, us.`date` ASC;"); st.setString(1, beginDateStr); @@ -1599,6 +1700,20 @@ public class UsageStatsRepository { String lastDate = ""; COUNTER_Platform_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + int ft_total = 0; int abstr = 0; @@ -1622,20 +1737,8 @@ public class UsageStatsRepository { datatype = rs.getString(1); } if (reportItem != null) { - switch (metricType) { - case (""): - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), rs.getString(4))); - break; - - case ("Total_Items_Investigations"): - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); - break; - - case ("Total_Items_Requests"): - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); - break; - - } + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(5) : null, metricTypeValue[3] == true ? rs.getString(6) : null)); } endC.setTime(postgresFormat.parse(rs.getString(2))); @@ -1647,7 +1750,9 @@ public class UsageStatsRepository { //add last report item reportItems.add(reportItem); } else { - st = connection.prepareStatement("SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views FROM " + usagestatsImpalaDB + ".usage_stats us, " + st = connection.prepareStatement("SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id GROUP BY rc.type, us.`date` order by rc.type, us.`date` ASC;"); st.setString(1, beginDateStr); @@ -1660,6 +1765,19 @@ public class UsageStatsRepository { String lastDate = ""; COUNTER_Platform_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } int ft_total = 0; int abstr = 0; @@ -1683,21 +1801,8 @@ public class UsageStatsRepository { datatype = rs.getString(1); } if (reportItem != null) { - switch (metricType) { - case (""): - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), rs.getString(4))); - break; - - case ("Total_Items_Investigations"): - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); - break; - - case ("Total_Items_Requests"): - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); - break; - - } - + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(5) : null, metricTypeValue[3] == true ? rs.getString(6) : null)); } endC.setTime(postgresFormat.parse(rs.getString(2))); endC.add(Calendar.MONTH, 1); @@ -1740,7 +1845,9 @@ public class UsageStatsRepository { try { connection = usageStatsDB.getConnection(); - st = connection.prepareStatement("SELECT us.`date`, sum(us.downloads) as downloads, sum(us.views) as views FROM " + usagestatsImpalaDB + ".usage_stats us " + st = connection.prepareStatement("SELECT us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? GROUP BY us.`date` order by us.`date` ASC;"); st.setString(1, beginDateStr); @@ -1769,7 +1876,8 @@ public class UsageStatsRepository { endC.setTime(postgresFormat.parse(rs.getString(1))); endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); //iterate on data types - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(1))), report_dateFormat.format(endC.getTime()), rs.getString(2), rs.getString(3))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(1))), report_dateFormat.format(endC.getTime()), + rs.getString(2), rs.getString(3), rs.getString(4), rs.getString(5))); endC.setTime(postgresFormat.parse(rs.getString(1))); endC.add(Calendar.MONTH, 1); lastDate = postgresFormat.format(endC.getTime()); @@ -1816,7 +1924,8 @@ public class UsageStatsRepository { if (dataType.equals("")) { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " @@ -1828,52 +1937,39 @@ public class UsageStatsRepository { rs = st.executeQuery(); COUNTER_Item_Usage reportItem = null; - boolean metricFound = false; - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); - reportItems.add(reportItem); + while (rs.next()) { + reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), null, "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - metricFound = true; - } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); - reportItems.add(reportItem); - } - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); - reportItems.add(reportItem); - } - metricFound = true; + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); } + } else { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id " @@ -1887,48 +1983,32 @@ public class UsageStatsRepository { rs = st.executeQuery(); COUNTER_Item_Usage reportItem = null; - boolean metricFound = false; + boolean[] metricTypeValue = {false, false, false, false}; - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); - reportItems.add(reportItem); - } - metricFound = true; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); - reportItems.add(reportItem); + while (rs.next()) { + reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), null, "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); - reportItems.add(reportItem); - } - metricFound = true; + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); } } } else if (granularity.equalsIgnoreCase("monthly")) { @@ -1936,7 +2016,8 @@ public class UsageStatsRepository { if (dataType.equals("")) { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " @@ -1950,6 +2031,20 @@ public class UsageStatsRepository { String result = ""; String lastDate = ""; COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } int ft_total = 0; int abstr = 0; @@ -1967,64 +2062,19 @@ public class UsageStatsRepository { endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); // //iterate on data types if (!datatype.equals(rs.getString(1))) { -// if (reportItem != null) { -// reportItems.add(reportItem); -// } -// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); datatype = rs.getString(1); } - boolean metricFound = false; - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; + + reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), null, "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; - } - // } -// -// //} + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + endC.setTime(postgresFormat.parse(rs.getString(2))); endC.add(Calendar.MONTH, 1); lastDate = postgresFormat.format(endC.getTime()); @@ -2034,7 +2084,8 @@ public class UsageStatsRepository { } else { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id " @@ -2049,6 +2100,20 @@ public class UsageStatsRepository { String result = ""; String lastDate = ""; COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } int ft_total = 0; int abstr = 0; @@ -2074,59 +2139,17 @@ public class UsageStatsRepository { // reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); datatype = rs.getString(1); } - boolean metricFound = false; - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); - -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; + reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), null, "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(20) : null)); + reportItems.add(reportItem); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); - reportItems.add(reportItem); - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; - } // } // // //} @@ -2159,7 +2182,8 @@ public class UsageStatsRepository { if (dataType.equals("")) { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " @@ -2172,52 +2196,37 @@ public class UsageStatsRepository { rs = st.executeQuery(); COUNTER_Item_Usage reportItem = null; - boolean metricFound = false; - - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); - reportItems.add(reportItem); - } - metricFound = true; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); - reportItems.add(reportItem); - } - metricFound = true; + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); - reportItems.add(reportItem); + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), null, "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - metricFound = true; + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); } } else { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " @@ -2229,52 +2238,27 @@ public class UsageStatsRepository { st.setString(3, repositoryIdentifier); st.setString(4, itemIdentifier); st.setString(5, dataType); - rs = st.executeQuery(); COUNTER_Item_Usage reportItem = null; - boolean metricFound = false; - - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); - reportItems.add(reportItem); - } - metricFound = true; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Items_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Items_Ivestigations")) { + metricTypeValue[1] = true; } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); - reportItems.add(reportItem); + while (rs.next()) { + reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), null, "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - while (rs.next()) { - reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(7).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); - reportItems.add(reportItem); - } - metricFound = true; + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); } } } else if (granularity.equalsIgnoreCase("monthly")) { @@ -2283,7 +2267,8 @@ public class UsageStatsRepository { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " @@ -2298,6 +2283,20 @@ public class UsageStatsRepository { String result = ""; String lastDate = ""; COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } int ft_total = 0; int abstr = 0; @@ -2315,64 +2314,18 @@ public class UsageStatsRepository { endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); // //iterate on data types if (!datatype.equals(rs.getString(1))) { -// if (reportItem != null) { -// reportItems.add(reportItem); -// } -// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); datatype = rs.getString(1); } - boolean metricFound = false; - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; + reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), null, "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; - } - // } -// -// //} + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + endC.setTime(postgresFormat.parse(rs.getString(2))); endC.add(Calendar.MONTH, 1); lastDate = postgresFormat.format(endC.getTime()); @@ -2382,7 +2335,8 @@ public class UsageStatsRepository { } else { st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " @@ -2399,6 +2353,557 @@ public class UsageStatsRepository { String lastDate = ""; COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + List identifiers = null; + + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { +// if (reportItem != null) { +// reportItems.add(reportItem); +// } +// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); + datatype = rs.getString(1); + } + + reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), null, "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } + } + /* + jedis.put(redis_key, "persistent", "false"); + jedis.put(redis_key, "query", st.toString()); + jedis.put(redis_key, "result", toJson(reportItems)); + jedis.put(redis_key, "fetchMode", "3"); + */ + } catch (Exception e) { + log.error("Batch Item Report failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } + } + + public void executeItemsAllRepoIR(List reportItems, + String itemIdentifier, Date beginDate, + Date endDate, List metricType, String dataType, String granularity) throws Exception { + SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM"); + String beginDateStr = postgresFormat.format(beginDate); + String endDateStr = postgresFormat.format(endDate); + + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + + try { + connection = usageStatsDB.getConnection(); + + if (granularity.equalsIgnoreCase("totals")) { + if (dataType.equals("")) { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year, ds.name order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + + rs = st.executeQuery(); + COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", rs.getString(8), rs.getString(1), Integer.toString(rs.getInt(6)), null, "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + } + + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year, ds.name order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + st.setString(4, dataType); + + rs = st.executeQuery(); + COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + String platform = getInstitutionName(rs.getString(8)); + reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), null, "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + } + + } + } else if (granularity.equalsIgnoreCase("monthly")) { + + if (dataType.equals("")) { + + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " + + "AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { + datatype = rs.getString(1); + } + reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", rs.getString(9), rs.getString(1), Integer.toString(rs.getInt(7)), null, "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(10) : null, metricTypeValue[3] == true ? rs.getString(11) : null)); + reportItems.add(reportItem); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + st.setString(4, dataType); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Item_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Item_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Item_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Item_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Item_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { +// if (reportItem != null) { +// reportItems.add(reportItem); +// } +// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); + datatype = rs.getString(1); + } + reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", rs.getString(9), rs.getString(1), Integer.toString(rs.getInt(7)), null, "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(10) : null, metricTypeValue[3] == true ? rs.getString(11) : null)); + reportItems.add(reportItem); + + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } + } + /* + jedis.put(redis_key, "persistent", "false"); + jedis.put(redis_key, "query", st.toString()); + jedis.put(redis_key, "result", toJson(reportItems)); + jedis.put(redis_key, "fetchMode", "3"); + */ + } catch (Exception e) { + log.error("Batch Item Report failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } + + public void executeBatchItemsDSR(List reportItems, + String repositoryIdentifier, String itemIdentifier, Date beginDate, + Date endDate, List metricType, String dataType, String granularity) throws Exception { + SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM"); + String beginDateStr = postgresFormat.format(beginDate); + String endDateStr = postgresFormat.format(endDate); + + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + String platform = getInstitutionName(repositoryIdentifier); + + if (itemIdentifier.equals("")) { + try { + connection = usageStatsDB.getConnection(); + + if (granularity.equalsIgnoreCase("totals")) { + if (dataType.equals("")) { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + + rs = st.executeQuery(); + COUNTER_Dataset_Usage reportItem = null; + + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); + } + + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + st.setString(4, dataType); + + rs = st.executeQuery(); + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + while (rs.next()) { + reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); + } + } + } else if (granularity.equalsIgnoreCase("monthly")) { + + if (dataType.equals("")) { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " + + "AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { + datatype = rs.getString(1); + } + + reportItem = new COUNTER_Dataset_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + st.setString(4, dataType); + rs = st.executeQuery(); + + String result = ""; + String lastDate = ""; + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + int ft_total = 0; int abstr = 0; @@ -2423,59 +2928,17 @@ public class UsageStatsRepository { // reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); datatype = rs.getString(1); } - boolean metricFound = false; - if (metricType.contains("Total_Items_Investigations") && metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); - -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; + reportItem = new COUNTER_Dataset_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - if (metricType.contains("Total_Items_Investigations") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); - reportItems.add(reportItem); - metricFound = true; - } - if (metricType.contains("Total_Items_Requests") && (metricFound == false)) { - reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Requests"); - String[] identifiersAll = rs.getString(8).split("#!#"); - for (int i = 0; i < identifiersAll.length; i++) { - String[] typeIdentifierArray = identifiersAll[i].split("#-#"); - reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); - } - - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); -// if (rs.getString(5).indexOf("doi") >= 0) { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); -// } else { -// reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); -// } - reportItems.add(reportItem); - metricFound = true; - } // } // // //} @@ -2500,6 +2963,524 @@ public class UsageStatsRepository { DbUtils.closeQuietly(st); DbUtils.closeQuietly(connection); } + } else { + try { + connection = usageStatsDB.getConnection(); + + if (granularity.equalsIgnoreCase("totals")) { + if (dataType.equals("")) { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + st.setString(4, itemIdentifier); + + rs = st.executeQuery(); + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); + } + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + st.setString(4, itemIdentifier); + st.setString(5, dataType); + + rs = st.executeQuery(); + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null)); + reportItems.add(reportItem); + } + } + } else if (granularity.equalsIgnoreCase("monthly")) { + + if (dataType.equals("")) { + + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " + + "AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + st.setString(4, itemIdentifier); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { + datatype = rs.getString(1); + } + reportItem = new COUNTER_Dataset_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, repositoryIdentifier); + st.setString(4, itemIdentifier); + st.setString(5, dataType); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Dataset_Usage reportItem = null; + + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { + datatype = rs.getString(1); + } + + reportItem = new COUNTER_Dataset_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } + } + /* + jedis.put(redis_key, "persistent", "false"); + jedis.put(redis_key, "query", st.toString()); + jedis.put(redis_key, "result", toJson(reportItems)); + jedis.put(redis_key, "fetchMode", "3"); + */ + } catch (Exception e) { + log.error("Batch Item Report failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } + } + + public void executeItemsAllRepoDSR(List reportItems, + String itemIdentifier, Date beginDate, + Date endDate, List metricType, String dataType, String granularity) throws Exception { + SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM"); + String beginDateStr = postgresFormat.format(beginDate); + String endDateStr = postgresFormat.format(endDate); + + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + + try { + connection = usageStatsDB.getConnection(); + + if (granularity.equalsIgnoreCase("totals")) { + if (dataType.equals("")) { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year, ds.name order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + + rs = st.executeQuery(); + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", rs.getString(8), rs.getString(1), Integer.toString(rs.getInt(6)), "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + } + + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year, ds.name order by rc.type ASC;"); + + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + st.setString(4, dataType); + + rs = st.executeQuery(); + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + while (rs.next()) { + String platform = getInstitutionName(rs.getString(8)); + reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", rs.getString(8), rs.getString(1), Integer.toString(rs.getInt(6)), "Regular"); + String[] identifiersAll = rs.getString(7).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), + metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(9) : null, metricTypeValue[3] == true ? rs.getString(10) : null)); + reportItems.add(reportItem); + } + + } + } else if (granularity.equalsIgnoreCase("monthly")) { + + if (dataType.equals("")) { + + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " + + "AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { + datatype = rs.getString(1); + } + reportItem = new COUNTER_Dataset_Usage(rs.getString(6), "OpenAIRE", rs.getString(9), rs.getString(1), Integer.toString(rs.getInt(7)), "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(10) : null, metricTypeValue[3] == true ? rs.getString(11) : null)); + reportItems.add(reportItem); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } else { + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, " + + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd, " + statsDB + ".datasource ds " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name order by rc.type, us.`date` ASC;"); + st.setString(1, beginDateStr); + st.setString(2, endDateStr); + st.setString(3, itemIdentifier); + st.setString(4, dataType); + + rs = st.executeQuery(); + String result = ""; + String lastDate = ""; + COUNTER_Dataset_Usage reportItem = null; + boolean[] metricTypeValue = {false, false, false, false}; + + if (metricType.contains("Total_Dataset_Requests")) { + metricTypeValue[0] = true; + } + if (metricType.contains("Total_Dataset_Investigations")) { + metricTypeValue[1] = true; + } + if (metricType.contains("Unique_Dataset_Requests")) { + metricTypeValue[2] = true; + } + if (metricType.contains("Unique_Dataset_Investigations")) { + metricTypeValue[3] = true; + } + + int ft_total = 0; + int abstr = 0; + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(postgresFormat.parse(endDateStr)); + endCal.add(Calendar.MONTH, 1); + Date endDateForZeros = endCal.getTime(); + + lastDate = beginDateStr; + String datatype = ""; + + while (rs.next()) { + Calendar endC = Calendar.getInstance(); + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE)); +// //iterate on data types + if (!datatype.equals(rs.getString(1))) { +// if (reportItem != null) { +// reportItems.add(reportItem); +// } +// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", ""); + datatype = rs.getString(1); + } + reportItem = new COUNTER_Dataset_Usage(rs.getString(6), "OpenAIRE", rs.getString(9), rs.getString(1), Integer.toString(rs.getInt(7)), "Regular"); + String[] identifiersAll = rs.getString(8).split("#!#"); + for (int i = 0; i < identifiersAll.length; i++) { + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); + } + + reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), + metricTypeValue[0] == true ? rs.getString(3) : null, metricTypeValue[1] == true ? rs.getString(4) : null, metricTypeValue[2] == true ? rs.getString(10) : null, metricTypeValue[3] == true ? rs.getString(11) : null)); + reportItems.add(reportItem); + + endC.setTime(postgresFormat.parse(rs.getString(2))); + endC.add(Calendar.MONTH, 1); + lastDate = postgresFormat.format(endC.getTime()); + //if (reportItem != null) { + //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); + } + } + } + /* + jedis.put(redis_key, "persistent", "false"); + jedis.put(redis_key, "query", st.toString()); + jedis.put(redis_key, "result", toJson(reportItems)); + jedis.put(redis_key, "fetchMode", "3"); + */ + } catch (Exception e) { + log.error("Batch Item Report failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); } } @@ -2517,7 +3498,7 @@ public class UsageStatsRepository { temp_c.set(Calendar.DAY_OF_MONTH, temp_c.getActualMaximum(Calendar.DAY_OF_MONTH)); Date temp_endDate = temp_c.getTime(); - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(from), report_dateFormat.format(temp_endDate), "0", "0")); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(from), report_dateFormat.format(temp_endDate), "0", "0", "0", "0")); fromCalendar.add(Calendar.MONTH, 1); from = fromCalendar.getTime(); } @@ -2548,4 +3529,51 @@ public class UsageStatsRepository { return "-1"; } + + public boolean checkServiceConnection() { + Connection connection = null; + try { + connection = usageStatsDB.getConnection(); + if (connection != null) { + return true; + } + } catch (Exception e) { + log.info(e); + }finally { + DbUtils.closeQuietly(connection); + } + return false; + } + + public ArrayList buildMembersList() { + PreparedStatement st = null; + Connection connection = null; + ResultSet rs = null; + ArrayList membersList = new ArrayList(); + + try { + connection = usageStatsDB.getConnection(); + st = connection.prepareStatement("select distinct us.repository_id, d.name, do.oid from " + statsDB + ".datasource d, " + + statsDB + ".datasource_oids do, " + usagestatsImpalaDB + ".usage_stats us " + + "where us.repository_id=d.id and us.repository_id=do.id and d.id=do.id and do.oid not like '%piwik%' order by do.oid asc"); + rs = st.executeQuery(); + while (rs.next()) { + ArrayList identifiers = new ArrayList(); + identifiers.add(new SUSHI_Org_Identifiers("OpenAIRE", rs.getString(1))); + identifiers.add(new SUSHI_Org_Identifiers("OpenDOAR", rs.getString(3))); + + membersList.add(new SUSHI_Consortium_Member_List(null, null, rs.getString(2), null, identifiers)); + } + + } catch (Exception e) { + log.error("No member found " + e); + } + finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + return membersList; + + } } diff --git a/src/main/java/eu/dnetlib/usagestats/services/SushiLiteService.java b/src/main/java/eu/dnetlib/usagestats/services/SushiLiteService.java index d720bf1..8e71de0 100755 --- a/src/main/java/eu/dnetlib/usagestats/services/SushiLiteService.java +++ b/src/main/java/eu/dnetlib/usagestats/services/SushiLiteService.java @@ -1,10 +1,12 @@ package eu.dnetlib.usagestats.services; +import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Report; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Report; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Platform_Report; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Title_Report; import eu.dnetlib.usagestats.sushilite.domain.ReportResponseWrapper; -import eu.dnetlib.usagestats.sushilite.domain.ReportStatus; +import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Consortium_Member_List; +import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Service_Status; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -19,12 +21,15 @@ public interface SushiLiteService { // String endDate, String repositoryIdentifier, String itemIdentifier, // String itemDataType, String hasDoi, String granularity, String callback, String pretty); - ReportStatus buildReportStatus(); + SUSHI_Service_Status buildReportStatus(); String displayReportStatus(); ArrayList buildReportSupported(); String displayReportsSupported(); + ArrayList buildMembersList(); + String displayConsortiumMemberList(); + COUNTER_Platform_Report buildReportPR(String customerID, String repositoryIdentifier, String beginDate,String endDate, String metricType, String dataType,String granularity); String displayReportPR(String customerID, String repositoryIdentifier, String beginDate,String endDate, String metricType, String dataType,String granularity); @@ -34,4 +39,6 @@ public interface SushiLiteService { COUNTER_Item_Report buildReportIR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate,String endDate, List metricType, String dataType,String granularity) throws Exception; String displayReportIR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate,String endDate, List metricType, String dataType,String granularity); + COUNTER_Dataset_Report buildReportDSR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate,String endDate, List metricType, String dataType,String granularity) throws Exception; + String displayReportDSR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate,String endDate, List metricType, String dataType,String granularity); } diff --git a/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java b/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java index 4fecceb..3f1d8b7 100755 --- a/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java +++ b/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.usagestats.repositories.UsageStatsRepository; import eu.dnetlib.usagestats.sushilite.domain.Alert; +import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Report; +import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Usage; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Report; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Usage; import eu.dnetlib.usagestats.sushilite.domain.Filter; @@ -11,12 +13,13 @@ import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Org_Identifiers; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Platform_Usage; import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Error_Model; import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Platform_Report; -import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Title_Report; -import eu.dnetlib.usagestats.sushilite.domain.ReportResponse; -import eu.dnetlib.usagestats.sushilite.domain.ReportResponseWrapper; -import eu.dnetlib.usagestats.sushilite.domain.ReportStatus; -import eu.dnetlib.usagestats.sushilite.domain.ReportSupported; -import java.sql.SQLException; +import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Service_Status; +import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Report_List; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileWriter; import org.springframework.stereotype.Service; import java.text.SimpleDateFormat; @@ -27,7 +30,10 @@ import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.logging.Level; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Value; @Service public class SushiLiteServiceImpl implements SushiLiteService { @@ -36,6 +42,17 @@ public class SushiLiteServiceImpl implements SushiLiteService { private final Logger log = Logger.getLogger(this.getClass()); + @Value("${compression.max_number_of_records}") + private int compression_max_number_of_records; + + @Value("${download.folder}") + private String download_folder; + + @Value("${sushi-lite.server}") + private String sushi_lite_server; + + boolean reportForCompression = false; + public SushiLiteServiceImpl(UsageStatsRepository usageStatsRepository) { this.usageStatsRepository = usageStatsRepository; } @@ -250,26 +267,40 @@ public class SushiLiteServiceImpl implements SushiLiteService { } @Override - public ReportStatus buildReportStatus() { + public SUSHI_Service_Status buildReportStatus() { + ZonedDateTime dateTime = ZonedDateTime.now(); // Gets the current date and time, with your default time-zone DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - Alert alert1 = new Alert("2020-12-28T13:32:44.632Z", "string"); - Alert alert2 = new Alert(dateTime.format(formatter), "string"); + Alert alert = new Alert(dateTime.format(formatter), "No alert"); ArrayList alertsList = new ArrayList(); - alertsList.add(alert1); - alertsList.add(alert2); + alertsList.add(alert); + try { + if (usageStatsRepository.checkServiceConnection() == true) { - ReportStatus reportStatus = new ReportStatus("desctiption", true, "registry_url", "note", alertsList); - - return (reportStatus); + SUSHI_Service_Status reportStatus = new SUSHI_Service_Status("COUNTER R5 Usage Reports by OpenAIRE UsageCounts Service", true, "https://provide.openaire.eu", "UsageCounts Service Info: https://usagecounts.openaire.eu", alertsList); + return (reportStatus); + } else { + return null; + } + } catch (Exception e) { + System.out.println(e); + } + return null; } @Override public String displayReportStatus() { ObjectMapper objectMapper = new ObjectMapper(); try { - return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(buildReportStatus()) + "
"; + if (buildReportStatus() != null) { + return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(buildReportStatus()) + "
"; + } else { + ZonedDateTime dateTime = ZonedDateTime.now(); // Gets the current date and time, with your default time-zone + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + SUSHI_Error_Model errorModel = new SUSHI_Error_Model("1000", "Fatal", "Service Not Available", "usagecounts.openaire.eu", "Request was for: " + dateTime.format(formatter)); + return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorModel) + "
"; + } } catch (JsonProcessingException ex) { java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class.getName()).log(Level.SEVERE, null, ex); } @@ -279,11 +310,19 @@ public class SushiLiteServiceImpl implements SushiLiteService { @Override public ArrayList buildReportSupported() { ArrayList reportSupportedList = new ArrayList(); - ReportSupported r1 = new ReportSupported("Report_Name", "Report_ID", "Release", "Report_Description", "Path"); - ReportSupported r2 = new ReportSupported("Report_Name2", "Report_ID2", "Release2", "Report_Description2", "Path2"); + SUSHI_Report_List r1 = new SUSHI_Report_List("Status Report", "", "5", "Current status of the reporting service supported by this API", "/status"); + SUSHI_Report_List r2 = new SUSHI_Report_List("Members Report", "", "5", "List of UsageCounts members", "/members"); + SUSHI_Report_List r3 = new SUSHI_Report_List("List Of Reports Report", "", "5", "List of reports supported by the API", "/reports/"); + SUSHI_Report_List r4 = new SUSHI_Report_List("Platform Master Report PR", "PR", "5", "A customizable report summarizing activity across a provider’s platforms that allows the user to apply filters and select other configuration options for the report. ", "/PR"); + SUSHI_Report_List r5 = new SUSHI_Report_List("Platform Usage Report", "PR_P1", "5", "Standard View of the Package Master Report that presents usage for the overall Platform broken down by Metric_Type.", "/PR_1"); + SUSHI_Report_List r6 = new SUSHI_Report_List("Platform Item Report", "IR", "5", "COUNTER 'Item Master Report", "/IR"); reportSupportedList.add(r1); reportSupportedList.add(r2); + reportSupportedList.add(r3); + reportSupportedList.add(r4); + reportSupportedList.add(r5); + reportSupportedList.add(r6); return reportSupportedList; } @@ -301,14 +340,19 @@ public class SushiLiteServiceImpl implements SushiLiteService { } @Override - public COUNTER_Platform_Report buildReportPR(String customerID, String repositoryIdentifier, String beginDate, + public COUNTER_Platform_Report buildReportPR(String customerID, String repoID, String beginDate, String endDate, String metricType, String dataType, String granularity) { List reportExceptions = new ArrayList<>(); ZonedDateTime dateTime = ZonedDateTime.now(); // Gets the current date and time, with your default time-zone DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); //Display Report Created Day - repositoryIdentifier = usageStatsRepository.getInstitutionID(repositoryIdentifier); + String repositoryIdentifier = usageStatsRepository.getInstitutionID(repoID); + if (!repositoryIdentifier.equals("")) { + if (repositoryIdentifier.equals("-1")) { + reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "RepositoryIdentifier: " + repoID + " is not valid")); + } + } Date beginDateParsed; if (!beginDate.equals("")) { @@ -318,6 +362,8 @@ public class SushiLiteServiceImpl implements SushiLiteService { temp.setTime(beginDateParsed); temp.set(Calendar.DAY_OF_MONTH, temp.getActualMinimum(Calendar.DAY_OF_MONTH)); beginDateParsed = temp.getTime(); + } else if (beginDateParsed == null) { + reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "usagecounts.openaire.eu", "Invalid Date Arguments", "Begin Date: " + beginDate + " is not a valid date")); } } else { Calendar temp = Calendar.getInstance(); @@ -335,6 +381,8 @@ public class SushiLiteServiceImpl implements SushiLiteService { temp.setTime(endDateParsed); temp.set(Calendar.DAY_OF_MONTH, temp.getActualMaximum(Calendar.DAY_OF_MONTH)); endDateParsed = temp.getTime(); + } else if (endDateParsed == null) { + reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "usagecounts.openaire.eu", "Invalid Date Arguments", "End Date: " + endDate + " is not a valid date")); } } else { Calendar temp = Calendar.getInstance(); @@ -366,20 +414,31 @@ public class SushiLiteServiceImpl implements SushiLiteService { institutionIdD.add(new SUSHI_Org_Identifiers("Openaire", repositoryIdentifier)); List reportItems = new ArrayList(); - usageStatsRepository.executeBatchItemsPR(reportItems, repositoryIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + if (reportExceptions.size() == 0) { + reportExceptions = null; + usageStatsRepository.executeBatchItemsPR(reportItems, repositoryIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + } + if (reportItems.isEmpty()) { reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); } - if (reportExceptions.size() == 0) { - reportExceptions = null; - } +// if (reportExceptions.size() == 0) { +// reportExceptions = null; +// } COUNTER_Platform_Report reportPr = new COUNTER_Platform_Report(dateTime.format(formatter), customerID, reportID, reportName, institutionName, institutionIdD, reportExceptions, reportFilters, reportItems); + + log.info("Total report items " + reportItems.size()); + + if (reportItems.size() > compression_max_number_of_records) { + log.info("Compression due to " + reportItems.size()); + reportForCompression = true; + } return reportPr; } @Override - public COUNTER_Platform_Report buildReportPR_P1(String customerID, String repositoryIdentifier, + public COUNTER_Platform_Report buildReportPR_P1(String customerID, String repoID, String beginDate, String endDate) { String granularity = "monthly"; @@ -389,8 +448,12 @@ public class SushiLiteServiceImpl implements SushiLiteService { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); //Display Report Created Day - repositoryIdentifier = usageStatsRepository.getInstitutionID(repositoryIdentifier); - + String repositoryIdentifier = usageStatsRepository.getInstitutionID(repoID); + if (!repositoryIdentifier.equals("")) { + if (repositoryIdentifier.equals("-1")) { + reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "RepositoryIdentifier: " + repoID + " is not valid")); + } + } Date beginDateParsed; if (!beginDate.equals("")) { beginDateParsed = tryParse(beginDate); @@ -461,7 +524,7 @@ public class SushiLiteServiceImpl implements SushiLiteService { } @Override - public COUNTER_Item_Report buildReportIR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate, + public COUNTER_Item_Report buildReportIR(String customerID, String repoID, String itemIdentifier, String beginDate, String endDate, List metricType, String dataType, String granularity) throws Exception { ZonedDateTime dateTime = ZonedDateTime.now(); // Gets the current date and time, with your default time-zone @@ -474,7 +537,177 @@ public class SushiLiteServiceImpl implements SushiLiteService { reportFilters.add(new Filter("BeginDate", beginDate)); reportFilters.add(new Filter("EndDate", endDate)); - repositoryIdentifier = usageStatsRepository.getInstitutionID(repositoryIdentifier); + String repositoryIdentifier = ""; + + if (!repoID.equals("")) { + repositoryIdentifier = usageStatsRepository.getInstitutionID(repoID); + if (repositoryIdentifier.equals("-1")) { + reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "RepositoryIdentifier: " + repoID + " is not valid")); + } + } + + orgIdentifiers.add(new SUSHI_Org_Identifiers("Openaire", repositoryIdentifier)); + + if (!granularity.equalsIgnoreCase("totals") && !granularity.equalsIgnoreCase("monthly")) { + reportExceptions.add(new SUSHI_Error_Model("3062", "Warning", "Invalid ReportAttribute Value", "usagecounts.openaire.eu", "Granularity: \'" + granularity + "\' unknown. Defaulting to Monthly")); + granularity = "Monthly"; + } + + Date beginDateParsed; + if (!beginDate.equals("")) { + beginDateParsed = tryParse(beginDate); + if (beginDateParsed != null && (granularity.toLowerCase().equals("monthly") || beginDate.length() == 7)) { + Calendar temp = Calendar.getInstance(); + temp.setTime(beginDateParsed); + temp.set(Calendar.DAY_OF_MONTH, temp.getActualMinimum(Calendar.DAY_OF_MONTH)); + beginDateParsed = temp.getTime(); + } else if (beginDateParsed == null) { + reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "Begin Date: " + beginDate + " is not a valid date")); + } + } else { + Calendar temp = Calendar.getInstance(); + temp.add(Calendar.MONTH, -1); + temp.set(Calendar.DAY_OF_MONTH, temp.getActualMinimum(Calendar.DAY_OF_MONTH)); + beginDateParsed = temp.getTime(); + reportExceptions.add(new SUSHI_Error_Model("3021", "Warning", "Unspecified Date Arguments", "usagecounts.openaire.eu", "Begin Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(beginDateParsed))); + } + + Date endDateParsed; + if (!endDate.equals("")) { + endDateParsed = tryParse(endDate); + if (endDateParsed != null && (granularity.toLowerCase().equals("monthly") || endDate.length() == 7)) { + Calendar temp = Calendar.getInstance(); + temp.setTime(endDateParsed); + temp.set(Calendar.DAY_OF_MONTH, temp.getActualMaximum(Calendar.DAY_OF_MONTH)); + endDateParsed = temp.getTime(); + } else if (endDateParsed == null) { + reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "End Date: " + endDate + " is not a valid date")); + } + } else { + Calendar temp = Calendar.getInstance(); + temp.add(Calendar.MONTH, -1); + temp.set(Calendar.DAY_OF_MONTH, temp.getActualMaximum(Calendar.DAY_OF_MONTH)); + endDateParsed = temp.getTime(); + reportExceptions.add(new SUSHI_Error_Model("3021", "Warning", "Unspecified Date Arguments", "usagecounts.openaire.eu", "End Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed))); + } + +// if (beginDateParsed == null) { +// reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "Begin Date: " + beginDate + " is not a valid date")); +// } +// if (endDateParsed == null) { +// reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "End Date: " + endDate + " is not a valid date")); +// } + if (beginDateParsed != null && endDateParsed != null && !beginDateParsed.before(endDateParsed)) { + reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "BeginDate \'" + new SimpleDateFormat("yyyy-MM-dd").format(beginDateParsed) + "\' is greater than EndDate \'" + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed) + "\'")); + } + + String institutionName=null; + if(repositoryIdentifier!="-1") + institutionName = usageStatsRepository.getInstitutionName(repositoryIdentifier); + + //if (!repositoryIdentifier.equals("")) { + // repoid = usageStatsRepository.getInstitutionID(repositoryIdentifier); +// if (repositoryIdentifier.equals("-1")) { +// reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "RepositoryIdentifier: " + repoID + " is not valid")); +// } + //} + String itemid = ""; + if (!itemIdentifier.equals("")) { + String[] split = itemIdentifier.split(":"); + switch (split[0].toLowerCase()) { + case "oid": + itemid = itemIdentifier; + break; + case "doi": + itemid = itemIdentifier; + break; +// case "openaire": +// itemid = itemIdentifier; +// break; + default: + reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "ItemIdentifier: " + itemIdentifier + " is not valid")); + itemid = "-1"; + } + } + if (itemid.equals("") && repoID.equals("")) { + reportExceptions.add(new SUSHI_Error_Model("3070", "Error", "Required Filter Missing", "usagecounts.openaire.eu", "ItemIdentifier or RepositoryIdentifier must be supplied")); + } + //if (reportName.equalsIgnoreCase("ar1")) { + if (!itemid.equals("-1") && !repositoryIdentifier.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) { + if (!itemid.equals("") && !repositoryIdentifier.equals("")) { + //if (dataType.equalsIgnoreCase("") || dataType.equalsIgnoreCase("article")) { + if (reportExceptions.size() == 0) { + //reportExceptions = null; + usageStatsRepository.executeItemIR(reportItems, repositoryIdentifier, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + } + + if (reportItems.isEmpty()) { + reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + } + // } else { + //reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + //} + } else if (!repoID.equals("")) { + if (reportExceptions.size() == 0) { + //reportExceptions = null; + usageStatsRepository.executeBatchItemsIR(reportItems, repositoryIdentifier, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + } + if (reportItems.isEmpty()) { + reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + } + } + } + if (repoID.equals("")) { + if (reportExceptions.isEmpty()) { + //reportExceptions = null; + usageStatsRepository.executeItemIR(reportItems, null, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + } + if (reportItems.isEmpty()) { + reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + } + } + + if (reportItems.isEmpty()) { + reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + } + //} else if (reportName.equals("")) { + // reportExceptions.add(new SUSHI_Error_Model("3050", "Error", "usagecounts.openaire.eu", "Report argument is missing", "You must supply a Report argument")); + //} else { + // reportExceptions.add(new SUSHI_Error_Model("3000", "Error", "usagecounts.openaire.eu", "Report " + reportName + " not supported", "Supported reports: AR1, IR1, RR1, BR1, BR2")); + //} + + if (reportExceptions.isEmpty()) { + reportExceptions = null; + } + + COUNTER_Item_Report reportResponse = new COUNTER_Item_Report(dateTime.format(formatter), customerID, "IR", "Item Report", institutionName, orgIdentifiers, + reportExceptions, reportFilters, reportItems); + log.info("Total report items " + reportItems.size()); + + if (reportItems.size() > compression_max_number_of_records) { + log.info("Compression due to " + reportItems.size()); + reportForCompression = true; + } + return reportResponse; + } + + @Override + public COUNTER_Dataset_Report buildReportDSR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate, + String endDate, List metricType, String dataType, String granularity) throws Exception { + + ZonedDateTime dateTime = ZonedDateTime.now(); // Gets the current date and time, with your default time-zone + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + List reportItems = new ArrayList<>(); + List reportExceptions = new ArrayList<>(); + List reportFilters = new ArrayList(); + List orgIdentifiers = new ArrayList<>(); + reportFilters.add(new Filter("BeginDate", beginDate)); + reportFilters.add(new Filter("EndDate", endDate)); + + if (!repositoryIdentifier.equals("")) { + repositoryIdentifier = usageStatsRepository.getInstitutionID(repositoryIdentifier); + } orgIdentifiers.add(new SUSHI_Org_Identifiers("Openaire", repositoryIdentifier)); @@ -559,9 +792,9 @@ public class SushiLiteServiceImpl implements SushiLiteService { } //if (reportName.equalsIgnoreCase("ar1")) { if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) { - if (!itemid.equals("")) { + if (!itemid.equals("") && !repoid.equals("")) { //if (dataType.equalsIgnoreCase("") || dataType.equalsIgnoreCase("article")) { - usageStatsRepository.executeItemIR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + usageStatsRepository.executeItemDSR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); if (reportItems.isEmpty()) { reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); } @@ -569,13 +802,19 @@ public class SushiLiteServiceImpl implements SushiLiteService { //reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); //} } else if (!repoid.equals("")) { - usageStatsRepository.executeBatchItemsIR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); - + usageStatsRepository.executeBatchItemsDSR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); if (reportItems.isEmpty()) { reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); } } } + if (repoid.equals("")) { + usageStatsRepository.executeItemDSR(reportItems, null, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + if (reportItems.isEmpty()) { + reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + } + } + // } // //} else if (reportName.equalsIgnoreCase("br1")) { // if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) { @@ -635,7 +874,6 @@ public class SushiLiteServiceImpl implements SushiLiteService { // if (!repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) { // usageStatsRepository.executeJournal(reportItems, repoid, dataType, beginDateParsed, endDateParsed, granularity); // } - if (reportItems.isEmpty()) { reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); } @@ -648,12 +886,11 @@ public class SushiLiteServiceImpl implements SushiLiteService { if (reportExceptions.size() == 0) { reportExceptions = null; } - COUNTER_Item_Report reportResponse = new COUNTER_Item_Report(dateTime.format(formatter), customerID, "IR", "Item Report", institutionName, orgIdentifiers, + COUNTER_Dataset_Report reportResponse = new COUNTER_Dataset_Report(dateTime.format(formatter), customerID, "DST", "Dataset Report", institutionName, orgIdentifiers, reportExceptions, reportFilters, reportItems); return reportResponse; } - // @Override // public COUNTER_Title_Report buildReportIR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate, // String endDate, String metricType, String dataType, String granularity) { @@ -731,45 +968,312 @@ public class SushiLiteServiceImpl implements SushiLiteService { // COUNTER_Title_Report reportTR = new COUNTER_Title_Report(dateTime.format(formatter), customerID, reportID, reportName, institutionName, institutionIdD, reportExceptions, reportFilters, reportItems); // return reportTR; // } + @Override public String displayReportPR(String customerID, String repositoryIdentifier, String beginDate, String endDate, String metricType, String dataType, String granularity) { ObjectMapper objectMapper = new ObjectMapper(); - log.info((beginDate + " " + endDate)); + COUNTER_Platform_Report report = buildReportPR(customerID, repositoryIdentifier, beginDate, endDate, metricType, dataType, granularity); + try { - return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(buildReportPR(customerID, repositoryIdentifier, beginDate, endDate, metricType, dataType, granularity)) + "
"; - } catch (JsonProcessingException ex) { + if (reportForCompression == false) { + return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(report) + "
"; + } else { + log.info((beginDate + " " + endDate)); + try { + java.sql.Timestamp timestamp1 = new java.sql.Timestamp(System.currentTimeMillis()); + //System.out.println("String start " + timestamp1); + log.info("Start building report..." + timestamp1); + //String outputname = "PR" + "_" + repositoryIdentifier.replace("_", "").replace(":", "") +"_"+ beginDate.replace("-", "") + "_" + endDate.replace("-", ""); + String outputname = "PR" + "_" + repositoryIdentifier.substring(repositoryIdentifier.indexOf(":") + 1) + "_" + beginDate.replace("-", "") + "_" + endDate.replace("-", ""); + String directory = new File(download_folder).getAbsolutePath(); + //writer.writeValue(new File(directory + "/" + outputname + ".json"), tmpReport); + BufferedWriter writer = new BufferedWriter(new FileWriter(directory + "/" + outputname + ".json")); + writer.write(objectMapper.writer().writeValueAsString(report)); + writer.close(); + + FileOutputStream fos = new FileOutputStream(directory + "/" + outputname + ".zip"); + ZipOutputStream zipOut = new ZipOutputStream(fos); + File fileToZip = new File(directory + "/" + outputname + ".json"); + FileInputStream fis = new FileInputStream(fileToZip); + ZipEntry zipEntry = new ZipEntry(fileToZip.getName()); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + zipOut.close(); + fis.close(); + fos.close(); + fileToZip.delete(); + + java.sql.Timestamp timestamp2 = new java.sql.Timestamp(System.currentTimeMillis()); + //System.out.println("String end " + timestamp2); + log.info("Report created..." + timestamp2); + + reportForCompression = false; + return new String(sushi_lite_server + "/download/" + outputname + ".zip"); + } catch (JsonProcessingException ex) { + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class.getName()).log(Level.SEVERE, null, ex); + } + } + } catch (Exception ex) { java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class.getName()).log(Level.SEVERE, null, ex); } + return null; } @Override public String displayReportPR_P1(String customerID, String repositoryIdentifier, String beginDate, - String endDate) { + String endDate + ) { ObjectMapper objectMapper = new ObjectMapper(); log.info((beginDate + " " + endDate)); try { return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(buildReportPR_P1(customerID, repositoryIdentifier, beginDate, endDate)) + "
"; + } catch (JsonProcessingException ex) { - java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class.getName()).log(Level.SEVERE, null, ex); + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class + .getName()).log(Level.SEVERE, null, ex); } return null; } @Override - public String displayReportIR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate, - String endDate, List metricType, String dataType, String granularity) { + public String displayReportIR(String customerID, String repositoryIdentifier, + String itemIdentifier, String beginDate, + String endDate, List metricType, + String dataType, String granularity + ) { + ObjectMapper objectMapper = new ObjectMapper(); - log.info((beginDate + " " + endDate)); try { - return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(buildReportIR(customerID, repositoryIdentifier, itemIdentifier, beginDate, endDate, metricType, dataType, granularity)) + "
"; + COUNTER_Item_Report report = buildReportIR(customerID, repositoryIdentifier, itemIdentifier, beginDate, endDate, metricType, dataType, granularity); + if (reportForCompression == false) { + return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(report) + "
"; + } else { + log.info((beginDate + " " + endDate)); + + try { + java.sql.Timestamp timestamp1 = new java.sql.Timestamp(System.currentTimeMillis()); + //System.out.println("String start " + timestamp1); + log.info("Start building report..." + timestamp1); + String outputname = "IR" + "_" + repositoryIdentifier.substring(repositoryIdentifier.indexOf(":") + 1) + "_" + beginDate.replace("-", "") + "_" + endDate.replace("-", ""); + String directory = new File(download_folder).getAbsolutePath(); + BufferedWriter writer = new BufferedWriter(new FileWriter(directory + "/" + outputname + ".json")); + writer.write(objectMapper.writer().writeValueAsString(report)); + writer.close(); + + FileOutputStream fos = new FileOutputStream(directory + "/" + outputname + ".zip"); + ZipOutputStream zipOut = new ZipOutputStream(fos); + File fileToZip = new File(directory + "/" + outputname + ".json"); + FileInputStream fis = new FileInputStream(fileToZip); + ZipEntry zipEntry = new ZipEntry(fileToZip.getName()); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + zipOut.close(); + fis.close(); + fos.close(); + fileToZip.delete(); + + java.sql.Timestamp timestamp2 = new java.sql.Timestamp(System.currentTimeMillis()); + //System.out.println("String end " + timestamp2); + log.info("Report created..." + timestamp2); + reportForCompression = false; + return new String(sushi_lite_server + "/download/" + outputname + ".zip"); + + } catch (Exception ex) { + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class + .getName()).log(Level.SEVERE, null, ex); + + } + + } } catch (Exception ex) { - java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class.getName()).log(Level.SEVERE, null, ex); + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class + .getName()).log(Level.SEVERE, null, ex); } return null; } + @Override + public String displayReportDSR(String customerID, String repositoryIdentifier, + String itemIdentifier, String beginDate, + String endDate, List metricType, + String dataType, String granularity + ) { + + ObjectMapper objectMapper = new ObjectMapper(); + try { + COUNTER_Dataset_Report report = buildReportDSR(customerID, repositoryIdentifier, itemIdentifier, beginDate, endDate, metricType, dataType, granularity); + if (reportForCompression == false) { + return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(report) + "
"; + } else { + log.info((beginDate + " " + endDate)); + + try { + java.sql.Timestamp timestamp1 = new java.sql.Timestamp(System.currentTimeMillis()); + //System.out.println("String start " + timestamp1); + log.info("Start building report..." + timestamp1); + String outputname = "DSR" + "_" + repositoryIdentifier.substring(repositoryIdentifier.indexOf(":") + 1) + "_" + beginDate.replace("-", "") + "_" + endDate.replace("-", ""); + String directory = new File(download_folder).getAbsolutePath(); + BufferedWriter writer = new BufferedWriter(new FileWriter(directory + "/" + outputname + ".json")); + writer.write(objectMapper.writer().writeValueAsString(report)); + writer.close(); + + FileOutputStream fos = new FileOutputStream(directory + "/" + outputname + ".zip"); + ZipOutputStream zipOut = new ZipOutputStream(fos); + File fileToZip = new File(directory + "/" + outputname + ".json"); + FileInputStream fis = new FileInputStream(fileToZip); + ZipEntry zipEntry = new ZipEntry(fileToZip.getName()); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + zipOut.close(); + fis.close(); + fos.close(); + fileToZip.delete(); + + java.sql.Timestamp timestamp2 = new java.sql.Timestamp(System.currentTimeMillis()); + //System.out.println("String end " + timestamp2); + log.info("Report created..." + timestamp2); + reportForCompression = false; + return new String(sushi_lite_server + "/download/" + outputname + ".zip"); + + } catch (Exception ex) { + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class + .getName()).log(Level.SEVERE, null, ex); + + } + + } + } catch (Exception ex) { + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class + .getName()).log(Level.SEVERE, null, ex); + } + return null; + } + @Override + public ArrayList buildMembersList() { + ArrayList consortiumMembers = usageStatsRepository.buildMembersList(); + return consortiumMembers; + } + + @Override + public String displayConsortiumMemberList() { + ObjectMapper objectMapper = new ObjectMapper(); + try { + return "
" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(buildMembersList()) + "
"; + + } catch (Exception ex) { + java.util.logging.Logger.getLogger(SushiLiteServiceImpl.class + .getName()).log(Level.SEVERE, null, ex); + } + return null; + + } +} + +class JSONUtil { + + public static String escape(String input) { + StringBuilder output = new StringBuilder(); + + for (int i = 0; i < input.length(); i++) { + char ch = input.charAt(i); + int chx = (int) ch; + + // let's not put any nulls in our strings + assert (chx != 0); + + if (ch == '\n') { + output.append("\\n"); + } else if (ch == '\t') { + output.append("\\t"); + } else if (ch == '\r') { + output.append("\\r"); + } else if (ch == '\\') { + output.append("\\\\"); + } else if (ch == '"') { + output.append("\\\""); + } else if (ch == '\b') { + output.append("\\b"); + } else if (ch == '\f') { + output.append("\\f"); + } else if (chx >= 0x10000) { + assert false : "Java stores as u16, so it should never give us a character that's bigger than 2 bytes. It literally can't."; + } else if (chx > 127) { + output.append(String.format("\\u%04x", chx)); + } else { + output.append(ch); + } + } + + return output.toString(); + } + + public static String unescape(String input) { + StringBuilder builder = new StringBuilder(); + + int i = 0; + while (i < input.length()) { + char delimiter = input.charAt(i); + i++; // consume letter or backslash + + if (delimiter == '\\' && i < input.length()) { + + // consume first after backslash + char ch = input.charAt(i); + i++; + + if (ch == '\\' || ch == '/' || ch == '"' || ch == '\'') { + builder.append(ch); + } else if (ch == 'n') { + builder.append('\n'); + } else if (ch == 'r') { + builder.append('\r'); + } else if (ch == 't') { + builder.append('\t'); + } else if (ch == 'b') { + builder.append('\b'); + } else if (ch == 'f') { + builder.append('\f'); + } else if (ch == 'u') { + + StringBuilder hex = new StringBuilder(); + + // expect 4 digits + if (i + 4 > input.length()) { + throw new RuntimeException("Not enough unicode digits! "); + } + for (char x : input.substring(i, i + 4).toCharArray()) { + if (!Character.isLetterOrDigit(x)) { + throw new RuntimeException("Bad character in unicode escape."); + } + hex.append(Character.toLowerCase(x)); + } + i += 4; // consume those four digits. + + int code = Integer.parseInt(hex.toString(), 16); + builder.append((char) code); + } else { + throw new RuntimeException("Illegal escape sequence: \\" + ch); + } + } else { // it's not a backslash, or it's the last character. + builder.append(delimiter); + } + } + return builder.toString(); + } } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index ea54386..21fdc52 100755 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -8,17 +8,20 @@ log4j.logger.org.springframework = INFO, S log4j.additivity.org.springframework = false log4j.appender.R=org.apache.log4j.RollingFileAppender -#log4j.appender.R.File=/var/log/dnet/usageStatsAPI/usageStatsAPI.log -log4j.appender.R.File=/Users/dpie/Desktop/usageStatsAPI.log -#log4j.appender.R.File=/Volumes/Zeus/dpie/Desktop/usageStatsAPI.log +#log4j.appender.R.File=/var/log/dnet/usageStatsAPI/usageStatsAPI_R5.log +#log4j.appender.R.File=/home/dimitris.pierrakos/Sushilite_R5/usageStatsAPI_R5.log +#log4j.appender.R.File=/Users/dpie/Desktop/usageStatsAPI_R5.log +log4j.appender.R.File=/srv/springboot/9880/usageStatsAPI_R5.log loglog4j.appender.R.MaxFileSize=10MB log4j.appender.R.MaxBackupIndex=10 log4j.appender.R.layout=org.apache.log4j.PatternLayout log4j.appender.R.layout.ConversionPattern= %d %p %t [%c] - %m%n log4j.appender.S=org.apache.log4j.RollingFileAppender -#log4j.appender.S.File=/var/log/dnet/usageStatsAPI/usageStatsAPI-spring.log -log4j.appender.S.File=/Users/dpie/Desktop/usageStatsAPI-spring.log -#log4j.appender.S.File=/Volumes/Zeus/dpie/Desktop/usageStatsAPI-spring.log +#log4j.appender.S.File=/var/log/dnet/usageStatsAPI/usageStatsAPI_R5-spring.log +#log4j.appender.S.File=/home/dimitris.pierrakos/Sushilite_R5/usageStatsAPI_R5-spring.log +#log4j.appender.S.File=/Users/dpie/Desktop/usageStatsAPI_R5-spring.log +#log4j.appender.S.File=/Volumes/Zeus/dpie/Desktop/usageStatsAPI_R5-spring.log +log4j.appender.S.File=/srv/springboot/9880/usageStatsAPI_R5-spring.log log4j.appender.S.MaxFileSize=10MB log4j.appender.S.MaxBackupIndex=10 log4j.appender.S.layout=org.apache.log4j.PatternLayout diff --git a/src/main/resources/static/index.html b/src/main/resources/static/index.html index 537c250..0059da9 100644 --- a/src/main/resources/static/index.html +++ b/src/main/resources/static/index.html @@ -69,12 +69,18 @@

Supported Reports


diff --git a/src/main/resources/static/sushilite/IR/index.html b/src/main/resources/static/sushilite/IR/index.html new file mode 100644 index 0000000..fe917d1 --- /dev/null +++ b/src/main/resources/static/sushilite/IR/index.html @@ -0,0 +1,241 @@ + + + + + + + + + + OpenAIRE SUSHI Lite Client + + + + + + + + +
+ + + + +
+
+
+
+
+
+
+
+
+

Report Request

+
+
+ Report Name: +
+
+ +
+
+
+
+ Release: +
+
+ +
+
+
+
+ Requestor: +
+
+ +
+
+

Report Filters

+
Date range
+

Valid date formats are yyyy-mm-dd or yyyy-mm. Default range is the last available month. +

+
+
+ Begin Date: +
+
+ +
+
+
+
+ End Date: +
+
+ +
+
+
Filters
+

Provide either a Repository Identifier or an Item Identifier
+ Identifier format: namespace:value
+ Valid namespace for Repository Identifier: openaire or opendoar.
+ Valid namespace for Item Identifier: openaire, doi or oid(for OAI-PMH). +

+
+
+ Repository Identifier: +
+
+ +
+
+
+
+ Item Identifier: +
+
+ +
+
+

Metric Type

+

Total_Item_Requests

+

Total_Item_Investigations

+

Unique_Item_Requests

+

Unique_Item_Investigations

+
+
+ Item Data Type: +
+
+ +
+
+ +

Report Attributes

+

Valid Granularity values: Monthly or Totals

+
+
+ Granularity: +
+
+ +
+
+ + + +
+
+
+
+ + +
+ + diff --git a/src/main/resources/static/sushilite/PR/index.html b/src/main/resources/static/sushilite/PR/index.html new file mode 100644 index 0000000..878c86e --- /dev/null +++ b/src/main/resources/static/sushilite/PR/index.html @@ -0,0 +1,237 @@ + + + + + + + + + + OpenAIRE SUSHI Lite Client + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+

Report Request

+
+
+ Report Name: +
+
+ +
+
+
+
+ Release: +
+
+ +
+
+
+
+ Requestor: +
+
+ +
+
+

Report Filters

+
Date range
+

Valid date formats are yyyy-mm-dd or yyyy-mm. Default range is the last available month. +

+
+
+ Begin Date: +
+
+ +
+
+
+
+ End Date: +
+
+ +
+
+ +

Provide the Repository Identifier
+ Identifier format: namespace:value
+ Valid namespace for Repository Identifier: openaire or opendoar.
+

+
+
+ Repository Identifier: +
+
+ +
+
+

Metric Type

+

Total_Item_Requests

+

Total_Item_Investigations

+

Unique_Item_Requests

+

Unique_Item_Investigations

+ +

Optional filter to only show results for a single item type, e.g. article, book, etc.

+
+
+ Item Data Type: +
+
+ +
+
+

Report Attributes

+

Valid Granularity values: Monthly or Totals

+
+
+ Granularity: +
+
+ +
+
+ + + + + +
+
+
+
+ + +
+ + diff --git a/src/main/resources/static/sushilite/PR_P1/index.html b/src/main/resources/static/sushilite/PR_P1/index.html new file mode 100644 index 0000000..ea40593 --- /dev/null +++ b/src/main/resources/static/sushilite/PR_P1/index.html @@ -0,0 +1,175 @@ + + + + + + + + + + OpenAIRE SUSHI Lite Client + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+

Report Request

+
+
+ Report Name: +
+
+ +
+
+
+
+ Release: +
+
+ +
+
+
+
+ Requestor: +
+
+ +
+
+

Report Filters

+
Date range
+

Valid date formats are yyyy-mm-dd or yyyy-mm. Default range is the last available month. +

+
+
+ Begin Date: +
+
+ +
+
+
+
+ End Date: +
+
+ +
+
+
Filters
+

Provide a Repository Identifier
+ Identifier format: namespace:value
+ Valid namespace for Repository Identifier: openaire or opendoar.
+

+
+
+ Repository Identifier: +
+
+ +
+
+ + + +
+
+
+
+
+ + +
+ + diff --git a/src/main/resources/static/sushilite/index.html b/src/main/resources/static/sushilite/index.html index 521fa71..b23ff08 100644 --- a/src/main/resources/static/sushilite/index.html +++ b/src/main/resources/static/sushilite/index.html @@ -69,12 +69,19 @@

Supported Reports


diff --git a/usageStatsAPI.properties b/usageStatsAPI.properties new file mode 100644 index 0000000..fc78772 --- /dev/null +++ b/usageStatsAPI.properties @@ -0,0 +1,21 @@ +name=usageStatsAPI +logging.config=classpath:log4j.properties +#usagestats.driverClassName=com.cloudera.impala.jdbc41.Driver +spring.datasource.driverClassName=com.cloudera.impala.jdbc41.Driver +spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.HSQLDialect +spring.datasource.hikari.max-lifetime=600000 +usagestats.url=jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;UseNativeQuery=1;AutoReconnect=1 +usagestats.redis.hostname=10.19.65.40 +usagestats.redis.port=6379 +usagestats.redis_scheme=tcp +spring.jackson.serialization.INDENT_OUTPUT=true +prod.statsdb=openaire_prod_stats +prod.usagestatsImpalaDB=openaire_prod_usage_stats +compression.max_number_of_records=100 +download.folder=/home/dimitris.pierrakos/Sushilite_R5/DownloadSushiReports +sushi-lite.server=https://beta.services.openaire.eu/usagestats_r5 +server.servlet.context-path=/usagestats_r5 +#server.port=8080 +#spring.datasource.testWhileIdle = true +#spring.datasource.timeBetweenEvictionRunsMillis = 60000 +#spring.datasource.validationQuery = SELECT 1 \ No newline at end of file diff --git a/usageStatsAPI_R5.properties b/usageStatsAPI_R5.properties new file mode 100644 index 0000000..fc78772 --- /dev/null +++ b/usageStatsAPI_R5.properties @@ -0,0 +1,21 @@ +name=usageStatsAPI +logging.config=classpath:log4j.properties +#usagestats.driverClassName=com.cloudera.impala.jdbc41.Driver +spring.datasource.driverClassName=com.cloudera.impala.jdbc41.Driver +spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.HSQLDialect +spring.datasource.hikari.max-lifetime=600000 +usagestats.url=jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;UseNativeQuery=1;AutoReconnect=1 +usagestats.redis.hostname=10.19.65.40 +usagestats.redis.port=6379 +usagestats.redis_scheme=tcp +spring.jackson.serialization.INDENT_OUTPUT=true +prod.statsdb=openaire_prod_stats +prod.usagestatsImpalaDB=openaire_prod_usage_stats +compression.max_number_of_records=100 +download.folder=/home/dimitris.pierrakos/Sushilite_R5/DownloadSushiReports +sushi-lite.server=https://beta.services.openaire.eu/usagestats_r5 +server.servlet.context-path=/usagestats_r5 +#server.port=8080 +#spring.datasource.testWhileIdle = true +#spring.datasource.timeBetweenEvictionRunsMillis = 60000 +#spring.datasource.validationQuery = SELECT 1 \ No newline at end of file diff --git a/src/main/resources/usageStatsAPI.properties b/usageStatsAPI_local.properties similarity index 68% rename from src/main/resources/usageStatsAPI.properties rename to usageStatsAPI_local.properties index 76d7a14..a57b4d5 100644 --- a/src/main/resources/usageStatsAPI.properties +++ b/usageStatsAPI_local.properties @@ -1,8 +1,9 @@ name=usageStatsAPI -logging.config=log4j.properties +logging.config=classpath:log4j.properties #usagestats.driverClassName=com.cloudera.impala.jdbc41.Driver spring.datasource.driverClassName=com.cloudera.impala.jdbc41.Driver spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.HSQLDialect +spring.datasource.hikari.max-lifetime=600000 usagestats.url=jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;UseNativeQuery=1;AutoReconnect=1 usagestats.redis.hostname=localhost usagestats.redis.port=6379 @@ -10,9 +11,9 @@ spring.jackson.serialization.INDENT_OUTPUT=true prod.statsdb=openaire_prod_stats prod.usagestatsImpalaDB=openaire_prod_usage_stats compression.max_number_of_records=100 -download.folder=/Users/dpie/Desktop/DownloadSushiLite -sushi-lite.server=http://localhost:8080/usagestats -server.servlet.context-path.=/usagestats_r5 +download.folder=/Users/dpie/Downloads/DownloadSushiLite +sushi-lite.server=http://localhost:8080/usagestats_r5 +server.servlet.context-path=/usagestats_r5 server.port=8080 #spring.datasource.testWhileIdle = true #spring.datasource.timeBetweenEvictionRunsMillis = 60000